Merge pull request #180

fix(export): 403 when exporting

.env.example

@@ -1,6 +1,7 @@
 SERVER_NAME=wygiwyh_server
 DB_NAME=wygiwyh_pg
-PROCRASTINATE_NAME=wygiwyh_procrastinate
+
+TZ=UTC # Change to your timezone. This only affects some async tasks.
 
 DEBUG=false
 URL = https://...
@@ -9,7 +10,6 @@ SECRET_KEY=<GENERATE A SAFE SECRET KEY AND PLACE IT HERE>
 DJANGO_ALLOWED_HOSTS=localhost 127.0.0.1 [::1]
 OUTBOUND_PORT=9005
 
-SQL_ENGINE=django.db.backends.postgresql
 SQL_DATABASE=wygiwyh
 SQL_USER=wygiwyh
 SQL_PASSWORD=<INSERT A SAFE PASSWORD HERE>
@@ -18,3 +18,11 @@ SQL_PORT=5432
 
 # Gunicorn
 WEB_CONCURRENCY=4
+
+# App Configs
+# Enable this if you want to keep deleted transactions in the database
+ENABLE_SOFT_DELETE=false
+# If ENABLE_SOFT_DELETE is true, transactions deleted for more than KEEP_DELETED_TRANSACTIONS_FOR days will be truly deleted. Set to 0 to keep all.
+KEEP_DELETED_TRANSACTIONS_FOR=365
+
+TASK_WORKERS=1 # This only work if you're using the single container option. Increase to have more open queues via procrastinate, you probably don't need to increase this.

.github/workflows/release.yml

@@ -2,7 +2,20 @@ name: Release Pipeline
 
 on:
   release:
-    types: [created]
+    types: [ created ]
+  push:
+    branches: [ main ]
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'Custom tag name for the image'
+        required: true
+        type: string
+      ref:
+        description: 'Git ref to checkout (branch, tag, or SHA)'
+        required: true
+        default: 'main'
+        type: string
 
 env:
   IMAGE_NAME: wygiwyh
@@ -16,6 +29,13 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.inputs.ref }}
+        if: github.event_name == 'workflow_dispatch'
+
+      - name: Checkout code (non-manual)
+        uses: actions/checkout@v4
+        if: github.event_name != 'workflow_dispatch'
 
       - name: Log in to Docker Hub
         uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
@@ -29,29 +49,49 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Cache Docker layers
-        uses: actions/cache@v3
-        with:
-          path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-buildx-
-
-      - name: Build and push image
+      - name: Build and push nightly image
+        if: github.event_name == 'push'
         uses: docker/build-push-action@v6
         with:
           context: .
           file: ./docker/prod/django/Dockerfile
           push: true
           provenance: false
+          build-args: |
+            VERSION=nightly
+          tags: ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.IMAGE_NAME }}:nightly
+          platforms: linux/amd64,linux/arm64
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Build and push release image
+        if: github.event_name == 'release'
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./docker/prod/django/Dockerfile
+          push: true
+          provenance: false
+          build-args: |
+            VERSION=${{ github.event.release.tag_name }}
           tags: |
             ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.IMAGE_NAME }}:latest
             ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.IMAGE_NAME }}:${{ github.event.release.tag_name }}
           platforms: linux/amd64,linux/arm64
-          cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
 
-      - name: Move cache
-        run: |
-          rm -rf /tmp/.buildx-cache
-          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+      - name: Build and push custom image
+        if: github.event_name == 'workflow_dispatch'
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./docker/prod/django/Dockerfile
+          push: true
+          provenance: false
+          build-args: |
+            VERSION=${{ github.event.inputs.tag }}
+          tags: ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.IMAGE_NAME }}:${{ github.event.inputs.tag }}
+          platforms: linux/amd64,linux/arm64
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

README.md

@@ -6,17 +6,21 @@
   <br>
 </h1>
 
-<h4 align="center">An optionated and powerful finance tracker.</h4>
+<h4 align="center">An opinionated and powerful finance tracker.</h4>
 
 <p align="center">
   <a href="#why-wygiwyh">Why</a> •
   <a href="#key-features">Features</a> •
   <a href="#how-to-use">Usage</a> •
-  <a href="#how-it-works">How</a>
+  <a href="#how-it-works">How</a> •
+  <a href="#caveats-and-warnings">Caveats and Warnings</a> •
+  <a href="#built-with">Built with</a>
 </p>
 
 **WYGIWYH** (_What You Get Is What You Have_) is a powerful, principles-first finance tracker designed for people who prefer a no-budget, straightforward approach to managing their money. With features like multi-currency support, customizable transactions, and a built-in dollar-cost averaging tracker, WYGIWYH helps you take control of your finances with simplicity and flexibility.
 
+<img src=".github/img/monthly_view.png" width="18%"></img> <img src=".github/img/yearly.png" width="18%"></img> <img src=".github/img/networth.png" width="18%"></img> <img src=".github/img/calendar.png" width="18%"></img> <img src=".github/img/all_transactions.png" width="18%"></img> 
+
 # Why WYGIWYH?
 Managing money can feel unnecessarily complex, but it doesn’t have to be. WYGIWYH (pronounced "wiggy-wih") is based on a simple principle:
 
@@ -53,10 +57,10 @@ To run this application, you'll need [Docker](https://docs.docker.com/engine/ins
 From your command line:
 
 ```bash
-# Clone this repository
+# Create a folder for WYGIWYH (optional)
 $ mkdir WYGIWYH
 
-# Go into the repository
+# Go into the folder
 $ cd WYGIWYH
 
 $ touch docker-compose.yml
@@ -75,277 +79,77 @@ $ docker compose up -d
 $ docker compose exec -it web python manage.py createsuperuser
 ```
 
-# How it works
+> [!NOTE]
+> If you're using Unraid, you don't need to follow these steps, use the app on the store. Make sure to read the [Unraid section](#unraid) and [Environment Variables](#environment-variables) for an explanation of all available variables
 
-## Models
+## Running locally
 
-### Transactions
+If you want to run WYGIWYH locally, on your env file:
 
-Transactions are the core feature of WYGIWYH, representing expenses or income in your accounts. Each transaction consists of the following fields:
+1. Remove `URL`
+2. Set `HTTPS_ENABLED` to `false`
+3. Leave the default `DJANGO_ALLOWED_HOSTS` (localhost 127.0.0.1 [::1])
 
-#### Type
-
-- **Income**: A positive amount entering your account
-- **Expense**: A negative amount exiting your account
-
-#### Paid Status
-
-A transaction can be either:
-
-- **Current**: When marked as paid
-- **Projected**: When marked as unpaid
-
-#### Account
-
-The account associated with the transaction. Required, limited to one account per transaction.
-
-#### Entity
-
-The party involved in the transaction:
-
-- For **Income**: The paying entity
-- For **Expense**: The receiving entity
-
-Optional field.
-
-#### Date
-
-The date when the transaction occurred. Required.
-
-#### Reference Date
-
-One of **WYGIWYH**'s key features. The reference date determines which month a transaction should count towards. For example, you can have a transaction that occurred on January 26th count towards February's finances.
-
-Optional - defaults to the transaction date's month if not specified.
-
-> [!CAUTION]
-> While designed primarily for credit card closing dates, this feature allows for debt rolling across months. Use responsibly to maintain accurate financial tracking.
-
-#### Type
-
-- Income, meaning a positive amount (usually) entering your account
-- Expense, meaning a negative amount exiting your account
-
-#### Description
-
-The name or purpose of the transaction. Required.
-
-#### Amount
-
-The monetary value of the transaction. Required.
-
-#### Category
-
-The primary classification of the transaction. Optional.
-
-#### Tags
-
-Additional labels for transaction categorization. Optional.
-
-#### Notes
-
-Additional information about the transaction. Optional.
-
-![img_4.png](.github/img/readme_transaction.png)
-
-### Installment Plan
-
-An Installment Plan is a helper model that generates a series of recurring transactions over a fixed period.
-
-#### Core Fields
-
-- **Account**: The account for all transactions in the plan. Required.
-- **Entity**: The paying or receiving party for all transactions. Optional.
-- **Description**: The name of the installment plan, used for all transactions. Required.
-- **Notes**: Additional information applied to all transactions. Optional.
-
-#### Installment Configuration
-
-- **Number of Installments**: Total number of transactions to create (e.g., 1/10, 2/10)
-- **Installment Start**: Initial counting point
-- **Start Date**: Date of the first transaction
-- **Reference Date**: Reference date for the first transaction
-- **Recurrence**: Frequency of transactions (e.g., Monthly)
-
-![img_1.png](.github/img/readme_installment_plan.png)
-
-### Transaction Details
-
-- **Amount**: Value for each transaction. Required.
-- **Category**: Primary classification for all transactions. Optional.
-- **Tags**: Labels applied to all transactions. Optional.
-
-### Recurring Transaction
-A Recurring Transaction is a helper model that generates recurring transactions indefinitely or until a certain date.
-
-#### Core Fields
-
-- **Account**: The account for all transactions in the plan. Required.
-- **Entity**: The paying or receiving party for all transactions. Optional.
-- **Description**: The name of the recurring transaction, used for all transactions. Required.
-- **Notes**: Additional information applied to all transactions. Optional.
-
-#### Recurring Transaction Configuration
-
-- **Start Date**: Date of the first transaction. Required.
-- **Reference Date**: Reference date for the first transaction. Optional.
-- **Recurrence Type**: Frequency of transactions (e.g., Monthly). Required.
-- **Recurrence Interval**: The interval between transactions (e.g. every 1 month, every 2 weeks, etc.). Required.
-- **End date**: When new transactions should stop being created. Optional.
-
-#### Transaction Details
-
-- **Amount**: Value for each transaction. Required.
-- **Category**: Primary classification for all transactions. Optional.
-- **Tags**: Labels applied to all transactions. Optional.
-
-#### Other information
-
-- Recurring transactions are checked and created every midnight using Procrastinate.
-- **WYGIWYH** tries to keep at most **6** future transactions created at any time.
-- If you delete a recurring transaction it will not be recreated.
-- You can stop or pause a recurring transaction at any time on the config page (/recurring-trasanctions/)
-
-![img_3.png](.github/img/readme_recurring_transaction.png)
-
-### Account
-
-TO-DO
-
-### Account Groups
-
-TO-DO
-
-### Currency
-
-TO-DO
-
-### Exchange Rate
-
-TO-DO
-
-### Category
-
-TO-DO
-
-### Tag
-
-TO-DO
-
-### Entity
-
-TO-DO
-
-### Rule
-
-TO-DO
-
----
-
-## Helper actions
-
-### Transfer
-
-A transfer happens when you move a monetary value from one account to another. This will create two transactions, one expense and one income with the values set by the user.
-
-Contrary to other finance trackers, due to our multi-currency support, **WYGIWYH**'s transfer system allows for non-zero transfers.
-
-![img.png](.github/img/readme_transfer.png)
-
-### Balance (Account Reconciliation)
-
-A balance is a easy way of updating your accounts balance. It creates a transaction with the difference between the balance currently in **WYGIWYH** and the new balance informed by you.
-
-This can be useful for savings accounts or other interest accruing investments.![img_2.png](.github/img/readme_balance.png)
-
----
-
-## Views
-
-### Monthly
-
-TO-DO
-
-### Yearly by currency
-
-TO-DO
-
-### Yearly by account
-
-TO-DO
-
-### Calendar
-
-TO-DO
-
-### Networh
-
-#### Current
-
-TO-DO
-
-#### Projected
-
-TO-DO
-
-### All Transactions
-
-TO-DO
-
-### Configuration and Management
-
-TO-DO
-
----
-
-## Tools
-
-### Calculator
-
-The calculator is a floating widget that can be toggled by clicking the calculator icon on the navbar.
-
-It allows for any math expression supported by [math.js](https://mathjs.org).
-
-![calculator](.github/img/readme_calculator.gif)
-
-### Dollar Cost Average Tracker
-
-The DCA Tracker can be accessed from the navbar's **Tools** menu.
-
-It allows for tracking DCA strategies and getting helpful information and insights.
-
-> [!IMPORTANT]
-> Currently DCA exists separately from your main transactions. You will need to add your entries manually.
-
-<img src=".github/img/readme_dca_1.png" width="45%"></img> <img src=".github/img/readme_dca_2.png" width="45%"></img>
-
-### Unit Price Calculator
-
-The Unit Price Calculator can be accessed from the navbar's **Tools** menu.
-
-This is a self-contained tool for comparing and finding the most cost-efficient item quickly and easily.
-
-Input the price and the amount of each item, the cheapeast will be highlighted in green, and the most expensive in red.
-
-You can add additional items by clicking the _Add_ button at the end of the page.
+You can now access localhost:OUTBOUND_PORT
 
 > [!NOTE]
-> This doesn't do unit convertion. The amount of all items needs to be on the same the unit for proper functioning.
+> - If you're planning on running this behind Tailscale or other similar service also add your machine given IP to `DJANGO_ALLOWED_HOSTS`
+> - If you're going to use another IP that isn't localhost, add it to `DJANGO_ALLOWED_HOSTS`, without `http://`
 
-![img.png](.github/img/readme_unit_price_calculator.png)
 
-### Currency Converter
+## Latest changes
+Features are only added to `main` when ready, if you want to run the latest version, you must build from source or use the `:nightly` tag on docker. Keep in mind that there can be undocumented breaking changes.
 
-TO-DO
+All the required Dockerfiles are [here](https://github.com/eitchtee/WYGIWYH/tree/main/docker/prod).
+
+## Unraid
+
+[nwithan8](https://github.com/nwithan8) has kindly provided a Unraid template for WYGIWYH, have a look at the [unraid_templates](https://github.com/nwithan8/unraid_templates) repo.
+
+WYGIWYH is available on the Unraid Store. You'll need to provision your own postgres (version 15 or up) database.
+
+To create the first user, open the container's console using Unraid's UI, by clicking on WYGIWYH icon on the Docker page and selecting `Console`, then type `python manage.py createsuperuser`, you'll them be prompted to input your e-mail and password.
+
+## Environment Variables
+
+| variable                      | type        | default                           | explanation                                                                                                                                                                                                                              |
+|-------------------------------|-------------|-----------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| DJANGO_ALLOWED_HOSTS          | string      | localhost 127.0.0.1               | A list of space separated domains and IPs representing the host/domain names that WYGIWYH site can serve. [Click here](https://docs.djangoproject.com/en/5.1/ref/settings/#allowed-hosts) for more details                               |
+| HTTPS_ENABLED                 | true\|false | false                             | Whether to use secure cookies. If this is set to true, the cookie will be marked as “secure”, which means browsers may ensure that the cookie is only sent under an HTTPS connection                                                     |
+|              URL              | string      | http://localhost http://127.0.0.1 | A list of space separated domains and IPs (with the protocol) representing the trusted origins for unsafe requests (e.g. POST). [Click here](https://docs.djangoproject.com/en/5.1/ref/settings/#csrf-trusted-origins ) for more details |
+| SECRET_KEY                    | string      | ""                                | This is used to provide cryptographic signing, and should be set to a unique, unpredictable value.                                                                                                                                       |
+| DEBUG                         | true\|false | false                             | Turns DEBUG mode on or off, this is useful to gather more data about possible errors you're having. Don't use in production.                                                                                                             |
+| SQL_DATABASE                  | string      | None *required                    | The name of your postgres database                                                                                                                                                                                                       |
+| SQL_USER                      | string      | user                              | The username used to connect to your postgres database                                                                                                                                                                                   |
+| SQL_PASSWORD                  | string      | password                          | The password used to connect to your postgres database                                                                                                                                                                                   |
+| SQL_HOST                      | string      | localhost                         | The address used to connect to your postgres database                                                                                                                                                                                    |
+| SQL_PORT                      | string      | 5432                              | The port used to connect to your postgres database                                                                                                                                                                                       |
+| SESSION_EXPIRY_TIME           | int         | 2678400 (31 days)                 | The age of session cookies, in seconds. E.g. how long you will stay logged in                                                                                                                                                            |
+| ENABLE_SOFT_DELETE            | true\|false | false                             | Whether to enable transactions soft delete, if enabled, deleted transactions will remain in the database. Useful for imports and avoiding duplicate entries.                                                                             |
+| KEEP_DELETED_TRANSACTIONS_FOR | int         | 365                               | Time in days to keep soft deleted transactions for. If 0, will keep all transactions indefinitely. Only works if ENABLE_SOFT_DELETE is true.                                                                                             |
+| TASK_WORKERS                  | int         | 1                                 | How many workers to have for async tasks. One should be enough for most use cases                                                                                                                                                        |
+
+# How it works
+
+Check out our [Wiki](https://github.com/eitchtee/WYGIWYH/wiki) for more information.
+
+# Caveats and Warnings
+
+- I'm not an accountant, some terms and even calculations might be wrong. Make sure to open an issue if you see anything that could be improved.
+- Pretty much all calculations are done at run time, this can lead to some performance degradation. On my personal instance, I have 3000+ transactions over 4+ years and 4000+ exchange rates, and load times average at around 500ms for each page, not bad overall.
+- This isn't a budgeting or double-entry-accounting application, if you need those features there's a lot of options out there, if you really need them in WYGIWYH, open a discussion.
 
 # Built with
 
 WYGIWYH is possible thanks to a lot of amazing open source tools, to name a few:
 
-- Django
-- HTMX
-- _hyperscript
-- Procrastinate
-- Bootstrap
-- Tailwind
-- Webpack
+* Django
+* HTMX
+* _hyperscript
+* Procrastinate
+* Bootstrap
+* Tailwind
+* Webpack
+* PostgreSQL
+* Django REST framework
+* Alpine.js

app/WYGIWYH/settings.py

@@ -26,15 +26,13 @@ ROOT_DIR = Path(__file__).resolve().parent.parent.parent
 # See https://docs.djangoproject.com/en/5.1/howto/deployment/checklist/
 
 # SECURITY WARNING: keep the secret key used in production secret!
-SECRET_KEY = "django-insecure-##6^&g49xwn7s67xc&33vf&=*4ibqfzn#xa*p-1sy8ag+zjjb9"
+SECRET_KEY = os.getenv("SECRET_KEY", "")
 
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = os.getenv("DEBUG", "false").lower() == "true"
 
-ALLOWED_HOSTS = os.environ.get("DJANGO_ALLOWED_HOSTS", "localhost 127.0.0.1").split(" ")
-CSRF_TRUSTED_ORIGINS = os.environ.get("URL", "http://localhost http://127.0.0.1").split(
-    " "
-)
+ALLOWED_HOSTS = os.getenv("DJANGO_ALLOWED_HOSTS", "localhost 127.0.0.1").split(" ")
+CSRF_TRUSTED_ORIGINS = os.getenv("URL", "http://localhost http://127.0.0.1").split(" ")
 
 # Application definition
 
@@ -57,6 +55,7 @@ INSTALLED_APPS = [
     "hijack",
     "hijack.contrib.admin",
     "django_filters",
+    "import_export",
     "apps.users.apps.UsersConfig",
     "procrastinate.contrib.django",
     "apps.transactions.apps.TransactionsConfig",
@@ -64,6 +63,8 @@ INSTALLED_APPS = [
     "apps.accounts.apps.AccountsConfig",
     "apps.common.apps.CommonConfig",
     "apps.net_worth.apps.NetWorthConfig",
+    "apps.import_app.apps.ImportConfig",
+    "apps.export_app.apps.ExportConfig",
     "apps.api.apps.ApiConfig",
     "cachalot",
     "rest_framework",
@@ -72,9 +73,12 @@ INSTALLED_APPS = [
     "apps.rules.apps.RulesConfig",
     "apps.calendar_view.apps.CalendarViewConfig",
     "apps.dca.apps.DcaConfig",
+    "pwa",
 ]
 
 MIDDLEWARE = [
+    "django_browser_reload.middleware.BrowserReloadMiddleware",
+    "apps.common.middleware.thread_local.ThreadLocalMiddleware",
     "debug_toolbar.middleware.DebugToolbarMiddleware",
     "django.middleware.security.SecurityMiddleware",
     "whitenoise.middleware.WhiteNoiseMiddleware",
@@ -86,7 +90,6 @@ MIDDLEWARE = [
     "apps.common.middleware.localization.LocalizationMiddleware",
     "django.contrib.messages.middleware.MessageMiddleware",
     "django.middleware.clickjacking.XFrameOptionsMiddleware",
-    "django_browser_reload.middleware.BrowserReloadMiddleware",
     "hijack.middleware.HijackUserMiddleware",
 ]
 
@@ -124,12 +127,12 @@ WSGI_APPLICATION = "WYGIWYH.wsgi.application"
 
 DATABASES = {
     "default": {
-        "ENGINE": os.environ.get("SQL_ENGINE", "django.db.backends.sqlite3"),
-        "NAME": os.environ.get("SQL_DATABASE", BASE_DIR / "db.sqlite3"),
-        "USER": os.environ.get("SQL_USER", "user"),
-        "PASSWORD": os.environ.get("SQL_PASSWORD", "password"),
-        "HOST": os.environ.get("SQL_HOST", "localhost"),
-        "PORT": "5432",
+        "ENGINE": "django.db.backends.postgresql",
+        "NAME": os.getenv("SQL_DATABASE"),
+        "USER": os.getenv("SQL_USER", "user"),
+        "PASSWORD": os.getenv("SQL_PASSWORD", "password"),
+        "HOST": os.getenv("SQL_HOST", "localhost"),
+        "PORT": os.getenv("SQL_PORT", "5432"),
     }
 }
 
@@ -161,10 +164,11 @@ AUTH_USER_MODEL = "users.User"
 LANGUAGE_CODE = "en"
 LANGUAGES = (
     ("en", "English"),
+    ("nl", "Nederlands"),
     ("pt-br", "Português (Brasil)"),
 )
 
-TIME_ZONE = "UTC"
+TIME_ZONE = os.getenv("TZ", "UTC")
 
 USE_I18N = True
 
@@ -218,7 +222,7 @@ SESSION_COOKIE_SECURE = os.getenv("HTTPS_ENABLED", "false").lower() == "true"
 
 DEBUG_TOOLBAR_CONFIG = {
     "ROOT_TAG_EXTRA_ATTRS": "hx-preserve",
-    "SHOW_TOOLBAR_CALLBACK": lambda r: False,  # disables it}
+    # "SHOW_TOOLBAR_CALLBACK": lambda r: False,  # disables it
 }
 DEBUG_TOOLBAR_PANELS = [
     "debug_toolbar.panels.history.HistoryPanel",
@@ -275,29 +279,32 @@ if "procrastinate" in sys.argv:
         "version": 1,
         "disable_existing_loggers": False,
         "formatters": {
-            "procrastinate": {
-                "format": "%(asctime)s %(levelname)-7s %(name)s %(message)s"
+            "standard": {
+                "format": "[%(asctime)s] - %(levelname)s - %(name)s - %(message)s",
+                "datefmt": "%Y-%m-%d %H:%M:%S",
             },
         },
         "handlers": {
             "procrastinate": {
-                "level": "DEBUG",
+                "level": "INFO",
                 "class": "logging.StreamHandler",
-                "formatter": "procrastinate",
+                "formatter": "standard",
             },
             "console": {
                 "class": "logging.StreamHandler",
+                "formatter": "standard",
+                "level": "INFO",
             },
         },
         "loggers": {
             "procrastinate": {
                 "handlers": ["procrastinate"],
-                "level": "INFO",
                 "propagate": False,
             },
             "root": {
                 "handlers": ["console"],
                 "level": "INFO",
+                "propagate": False,
             },
         },
     }
@@ -306,24 +313,25 @@ else:
         "version": 1,
         "disable_existing_loggers": False,
         "formatters": {
-            "procrastinate": {
-                "format": "%(asctime)s %(levelname)-7s %(name)s %(message)s"
+            "standard": {
+                "format": "[%(asctime)s] - %(levelname)s - %(name)s - %(message)s",
+                "datefmt": "%Y-%m-%d %H:%M:%S",
             },
         },
         "handlers": {
-            "procrastinate": {
-                "level": "DEBUG",
-                "class": "logging.StreamHandler",
-                "formatter": "procrastinate",
-            },
             "console": {
                 "class": "logging.StreamHandler",
+                "formatter": "standard",
+                "level": "INFO",
+            },
+            "procrastinate": {
+                "level": "INFO",
+                "class": "logging.StreamHandler",
             },
         },
         "loggers": {
             "procrastinate": {
                 "handlers": None,
-                "level": "INFO",
                 "propagate": False,
             },
             "root": {
@@ -334,3 +342,54 @@ else:
     }
 
 CACHALOT_UNCACHABLE_TABLES = ("django_migrations", "procrastinate_jobs")
+
+
+# PWA
+PWA_APP_NAME = SITE_TITLE
+PWA_APP_DESCRIPTION = "A simple and powerful finance tracker"
+PWA_APP_THEME_COLOR = "#fbb700"
+PWA_APP_BACKGROUND_COLOR = "#222222"
+PWA_APP_DISPLAY = "standalone"
+PWA_APP_SCOPE = "/"
+PWA_APP_ORIENTATION = "any"
+PWA_APP_START_URL = "/"
+PWA_APP_STATUS_BAR_COLOR = "default"
+PWA_APP_ICONS = [
+    {"src": "/static/img/favicon/android-icon-192x192.png", "sizes": "192x192"}
+]
+PWA_APP_ICONS_APPLE = [
+    {"src": "/static/img/favicon/apple-icon-180x180.png", "sizes": "180x180"}
+]
+PWA_APP_SPLASH_SCREEN = [
+    {
+        "src": "/static/img/pwa/splash-640x1136.png",
+        "media": "(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2)",
+    }
+]
+PWA_APP_DIR = "ltr"
+PWA_APP_LANG = "en-US"
+PWA_APP_SHORTCUTS = [
+    {
+        "name": "New Transaction",
+        "url": "/add/",
+        "description": "Add new transaction",
+    }
+]
+PWA_APP_SCREENSHOTS = [
+    {
+        "src": "/static/img/pwa/splash-750x1334.png",
+        "sizes": "750x1334",
+        "type": "image/png",
+        "form_factor": "wide",
+    },
+    {
+        "src": "/static/img/pwa/splash-750x1334.png",
+        "sizes": "750x1334",
+        "type": "image/png",
+    },
+]
+PWA_SERVICE_WORKER_PATH = BASE_DIR / "templates" / "pwa" / "serviceworker.js"
+
+ENABLE_SOFT_DELETE = os.getenv("ENABLE_SOFT_DELETE", "false").lower() == "true"
+KEEP_DELETED_TRANSACTIONS_FOR = int(os.getenv("KEEP_DELETED_ENTRIES_FOR", "365"))
+APP_VERSION = os.getenv("APP_VERSION", "unknown")

app/WYGIWYH/urls.py

@@ -27,6 +27,7 @@ urlpatterns = [
     path("hijack/", include("hijack.urls")),
     path("__debug__/", include("debug_toolbar.urls")),
     path("__reload__/", include("django_browser_reload.urls")),
+    path("", include("pwa.urls")),
     # path("api/", include("rest_framework.urls")),
     path("api/", include("apps.api.urls")),
     path("api/schema/", SpectacularAPIView.as_view(), name="schema"),
@@ -47,4 +48,7 @@ urlpatterns = [
     path("", include("apps.calendar_view.urls")),
     path("", include("apps.dca.urls")),
     path("", include("apps.mini_tools.urls")),
+    path("", include("apps.import_app.urls")),
+    path("", include("apps.export_app.urls")),
+    path("", include("apps.insights.urls")),
 ]

app/apps/accounts/forms.py

@@ -53,6 +53,7 @@ class AccountGroupForm(forms.ModelForm):
 
 class AccountForm(forms.ModelForm):
     group = DynamicModelChoiceField(
+        create_field="name",
         label=_("Group"),
         model=AccountGroup,
         required=False,
@@ -112,6 +113,7 @@ class AccountBalanceForm(forms.Form):
         max_digits=42, decimal_places=30, required=False, label=_("New balance")
     )
     category = DynamicModelChoiceField(
+        create_field="name",
         model=TransactionCategory,
         required=False,
         label=_("Category"),

app/apps/accounts/migrations/0007_make_account_names_unique.py

@@ -0,0 +1,38 @@
+from django.db import migrations, models
+
+
+def make_names_unique(apps, schema_editor):
+    Account = apps.get_model("accounts", "Account")
+
+    # Get all accounts ordered by id
+    accounts = Account.objects.all().order_by("id")
+
+    # Track seen names
+    seen_names = {}
+
+    for account in accounts:
+        original_name = account.name
+        counter = seen_names.get(original_name, 0)
+
+        while account.name in seen_names:
+            counter += 1
+            account.name = f"{original_name} ({counter})"
+
+        seen_names[account.name] = counter
+        account.save()
+
+
+def reverse_migration(apps, schema_editor):
+    # Can't restore original names, so do nothing
+    pass
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("accounts", "0006_rename_archived_account_is_archived_and_more"),
+    ]
+
+    operations = [
+        migrations.RunPython(make_names_unique, reverse_migration),
+    ]

app/apps/accounts/migrations/0008_alter_account_name.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.1.5 on 2025-01-24 00:54
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('accounts', '0007_make_account_names_unique'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='account',
+            name='name',
+            field=models.CharField(max_length=255, unique=True, verbose_name='Name'),
+        ),
+    ]

app/apps/accounts/models.py

@@ -18,7 +18,7 @@ class AccountGroup(models.Model):
 
 
 class Account(models.Model):
-    name = models.CharField(max_length=255, verbose_name=_("Name"))
+    name = models.CharField(max_length=255, verbose_name=_("Name"), unique=True)
     group = models.ForeignKey(
         AccountGroup,
         on_delete=models.SET_NULL,

app/apps/accounts/views/account_groups.py

@@ -2,9 +2,7 @@ from django.contrib import messages
 from django.contrib.auth.decorators import login_required
 from django.http import HttpResponse
 from django.shortcuts import render, get_object_or_404
-from django.urls import reverse
 from django.utils.translation import gettext_lazy as _
-from django.views.decorators.csrf import csrf_exempt
 from django.views.decorators.http import require_http_methods
 
 from apps.accounts.forms import AccountGroupForm
@@ -89,7 +87,6 @@ def account_group_edit(request, pk):
 
 @only_htmx
 @login_required
-@csrf_exempt
 @require_http_methods(["DELETE"])
 def account_group_delete(request, pk):
     account_group = get_object_or_404(AccountGroup, id=pk)

app/apps/accounts/views/accounts.py

@@ -2,9 +2,7 @@ from django.contrib import messages
 from django.contrib.auth.decorators import login_required
 from django.http import HttpResponse
 from django.shortcuts import render, get_object_or_404
-from django.urls import reverse
 from django.utils.translation import gettext_lazy as _
-from django.views.decorators.csrf import csrf_exempt
 from django.views.decorators.http import require_http_methods
 
 from apps.accounts.forms import AccountForm
@@ -89,7 +87,6 @@ def account_edit(request, pk):
 
 @only_htmx
 @login_required
-@csrf_exempt
 @require_http_methods(["DELETE"])
 def account_delete(request, pk):
     account = get_object_or_404(Account, id=pk)

app/apps/api/fields/transactions.py

@@ -1,6 +1,7 @@
 from drf_spectacular.types import OpenApiTypes
 from drf_spectacular.utils import extend_schema_field
 from rest_framework import serializers
+from django.utils.translation import gettext_lazy as _
 
 from apps.transactions.models import (
     TransactionCategory,
@@ -25,13 +26,13 @@ class TransactionCategoryField(serializers.Field):
                 return TransactionCategory.objects.get(pk=data)
             except TransactionCategory.DoesNotExist:
                 raise serializers.ValidationError(
-                    "Category with this ID does not exist."
+                    _("Category with this ID does not exist.")
                 )
         elif isinstance(data, str):
             category, created = TransactionCategory.objects.get_or_create(name=data)
             return category
         raise serializers.ValidationError(
-            "Invalid category data. Provide an ID or name."
+            _("Invalid category data. Provide an ID or name.")
         )
 
     @staticmethod
@@ -61,13 +62,13 @@ class TransactionTagField(serializers.Field):
                     tag = TransactionTag.objects.get(pk=item)
                 except TransactionTag.DoesNotExist:
                     raise serializers.ValidationError(
-                        f"Tag with ID {item} does not exist."
+                        _("Tag with this ID does not exist.")
                     )
             elif isinstance(item, str):
                 tag, created = TransactionTag.objects.get_or_create(name=item)
             else:
                 raise serializers.ValidationError(
-                    "Invalid tag data. Provide an ID or name."
+                    _("Invalid tag data. Provide an ID or name.")
                 )
             tags.append(tag)
         return tags
@@ -85,13 +86,13 @@ class TransactionEntityField(serializers.Field):
                     entity = TransactionEntity.objects.get(pk=item)
                 except TransactionTag.DoesNotExist:
                     raise serializers.ValidationError(
-                        f"Entity with ID {item} does not exist."
+                        _("Entity with this ID does not exist.")
                     )
             elif isinstance(item, str):
                 entity, created = TransactionEntity.objects.get_or_create(name=item)
             else:
                 raise serializers.ValidationError(
-                    "Invalid entity data. Provide an ID or name."
+                    _("Invalid entity data. Provide an ID or name.")
                 )
             entities.append(entity)
         return entities

app/apps/api/serializers/__init__.py

@@ -1,3 +1,4 @@
 from .transactions import *
 from .accounts import *
 from .currencies import *
+from .dca import *

app/apps/api/serializers/accounts.py

@@ -1,4 +1,5 @@
 from rest_framework import serializers
+from rest_framework.permissions import IsAuthenticated
 
 from apps.api.serializers.currencies import CurrencySerializer
 from apps.accounts.models import AccountGroup, Account
@@ -6,6 +7,8 @@ from apps.currencies.models import Currency
 
 
 class AccountGroupSerializer(serializers.ModelSerializer):
+    permission_classes = [IsAuthenticated]
+
     class Meta:
         model = AccountGroup
         fields = "__all__"
@@ -31,6 +34,8 @@ class AccountSerializer(serializers.ModelSerializer):
         allow_null=True,
     )
 
+    permission_classes = [IsAuthenticated]
+
     class Meta:
         model = Account
         fields = [

app/apps/api/serializers/currencies.py

@@ -1,8 +1,12 @@
 from rest_framework import serializers
+from rest_framework.permissions import IsAuthenticated
+
 from apps.currencies.models import Currency, ExchangeRate
 
 
 class CurrencySerializer(serializers.ModelSerializer):
+    permission_classes = [IsAuthenticated]
+
     class Meta:
         model = Currency
         fields = "__all__"
@@ -24,6 +28,8 @@ class ExchangeRateSerializer(serializers.ModelSerializer):
         queryset=Currency.objects.all(), source="to_currency", write_only=True
     )
 
+    permission_classes = [IsAuthenticated]
+
     class Meta:
         model = ExchangeRate
         fields = "__all__"

app/apps/api/serializers/dca.py

@@ -0,0 +1,85 @@
+from rest_framework import serializers
+from rest_framework.permissions import IsAuthenticated
+
+from apps.dca.models import DCAEntry, DCAStrategy
+
+
+class DCAEntrySerializer(serializers.ModelSerializer):
+    profit_loss = serializers.DecimalField(
+        max_digits=42, decimal_places=30, read_only=True
+    )
+    profit_loss_percentage = serializers.DecimalField(
+        max_digits=42, decimal_places=30, read_only=True
+    )
+    current_value = serializers.DecimalField(
+        max_digits=42, decimal_places=30, read_only=True
+    )
+    entry_price = serializers.DecimalField(
+        max_digits=42, decimal_places=30, read_only=True
+    )
+
+    permission_classes = [IsAuthenticated]
+
+    class Meta:
+        model = DCAEntry
+        fields = [
+            "id",
+            "strategy",
+            "date",
+            "amount_paid",
+            "amount_received",
+            "notes",
+            "created_at",
+            "updated_at",
+            "profit_loss",
+            "profit_loss_percentage",
+            "current_value",
+            "entry_price",
+        ]
+        read_only_fields = ["created_at", "updated_at"]
+
+
+class DCAStrategySerializer(serializers.ModelSerializer):
+    entries = DCAEntrySerializer(many=True, read_only=True)
+    total_invested = serializers.DecimalField(
+        max_digits=42, decimal_places=30, read_only=True
+    )
+    total_received = serializers.DecimalField(
+        max_digits=42, decimal_places=30, read_only=True
+    )
+    average_entry_price = serializers.DecimalField(
+        max_digits=42, decimal_places=30, read_only=True
+    )
+    total_entries = serializers.IntegerField(read_only=True)
+    current_total_value = serializers.DecimalField(
+        max_digits=42, decimal_places=30, read_only=True
+    )
+    total_profit_loss = serializers.DecimalField(
+        max_digits=42, decimal_places=30, read_only=True
+    )
+    total_profit_loss_percentage = serializers.DecimalField(
+        max_digits=42, decimal_places=30, read_only=True
+    )
+
+    permission_classes = [IsAuthenticated]
+
+    class Meta:
+        model = DCAStrategy
+        fields = [
+            "id",
+            "name",
+            "target_currency",
+            "payment_currency",
+            "notes",
+            "created_at",
+            "updated_at",
+            "entries",
+            "total_invested",
+            "total_received",
+            "average_entry_price",
+            "total_entries",
+            "current_total_value",
+            "total_profit_loss",
+            "total_profit_loss_percentage",
+        ]
+        read_only_fields = ["created_at", "updated_at"]

app/apps/api/serializers/transactions.py

@@ -19,6 +19,7 @@ from apps.transactions.models import (
     TransactionTag,
     InstallmentPlan,
     TransactionEntity,
+    RecurringTransaction,
 )
 
 
@@ -47,11 +48,82 @@ class TransactionEntitySerializer(serializers.ModelSerializer):
 
 
 class InstallmentPlanSerializer(serializers.ModelSerializer):
+    category = TransactionCategoryField(required=False)
+    tags = TransactionTagField(required=False)
+    entities = TransactionEntityField(required=False)
+
     permission_classes = [IsAuthenticated]
 
     class Meta:
         model = InstallmentPlan
-        fields = "__all__"
+        fields = [
+            "id",
+            "account",
+            "type",
+            "description",
+            "number_of_installments",
+            "installment_start",
+            "installment_total_number",
+            "start_date",
+            "reference_date",
+            "end_date",
+            "recurrence",
+            "installment_amount",
+            "category",
+            "tags",
+            "entities",
+            "notes",
+        ]
+        read_only_fields = ["installment_total_number", "end_date"]
+
+    def create(self, validated_data):
+        instance = super().create(validated_data)
+        instance.create_transactions()
+        return instance
+
+    def update(self, instance, validated_data):
+        instance = super().update(instance, validated_data)
+        instance.update_transactions()
+        return instance
+
+
+class RecurringTransactionSerializer(serializers.ModelSerializer):
+    category = TransactionCategoryField(required=False)
+    tags = TransactionTagField(required=False)
+    entities = TransactionEntityField(required=False)
+
+    class Meta:
+        model = RecurringTransaction
+        fields = [
+            "id",
+            "is_paused",
+            "account",
+            "type",
+            "amount",
+            "description",
+            "category",
+            "tags",
+            "entities",
+            "notes",
+            "reference_date",
+            "start_date",
+            "end_date",
+            "recurrence_type",
+            "recurrence_interval",
+            "last_generated_date",
+            "last_generated_reference_date",
+        ]
+        read_only_fields = ["last_generated_date", "last_generated_reference_date"]
+
+    def create(self, validated_data):
+        instance = super().create(validated_data)
+        instance.create_upcoming_transactions()
+        return instance
+
+    def update(self, instance, validated_data):
+        instance = super().update(instance, validated_data)
+        instance.update_unpaid_transactions()
+        return instance
 
 
 class TransactionSerializer(serializers.ModelSerializer):

app/apps/api/urls.py

@@ -9,10 +9,13 @@ router.register(r"categories", views.TransactionCategoryViewSet)
 router.register(r"tags", views.TransactionTagViewSet)
 router.register(r"entities", views.TransactionEntityViewSet)
 router.register(r"installment-plans", views.InstallmentPlanViewSet)
+router.register(r"recurring-transactions", views.RecurringTransactionViewSet)
 router.register(r"account-groups", views.AccountGroupViewSet)
 router.register(r"accounts", views.AccountViewSet)
 router.register(r"currencies", views.CurrencyViewSet)
 router.register(r"exchange-rates", views.ExchangeRateViewSet)
+router.register(r"dca/strategies", views.DCAStrategyViewSet)
+router.register(r"dca/entries", views.DCAEntryViewSet)
 
 urlpatterns = [
     path("", include(router.urls)),

app/apps/api/views/__init__.py

@@ -1,3 +1,4 @@
 from .transactions import *
 from .accounts import *
 from .currencies import *
+from .dca import *

app/apps/api/views/dca.py

@@ -0,0 +1,41 @@
+from rest_framework import viewsets
+from rest_framework.decorators import action
+from rest_framework.response import Response
+from apps.dca.models import DCAStrategy, DCAEntry
+from apps.api.serializers import DCAStrategySerializer, DCAEntrySerializer
+
+
+class DCAStrategyViewSet(viewsets.ModelViewSet):
+    queryset = DCAStrategy.objects.all()
+    serializer_class = DCAStrategySerializer
+
+    @action(detail=True, methods=["get"])
+    def investment_frequency(self, request, pk=None):
+        strategy = self.get_object()
+        return Response(strategy.investment_frequency_data())
+
+    @action(detail=True, methods=["get"])
+    def price_comparison(self, request, pk=None):
+        strategy = self.get_object()
+        return Response(strategy.price_comparison_data())
+
+    @action(detail=True, methods=["get"])
+    def current_price(self, request, pk=None):
+        strategy = self.get_object()
+        price_data = strategy.current_price()
+        if price_data:
+            price, date = price_data
+            return Response({"price": price, "date": date})
+        return Response({"price": None, "date": None})
+
+
+class DCAEntryViewSet(viewsets.ModelViewSet):
+    queryset = DCAEntry.objects.all()
+    serializer_class = DCAEntrySerializer
+
+    def get_queryset(self):
+        queryset = DCAEntry.objects.all()
+        strategy_id = self.request.query_params.get("strategy", None)
+        if strategy_id is not None:
+            queryset = queryset.filter(strategy_id=strategy_id)
+        return queryset

app/apps/api/views/transactions.py

@@ -1,4 +1,4 @@
-from rest_framework import permissions, viewsets
+from rest_framework import viewsets
 
 from apps.api.serializers import (
     TransactionSerializer,
@@ -6,6 +6,7 @@ from apps.api.serializers import (
     TransactionTagSerializer,
     InstallmentPlanSerializer,
     TransactionEntitySerializer,
+    RecurringTransactionSerializer,
 )
 from apps.transactions.models import (
     Transaction,
@@ -13,6 +14,7 @@ from apps.transactions.models import (
     TransactionTag,
     InstallmentPlan,
     TransactionEntity,
+    RecurringTransaction,
 )
 from apps.rules.signals import transaction_updated, transaction_created
 
@@ -53,10 +55,7 @@ class InstallmentPlanViewSet(viewsets.ModelViewSet):
     queryset = InstallmentPlan.objects.all()
     serializer_class = InstallmentPlanSerializer
 
-    def perform_create(self, serializer):
-        instance = serializer.save()
-        instance.create_transactions()
 
-    def perform_update(self, serializer):
-        instance = serializer.save()
-        instance.create_transactions()
+class RecurringTransactionViewSet(viewsets.ModelViewSet):
+    queryset = RecurringTransaction.objects.all()
+    serializer_class = RecurringTransactionSerializer

app/apps/common/fields/forms/dynamic_select.py

@@ -1,6 +1,7 @@
 from django import forms
 from django.core.exceptions import ValidationError
 from django.db import transaction
+from django.utils.translation import gettext_lazy as _
 
 from apps.common.widgets.tom_select import TomSelect, TomSelectMultiple
 
@@ -8,18 +9,22 @@ from apps.common.widgets.tom_select import TomSelect, TomSelectMultiple
 class DynamicModelChoiceField(forms.ModelChoiceField):
     def __init__(self, model, *args, **kwargs):
         self.model = model
+        self.to_field_name = kwargs.pop("to_field_name", "pk")
+
+        self.create_field = kwargs.pop("create_field", None)
+
         self.queryset = kwargs.pop("queryset", model.objects.all())
-        super().__init__(queryset=self.queryset, *args, **kwargs)
-        self._created_instance = None
 
         self.widget = TomSelect(clear_button=True, create=True)
 
+        super().__init__(queryset=self.queryset, *args, **kwargs)
+        self._created_instance = None
+
     def to_python(self, value):
         if value in self.empty_values:
             return None
         try:
-            key = self.to_field_name or "pk"
-            return self.model.objects.get(**{key: value})
+            return self.model.objects.get(**{self.to_field_name: value})
         except (ValueError, TypeError, self.model.DoesNotExist):
             return value  # Return the raw value; we'll handle creation in clean()
 
@@ -47,12 +52,19 @@ class DynamicModelChoiceField(forms.ModelChoiceField):
                 else:
                     raise self.model.DoesNotExist
             except self.model.DoesNotExist:
-                try:
-                    with transaction.atomic():
-                        instance = self.model.objects.create(name=value)
-                        self._created_instance = instance
-                        return instance
-                except Exception as e:
+                if self.create_field:
+                    try:
+                        with transaction.atomic():
+                            instance, _ = self.model.objects.update_or_create(
+                                **{self.create_field: value}
+                            )
+                            self._created_instance = instance
+                            return instance
+                    except Exception as e:
+                        raise ValidationError(
+                            self.error_messages["invalid_choice"], code="invalid_choice"
+                        )
+                else:
                     raise ValidationError(
                         self.error_messages["invalid_choice"], code="invalid_choice"
                     )
@@ -111,12 +123,12 @@ class DynamicModelMultipleChoiceField(forms.ModelMultipleChoiceField):
         """
         try:
             with transaction.atomic():
-                new_instance = self.queryset.model(**{self.create_field: value})
-                new_instance.full_clean()
-                new_instance.save()
-            return new_instance
+                instance, _ = self.model.objects.update_or_create(
+                    **{self.create_field: value}
+                )
+            return instance
         except Exception as e:
-            raise ValidationError(f"Error creating new instance: {str(e)}")
+            raise ValidationError(_("Error creating new instance"))
 
     def clean(self, value):
         """
@@ -152,6 +164,6 @@ class DynamicModelMultipleChoiceField(forms.ModelMultipleChoiceField):
             try:
                 new_objects.append(self._create_new_instance(new_value))
             except ValidationError as e:
-                raise ValidationError(f"Error creating '{new_value}': {str(e)}")
+                raise ValidationError(_("Error creating new instance"))
 
         return existing_objects + new_objects

app/apps/common/fields/month_year.py

@@ -1,9 +1,11 @@
 import datetime
 
 from django import forms
-from django.db import models
 from django.core.exceptions import ValidationError
+from django.db import models
+from django.utils.translation import gettext_lazy as _
 
+from apps.common.widgets.datepicker import AirMonthYearPickerInput
 from apps.common.widgets.month_year import MonthYearWidget
 
 
@@ -18,7 +20,7 @@ class MonthYearModelField(models.DateField):
             # Set the day to 1
             return date.replace(day=1).date()
         except ValueError:
-            raise ValidationError("Invalid date format. Use YYYY-MM.")
+            raise ValidationError(_("Invalid date format. Use YYYY-MM."))
 
     def formfield(self, **kwargs):
         kwargs["widget"] = MonthYearWidget
@@ -27,7 +29,7 @@ class MonthYearModelField(models.DateField):
 
 
 class MonthYearFormField(forms.DateField):
-    widget = MonthYearWidget
+    widget = AirMonthYearPickerInput
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -42,7 +44,11 @@ class MonthYearFormField(forms.DateField):
             date = datetime.datetime.strptime(value, "%Y-%m")
             return date.replace(day=1).date()
         except ValueError:
-            raise ValidationError(_("Invalid date format. Use YYYY-MM."))
+            try:
+                date = datetime.datetime.strptime(value, "%Y-%m-%d")
+                return date.replace(day=1).date()
+            except ValueError:
+                raise ValidationError(_("Invalid date format. Use YYYY-MM."))
 
     def prepare_value(self, value):
         if isinstance(value, datetime.date):

app/apps/common/functions/format.py

@@ -0,0 +1,31 @@
+from apps.common.middleware.thread_local import get_current_user
+from django.utils.formats import get_format as original_get_format
+
+
+def get_format(format_type=None, lang=None, use_l10n=None):
+    user = get_current_user()
+
+    if user and user.is_authenticated and hasattr(user, "settings"):
+        user_settings = user.settings
+        if format_type == "THOUSAND_SEPARATOR":
+            number_format = getattr(user_settings, "number_format", None)
+            if number_format == "DC":
+                return "."
+            elif number_format == "CD":
+                return ","
+        elif format_type == "DECIMAL_SEPARATOR":
+            number_format = getattr(user_settings, "number_format", None)
+            if number_format == "DC":
+                return ","
+            elif number_format == "CD":
+                return "."
+        elif format_type == "SHORT_DATE_FORMAT":
+            date_format = getattr(user_settings, "date_format", None)
+            if date_format and date_format != "SHORT_DATE_FORMAT":
+                return date_format
+        elif format_type == "SHORT_DATETIME_FORMAT":
+            datetime_format = getattr(user_settings, "datetime_format", None)
+            if datetime_format and datetime_format != "SHORT_DATETIME_FORMAT":
+                return datetime_format
+
+    return original_get_format(format_type, lang, use_l10n)

app/apps/common/middleware/localization.py

@@ -1,14 +1,17 @@
 import zoneinfo
 
+from django.utils import formats
 from django.utils import timezone, translation
-from django.utils.translation import activate
+from django.utils.functional import lazy
 
+from apps.common.functions.format import get_format as custom_get_format
 from apps.users.models import UserSettings
 
 
 class LocalizationMiddleware:
     def __init__(self, get_response):
         self.get_response = get_response
+        self.patch_get_format()
 
     def __call__(self, request):
         tz = request.COOKIES.get("mytz")
@@ -33,9 +36,14 @@ class LocalizationMiddleware:
             timezone.activate(zoneinfo.ZoneInfo("UTC"))
 
         if user_language and user_language != "auto":
-            activate(user_language)
+            translation.activate(user_language)
         else:
             detected_language = translation.get_language_from_request(request)
-            activate(detected_language)
+            translation.activate(detected_language)
 
         return self.get_response(request)
+
+    @staticmethod
+    def patch_get_format():
+        formats.get_format = custom_get_format
+        formats.get_format_lazy = lazy(custom_get_format, str, list, tuple)

app/apps/common/middleware/thread_local.py

@@ -0,0 +1,73 @@
+"""
+    threadlocals middleware
+    ~~~~~~~~~~~~~~~~~~~~~~~
+
+    make the request object everywhere available (e.g. in model instance).
+
+    based on: http://code.djangoproject.com/wiki/CookBookThreadlocalsAndUser
+
+    Put this into your settings:
+    --------------------------------------------------------------------------
+        MIDDLEWARE_CLASSES = (
+            ...
+            'django_tools.middlewares.ThreadLocal.ThreadLocalMiddleware',
+            ...
+        )
+    --------------------------------------------------------------------------
+
+
+    Usage:
+    --------------------------------------------------------------------------
+    from django_tools.middlewares import ThreadLocal
+
+    # Get the current request object:
+    request = ThreadLocal.get_current_request()
+
+    # You can get the current user directly with:
+    user = ThreadLocal.get_current_user()
+    --------------------------------------------------------------------------
+
+    :copyleft: 2009-2017 by the django-tools team, see AUTHORS for more details.
+    :license: GNU GPL v3 or above, see LICENSE for more details.
+"""
+
+try:
+    from threading import local
+except ImportError:
+    from django.utils._threading_local import local
+
+try:
+    from django.utils.deprecation import MiddlewareMixin
+except ImportError:
+    MiddlewareMixin = object  # fallback for Django < 1.10
+
+
+_thread_locals = local()
+
+
+def get_current_request():
+    """returns the request object for this thread"""
+    return getattr(_thread_locals, "request", None)
+
+
+def get_current_user():
+    """returns the current user, if exist, otherwise returns None"""
+    request = get_current_request()
+    if request:
+        return getattr(request, "user", None)
+
+
+class ThreadLocalMiddleware(MiddlewareMixin):
+    """Simple middleware that adds the request object in thread local storage."""
+
+    def process_request(self, request):
+        _thread_locals.request = request
+
+    def process_response(self, request, response):
+        if hasattr(_thread_locals, "request"):
+            del _thread_locals.request
+        return response
+
+    def process_exception(self, request, exception):
+        if hasattr(_thread_locals, "request"):
+            del _thread_locals.request

app/apps/common/tasks.py

@@ -1,5 +1,8 @@
 import logging
 
+from asgiref.sync import sync_to_async
+from django.core import management
+
 from procrastinate import builtin_tasks
 from procrastinate.contrib.django import app
 
@@ -8,7 +11,7 @@ logger = logging.getLogger(__name__)
 
 
 @app.periodic(cron="0 4 * * *")
-@app.task(queueing_lock="remove_old_jobs", pass_context=True)
+@app.task(queueing_lock="remove_old_jobs", pass_context=True, name="remove_old_jobs")
 async def remove_old_jobs(context, timestamp):
     try:
         return await builtin_tasks.remove_old_jobs(
@@ -24,3 +27,16 @@ async def remove_old_jobs(context, timestamp):
             exc_info=True,
         )
         raise e
+
+
+@app.periodic(cron="0 6 1 * *")
+@app.task(queueing_lock="remove_expired_sessions", name="remove_expired_sessions")
+async def remove_expired_sessions(timestamp=None):
+    """Cleanup expired sessions by using Django management command."""
+    try:
+        await sync_to_async(management.call_command)("clearsessions", verbosity=0)
+    except Exception:
+        logger.error(
+            "Error while executing 'remove_expired_sessions' task",
+            exc_info=True,
+        )

app/apps/common/templatetags/formats.py

@@ -1,6 +1,6 @@
 from django import template
-from django.utils.formats import get_format
 
+from apps.common.functions.format import get_format
 
 register = template.Library()
 

app/apps/common/templatetags/json.py

@@ -0,0 +1,11 @@
+import json
+
+from django import template
+
+
+register = template.Library()
+
+
+@register.filter("json")
+def convert_to_json(value):
+    return json.dumps(value)

app/apps/common/templatetags/markdown.py

@@ -0,0 +1,52 @@
+from typing import Optional
+
+import mistune
+from django import template
+from django.utils.safestring import mark_safe
+from mistune import HTMLRenderer, Markdown, BlockParser, InlineParser, safe_entity
+from mistune.plugins.formatting import strikethrough as plugin_strikethrough
+from mistune.plugins.url import url as plugin_url
+
+
+register = template.Library()
+
+
+class CustomRenderer(HTMLRenderer):
+    def link(self, text: str, url: str, title: Optional[str] = None) -> str:
+        s = '<a rel="nofollow" target="_blank" href="' + self.safe_url(url) + '"'
+        if title:
+            s += ' title="' + safe_entity(title) + '"'
+        return s + ">" + text + "</a>"
+
+    def paragraph(self, text: str) -> str:
+        return text + "\n"
+
+    def softbreak(self) -> str:
+        return "\n"
+
+    def blank_line(self) -> str:
+        return "\n"
+
+
+block = BlockParser()
+block.rules = ["blank_line"]
+inline = InlineParser(hard_wrap=False)
+inline.rules = [
+    "emphasis",
+    "link",
+    "auto_link",
+    "auto_email",
+    "linebreak",
+    "softbreak",
+]
+markdown = Markdown(
+    renderer=CustomRenderer(escape=False),
+    block=block,
+    inline=inline,
+    plugins=[plugin_strikethrough, plugin_url],
+)
+
+
+@register.filter(name="limited_markdown")
+def limited_markdown(value):
+    return mark_safe(markdown(value))

app/apps/common/templatetags/settings.py

@@ -0,0 +1,9 @@
+from django import template
+from django.conf import settings
+
+register = template.Library()
+
+
+@register.simple_tag(name="settings")
+def settings_value(name):
+    return getattr(settings, name, "")

app/apps/common/urls.py

@@ -13,4 +13,9 @@ urlpatterns = [
         views.month_year_picker,
         name="month_year_picker",
     ),
+    path(
+        "cache/invalidate/",
+        views.invalidate_cache,
+        name="invalidate_cache",
+    ),
 ]

app/apps/common/utils/django.py

@@ -0,0 +1,161 @@
+def django_to_python_datetime(django_format):
+    mapping = {
+        # Day
+        "j": "%d",  # Day of the month without leading zeros
+        "d": "%d",  # Day of the month with leading zeros
+        "D": "%a",  # Day of the week, short version
+        "l": "%A",  # Day of the week, full version
+        # Month
+        "n": "%m",  # Month without leading zeros
+        "m": "%m",  # Month with leading zeros
+        "M": "%b",  # Month, short version
+        "F": "%B",  # Month, full version
+        # Year
+        "y": "%y",  # Year, 2 digits
+        "Y": "%Y",  # Year, 4 digits
+        # Time
+        "g": "%I",  # Hour (12-hour), without leading zeros
+        "G": "%H",  # Hour (24-hour), without leading zeros
+        "h": "%I",  # Hour (12-hour), with leading zeros
+        "H": "%H",  # Hour (24-hour), with leading zeros
+        "i": "%M",  # Minutes
+        "s": "%S",  # Seconds
+        "a": "%p",  # am/pm
+        "A": "%p",  # AM/PM
+        "P": "%I:%M %p",
+    }
+
+    python_format = django_format
+    for django_code, python_code in mapping.items():
+        python_format = python_format.replace(django_code, python_code)
+
+    return python_format
+
+
+def django_to_airdatepicker_datetime(django_format):
+    format_map = {
+        # Time
+        "h": "hh",  # Hour (12-hour)
+        "H": "H",  # Hour (24-hour)
+        "i": "m",  # Minutes
+        "A": "AA",  # AM/PM uppercase
+        "a": "aa",  # am/pm lowercase
+        "P": "h:mm AA",  # Localized time format (e.g., "2:30 PM")
+        # Date
+        "D": "E",  # Short weekday name
+        "l": "EEEE",  # Full weekday name
+        "j": "d",  # Day of month without leading zero
+        "d": "dd",  # Day of month with leading zero
+        "n": "M",  # Month without leading zero
+        "m": "MM",  # Month with leading zero
+        "M": "MMM",  # Short month name
+        "F": "MMMM",  # Full month name
+        "y": "yy",  # Year, 2 digits
+        "Y": "yyyy",  # Year, 4 digits
+    }
+
+    result = ""
+    i = 0
+    while i < len(django_format):
+        char = django_format[i]
+        if char == "\\":  # Handle escaped characters
+            if i + 1 < len(django_format):
+                result += django_format[i + 1]
+                i += 2
+            continue
+
+        if char in format_map:
+            result += format_map[char]
+        else:
+            result += char
+        i += 1
+
+    return result
+
+
+def django_to_airdatepicker_datetime_separated(django_format):
+    format_map = {
+        # Time formats
+        "h": "hh",  # Hour (12-hour)
+        "H": "HH",  # Hour (24-hour)
+        "i": "mm",  # Minutes
+        "A": "AA",  # AM/PM uppercase
+        "a": "aa",  # am/pm lowercase
+        "P": "h:mm aa",  # Localized time format
+        # Date formats
+        "D": "E",  # Short weekday name
+        "l": "EEEE",  # Full weekday name
+        "j": "d",  # Day of month without leading zero
+        "d": "dd",  # Day of month with leading zero
+        "n": "M",  # Month without leading zero
+        "m": "MM",  # Month with leading zero
+        "M": "MMM",  # Short month name
+        "F": "MMMM",  # Full month name
+        "y": "yy",  # Year, 2 digits
+        "Y": "yyyy",  # Year, 4 digits
+    }
+
+    # Define which characters belong to time format
+    time_chars = {"h", "H", "i", "A", "a", "P"}
+    date_chars = {"D", "l", "j", "d", "n", "m", "M", "F", "y", "Y"}
+
+    date_parts = []
+    time_parts = []
+    current_part = []
+    is_time = False
+
+    i = 0
+    while i < len(django_format):
+        char = django_format[i]
+
+        if char == "\\":  # Handle escaped characters
+            if i + 1 < len(django_format):
+                current_part.append(django_format[i + 1])
+                i += 2
+            continue
+
+        if char in format_map:
+            if char in time_chars:
+                # If we were building a date part, save it and start a time part
+                if current_part and not is_time:
+                    date_parts.append("".join(current_part))
+                    current_part = []
+                is_time = True
+                current_part.append(format_map[char])
+            elif char in date_chars:
+                # If we were building a time part, save it and start a date part
+                if current_part and is_time:
+                    time_parts.append("".join(current_part))
+                    current_part = []
+                is_time = False
+                current_part.append(format_map[char])
+        else:
+            # Handle separators
+            if char in "/:.-":
+                current_part.append(char)
+            elif char == " ":
+                if current_part:
+                    if is_time:
+                        time_parts.append("".join(current_part))
+                    else:
+                        date_parts.append("".join(current_part))
+                    current_part = []
+                current_part.append(char)
+
+        i += 1
+
+    # Don't forget the last part
+    if current_part:
+        if is_time:
+            time_parts.append("".join(current_part))
+        else:
+            date_parts.append("".join(current_part))
+
+    date_format = "".join(date_parts)
+    time_format = "".join(time_parts)
+
+    # Clean up multiple spaces while preserving necessary ones
+    date_format = " ".join(filter(None, date_format.split()))
+    time_format = " ".join(filter(None, time_format.split()))
+
+    return date_format, time_format

app/apps/common/views.py

@@ -1,17 +1,32 @@
 from dateutil.relativedelta import relativedelta
+
+from django.contrib import messages
+from django.contrib.auth.decorators import login_required
 from django.db.models import Count
 from django.db.models.functions import ExtractYear, ExtractMonth
+from django.http import HttpResponse
 from django.shortcuts import render
 from django.urls import reverse
 from django.utils import timezone
+from django.views.decorators.http import require_http_methods
+from django.utils.translation import gettext_lazy as _
 
+from cachalot.api import invalidate
+
+from apps.common.decorators.htmx import only_htmx
 from apps.transactions.models import Transaction
 
 
+@only_htmx
+@login_required
+@require_http_methods(["GET"])
 def toasts(request):
     return render(request, "common/fragments/toasts.html")
 
 
+@only_htmx
+@login_required
+@require_http_methods(["GET"])
 def month_year_picker(request):
     field = request.GET.get("field", "reference_date")
     for_ = request.GET.get("for", None)
@@ -84,3 +99,19 @@ def month_year_picker(request):
             "current_year": current_year,
         },
     )
+
+
+@only_htmx
+@login_required
+@require_http_methods(["GET"])
+def invalidate_cache(request):
+    invalidate()
+
+    messages.success(request, _("Cache cleared successfully"))
+
+    return HttpResponse(
+        status=204,
+        headers={
+            "HX-Trigger": "updated",
+        },
+    )

app/apps/common/widgets/datepicker.py

@@ -0,0 +1,302 @@
+import datetime
+
+from django.forms import widgets
+from django.utils import formats, translation, dates
+from django.utils.translation import gettext_lazy as _
+
+from apps.common.utils.django import (
+    django_to_python_datetime,
+    django_to_airdatepicker_datetime,
+    django_to_airdatepicker_datetime_separated,
+)
+from apps.common.functions.format import get_format
+
+
+class AirDatePickerInput(widgets.DateInput):
+    def __init__(
+        self,
+        attrs=None,
+        format=None,
+        clear_button=True,
+        auto_close=True,
+        read_only=True,
+        toggle_selected=None,
+        *args,
+        **kwargs,
+    ):
+        attrs = attrs or {}
+        super().__init__(attrs=attrs, format=format, *args, **kwargs)
+        self.clear_button = clear_button
+        self.auto_close = auto_close
+        self.read_only = read_only
+        self.toggle_selected = (
+            toggle_selected if isinstance(toggle_selected, bool) else self.clear_button
+        )
+
+    @staticmethod
+    def _get_current_language():
+        """Get current language code in format compatible with AirDatepicker"""
+        lang_code = translation.get_language()
+        # AirDatepicker uses simple language codes
+        return lang_code.split("-")[0]
+
+    def _get_format(self):
+        """Get the format string based on user settings or default"""
+        if self.format:
+            return self.format
+
+        return get_format("SHORT_DATE_FORMAT", use_l10n=True)
+
+    def build_attrs(self, base_attrs, extra_attrs=None):
+        attrs = super().build_attrs(base_attrs, extra_attrs)
+
+        attrs["data-now-button-txt"] = _("Today")
+        attrs["data-auto-close"] = str(self.auto_close).lower()
+        attrs["data-clear-button"] = str(self.clear_button).lower()
+        attrs["data-toggle-selected"] = str(self.toggle_selected).lower()
+        attrs["data-language"] = self._get_current_language()
+        attrs["data-date-format"] = django_to_airdatepicker_datetime(self._get_format())
+
+        if self.read_only:
+            attrs["readonly"] = True
+
+        return attrs
+
+    def format_value(self, value):
+        """Format the value for display in the widget."""
+        if value:
+            self.attrs["data-value"] = value
+
+        if value is None:
+            return ""
+        if isinstance(value, (datetime.date, datetime.datetime)):
+            return formats.date_format(value, format=self._get_format(), use_l10n=True)
+
+        return str(value)
+
+    def value_from_datadict(self, data, files, name):
+        """Parse the datetime string from the form data."""
+        value = super().value_from_datadict(data, files, name)
+        if value:
+            try:
+                # This does it's best to convert Django's PHP-Style date format to a datetime format and reformat the
+                # value to be read by Django. Probably could be improved
+                return datetime.datetime.strptime(
+                    value.strip(),
+                    django_to_python_datetime(self._get_format())
+                    or django_to_python_datetime(get_format("SHORT_DATETIME_FORMAT")),
+                ).strftime("%Y-%m-%d")
+            except (ValueError, TypeError) as e:
+                return value
+        return None
+
+
+class AirDateTimePickerInput(widgets.DateTimeInput):
+    def __init__(
+        self,
+        attrs=None,
+        format=None,
+        timepicker=True,
+        clear_button=True,
+        auto_close=True,
+        read_only=True,
+        toggle_selected=None,
+        *args,
+        **kwargs,
+    ):
+        attrs = attrs or {}
+        super().__init__(attrs=attrs, format=format, *args, **kwargs)
+        self.timepicker = timepicker
+        self.clear_button = clear_button
+        self.auto_close = auto_close
+        self.read_only = read_only
+        self.toggle_selected = (
+            toggle_selected if isinstance(toggle_selected, bool) else self.clear_button
+        )
+
+    @staticmethod
+    def _get_current_language():
+        """Get current language code in format compatible with AirDatepicker"""
+        lang_code = translation.get_language()
+        # AirDatepicker uses simple language codes
+        return lang_code.split("-")[0]
+
+    def _get_format(self):
+        """Get the format string based on user settings or default"""
+        if self.format:
+            return self.format
+
+        return get_format("SHORT_DATETIME_FORMAT", use_l10n=True)
+
+    def build_attrs(self, base_attrs, extra_attrs=None):
+        attrs = super().build_attrs(base_attrs, extra_attrs)
+
+        date_format, time_format = django_to_airdatepicker_datetime_separated(
+            self._get_format()
+        )
+
+        # Add data attributes for AirDatepicker configuration
+        attrs["data-now-button-txt"] = _("Now")
+        attrs["data-timepicker"] = str(self.timepicker).lower()
+        attrs["data-auto-close"] = str(self.auto_close).lower()
+        attrs["data-toggle-selected"] = str(self.toggle_selected).lower()
+        attrs["data-clear-button"] = str(self.clear_button).lower()
+        attrs["data-language"] = self._get_current_language()
+        attrs["data-date-format"] = date_format
+        attrs["data-time-format"] = time_format
+
+        if self.read_only:
+            attrs["readonly"] = True
+
+        return attrs
+
+    def format_value(self, value):
+        """Format the value for display in the widget."""
+        if value and isinstance(value, (datetime.date, datetime.datetime)):
+            self.attrs["data-value"] = datetime.datetime.strftime(
+                value, "%Y-%m-%dT%H:%M:00"
+            )
+        elif value and isinstance(value, str):
+            value = datetime.datetime.strptime(value, "%Y-%m-%d %H:%M:00")
+            self.attrs["data-value"] = datetime.datetime.strftime(
+                value, "%Y-%m-%dT%H:%M:00"
+            )
+
+        if value is None:
+            return ""
+        if isinstance(value, (datetime.date, datetime.datetime)):
+            return formats.date_format(value, format=self._get_format(), use_l10n=True)
+
+        return str(value)
+
+    def value_from_datadict(self, data, files, name):
+        """Parse the datetime string from the form data."""
+        value = super().value_from_datadict(data, files, name)
+        if value:
+            try:
+                # This does it's best to convert Django's PHP-Style date format to a datetime format and reformat the
+                # value to be read by Django. Probably could be improved
+                return datetime.datetime.strptime(
+                    value.strip(),
+                    django_to_python_datetime(self._get_format())
+                    or django_to_python_datetime(get_format("SHORT_DATETIME_FORMAT")),
+                ).strftime("%Y-%m-%d %H:%M:%S")
+            except (ValueError, TypeError) as e:
+                return value
+        return None
+
+
+class AirMonthYearPickerInput(AirDatePickerInput):
+    def __init__(self, attrs=None, format=None, *args, **kwargs):
+        super().__init__(attrs=attrs, format=format, *args, **kwargs)
+        # Store the display format for AirDatepicker
+        self.display_format = "MMMM yyyy"
+        # Store the Python format for internal use
+        self.python_format = "%B %Y"
+
+    @staticmethod
+    def _get_month_names():
+        """Get month names using Django's date translation"""
+        return {dates.MONTHS[i]: i for i in range(1, 13)}
+
+    def build_attrs(self, base_attrs, extra_attrs=None):
+        attrs = super().build_attrs(base_attrs, extra_attrs)
+
+        # Add data attributes for AirDatepicker configuration
+        attrs["data-now-button-txt"] = _("Today")
+        attrs["data-date-format"] = "MMMM yyyy"
+
+        return attrs
+
+    def format_value(self, value):
+        """Format the value for display in the widget."""
+        if value:
+            self.attrs["data-value"] = (
+                value  # We use this to dynamically select the initial date on AirDatePicker
+            )
+
+        if value is None:
+            return ""
+        if isinstance(value, str):
+            try:
+                value = datetime.datetime.strptime(value, "%Y-%m-%d").date()
+            except ValueError:
+                return value
+        if isinstance(value, (datetime.datetime, datetime.date)):
+            # Use Django's date translation
+            month_name = dates.MONTHS[value.month]
+            return f"{month_name} {value.year}"
+        return value
+
+    def value_from_datadict(self, data, files, name):
+        """Convert the value from the widget format back to a format Django can handle."""
+        value = super().value_from_datadict(data, files, name)
+        if value:
+            try:
+                # Split the value into month name and year
+                month_str, year_str = value.rsplit(" ", 1)
+                year = int(year_str)
+
+                # Get month number from translated month name
+                month_names = self._get_month_names()
+                month = month_names.get(month_str)
+
+                if month and year:
+                    # Return the first day of the month in Django's expected format
+                    return datetime.date(year, month, 1).strftime("%Y-%m-%d")
+            except (ValueError, KeyError):
+                return None
+        return None
+
+
+class AirYearPickerInput(AirDatePickerInput):
+    def __init__(self, attrs=None, format=None, *args, **kwargs):
+        super().__init__(attrs=attrs, format=format, *args, **kwargs)
+        # Store the display format for AirDatepicker
+        self.display_format = "yyyy"
+        # Store the Python format for internal use
+        self.python_format = "%Y"
+
+    def build_attrs(self, base_attrs, extra_attrs=None):
+        attrs = super().build_attrs(base_attrs, extra_attrs)
+
+        # Add data attributes for AirDatepicker configuration
+        attrs["data-now-button-txt"] = _("Today")
+        attrs["data-date-format"] = "yyyy"
+
+        return attrs
+
+    def format_value(self, value):
+        """Format the value for display in the widget."""
+        if value:
+            self.attrs["data-value"] = (
+                value  # We use this to dynamically select the initial date on AirDatePicker
+            )
+
+        if value is None:
+            return ""
+        if isinstance(value, str):
+            try:
+                value = datetime.datetime.strptime(value, "%Y-%m-%d").date()
+            except ValueError:
+                return value
+        if isinstance(value, (datetime.datetime, datetime.date)):
+            # Use Django's date translation
+            return f"{value.year}"
+        return value
+
+    def value_from_datadict(self, data, files, name):
+        """Convert the value from the widget format back to a format Django can handle."""
+        value = super().value_from_datadict(data, files, name)
+        if value:
+            try:
+                # Split the value into month name and year
+                year_str = value
+                year = int(year_str)
+
+                if year:
+                    # Return the first day of the month in Django's expected format
+                    return datetime.date(year, 1, 1).strftime("%Y-%m-%d")
+            except (ValueError, KeyError):
+                return None
+        return None

app/apps/common/widgets/decimal.py

@@ -1,7 +1,9 @@
 from decimal import Decimal, InvalidOperation
 
 from django import forms
-from django.utils.formats import get_format, number_format
+from django.utils.formats import number_format
+
+from apps.common.functions.format import get_format
 
 
 def convert_to_decimal(value: str):

app/apps/common/widgets/tom_select.py

@@ -1,4 +1,5 @@
 from django.forms import widgets, SelectMultiple
+from django.urls import reverse
 from django.utils.translation import gettext_lazy as _
 
 
@@ -129,3 +130,28 @@ class TomSelect(widgets.Select):
 
 class TomSelectMultiple(SelectMultiple, TomSelect):
     pass
+
+
+class TransactionSelect(TomSelect):
+    def __init__(self, income: bool = True, expense: bool = True, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self.load_income = income
+        self.load_expense = expense
+        self.create = False
+
+    def build_attrs(self, base_attrs, extra_attrs=None):
+        attrs = super().build_attrs(base_attrs, extra_attrs)
+
+        if self.load_income and self.load_expense:
+            attrs["data-load"] = reverse("transactions_search")
+        elif self.load_income and not self.load_expense:
+            attrs["data-load"] = reverse(
+                "transactions_search", kwargs={"filter_type": "income"}
+            )
+        elif self.load_expense and not self.load_income:
+            attrs["data-load"] = reverse(
+                "transactions_search", kwargs={"filter_type": "expenses"}
+            )
+
+        return attrs

app/apps/currencies/admin.py

@@ -1,6 +1,6 @@
 from django.contrib import admin
 
-from apps.currencies.models import Currency, ExchangeRate
+from apps.currencies.models import Currency, ExchangeRate, ExchangeRateService
 
 
 @admin.register(Currency)
@@ -11,4 +11,19 @@ class CurrencyAdmin(admin.ModelAdmin):
         return super().formfield_for_dbfield(db_field, request, **kwargs)
 
 
+@admin.register(ExchangeRateService)
+class ExchangeRateServiceAdmin(admin.ModelAdmin):
+    list_display = [
+        "name",
+        "service_type",
+        "is_active",
+        "interval_type",
+        "fetch_interval",
+        "last_fetch",
+    ]
+    list_filter = ["is_active", "service_type"]
+    search_fields = ["name"]
+    filter_horizontal = ["target_currencies"]
+
+
 admin.site.register(ExchangeRate)

app/apps/currencies/exchange_rates/base.py

@@ -0,0 +1,30 @@
+from abc import ABC, abstractmethod
+from decimal import Decimal
+from typing import List, Tuple, Optional
+from django.db.models import QuerySet
+
+from apps.currencies.models import Currency
+
+
+class ExchangeRateProvider(ABC):
+    rates_inverted = False
+
+    def __init__(self, api_key: Optional[str] = None):
+        self.api_key = api_key
+
+    @abstractmethod
+    def get_rates(
+        self, target_currencies: QuerySet, exchange_currencies: set
+    ) -> List[Tuple[Currency, Currency, Decimal]]:
+        """Fetch exchange rates for multiple currency pairs"""
+        raise NotImplementedError("Subclasses must implement get_rates method")
+
+    @classmethod
+    def requires_api_key(cls) -> bool:
+        """Return True if the service requires an API key"""
+        return True
+
+    @staticmethod
+    def invert_rate(rate: Decimal) -> Decimal:
+        """Invert the given rate."""
+        return Decimal("1") / rate

app/apps/currencies/exchange_rates/fetcher.py

@@ -0,0 +1,223 @@
+import logging
+from datetime import timedelta
+
+from django.db.models import QuerySet
+from django.utils import timezone
+
+from apps.currencies.exchange_rates.providers import (
+    SynthFinanceProvider,
+    CoinGeckoFreeProvider,
+    CoinGeckoProProvider,
+)
+from apps.currencies.models import ExchangeRateService, ExchangeRate, Currency
+
+logger = logging.getLogger(__name__)
+
+
+# Map service types to provider classes
+PROVIDER_MAPPING = {
+    "synth_finance": SynthFinanceProvider,
+    "coingecko_free": CoinGeckoFreeProvider,
+    "coingecko_pro": CoinGeckoProProvider,
+}
+
+
+class ExchangeRateFetcher:
+    def _should_fetch_at_hour(service: ExchangeRateService, current_hour: int) -> bool:
+        """Check if service should fetch rates at given hour based on interval type."""
+        try:
+            if service.interval_type == ExchangeRateService.IntervalType.NOT_ON:
+                blocked_hours = ExchangeRateService._parse_hour_ranges(
+                    service.fetch_interval
+                )
+                should_fetch = current_hour not in blocked_hours
+                logger.info(
+                    f"NOT_ON check for {service.name}: "
+                    f"current_hour={current_hour}, "
+                    f"blocked_hours={blocked_hours}, "
+                    f"should_fetch={should_fetch}"
+                )
+                return should_fetch
+
+            if service.interval_type == ExchangeRateService.IntervalType.ON:
+                allowed_hours = ExchangeRateService._parse_hour_ranges(
+                    service.fetch_interval
+                )
+
+                should_fetch = current_hour in allowed_hours
+
+                logger.info(
+                    f"ON check for {service.name}: "
+                    f"current_hour={current_hour}, "
+                    f"allowed_hours={allowed_hours}, "
+                    f"should_fetch={should_fetch}"
+                )
+
+                return should_fetch
+
+            if service.interval_type == ExchangeRateService.IntervalType.EVERY:
+                try:
+                    interval_hours = int(service.fetch_interval)
+
+                    if service.last_fetch is None:
+                        return True
+
+                    # Round down to nearest hour
+                    now = timezone.now().replace(minute=0, second=0, microsecond=0)
+                    last_fetch = service.last_fetch.replace(
+                        minute=0, second=0, microsecond=0
+                    )
+
+                    hours_since_last = (now - last_fetch).total_seconds() / 3600
+                    should_fetch = hours_since_last >= interval_hours
+
+                    logger.info(
+                        f"EVERY check for {service.name}: "
+                        f"hours_since_last={hours_since_last:.1f}, "
+                        f"interval={interval_hours}, "
+                        f"should_fetch={should_fetch}"
+                    )
+                    return should_fetch
+                except ValueError:
+                    logger.error(
+                        f"Invalid EVERY interval format for {service.name}: "
+                        f"expected single number, got '{service.fetch_interval}'"
+                    )
+                    return False
+
+            return False
+
+        except ValueError as e:
+            logger.error(f"Error parsing fetch_interval for {service.name}: {e}")
+            return False
+
+    @staticmethod
+    def fetch_due_rates(force: bool = False) -> None:
+        """
+        Fetch rates for all services that are due for update.
+        Args:
+            force (bool): If True, fetches all active services regardless of their schedule.
+        """
+        services = ExchangeRateService.objects.filter(is_active=True)
+        current_time = timezone.now().astimezone()
+        current_hour = current_time.hour
+
+        for service in services:
+            try:
+                if force:
+                    logger.info(f"Force fetching rates for {service.name}")
+                    ExchangeRateFetcher._fetch_service_rates(service)
+                    continue
+
+                # Check if service should fetch based on interval type
+                if ExchangeRateFetcher._should_fetch_at_hour(service, current_hour):
+                    logger.info(
+                        f"Fetching rates for {service.name}. "
+                        f"Last fetch: {service.last_fetch}, "
+                        f"Interval type: {service.interval_type}, "
+                        f"Current hour: {current_hour}"
+                    )
+                    ExchangeRateFetcher._fetch_service_rates(service)
+                else:
+                    logger.debug(
+                        f"Skipping {service.name}. "
+                        f"Current hour: {current_hour}, "
+                        f"Interval type: {service.interval_type}, "
+                        f"Fetch interval: {service.fetch_interval}"
+                    )
+
+            except Exception as e:
+                logger.error(f"Error checking fetch schedule for {service.name}: {e}")
+
+    @staticmethod
+    def _get_unique_currency_pairs(
+        service: ExchangeRateService,
+    ) -> tuple[QuerySet, set]:
+        """
+        Get unique currency pairs from both target_currencies and target_accounts
+        Returns a tuple of (target_currencies QuerySet, exchange_currencies set)
+        """
+        # Get currencies from target_currencies
+        target_currencies = set(service.target_currencies.all())
+
+        # Add currencies from target_accounts
+        for account in service.target_accounts.all():
+            if account.currency and account.exchange_currency:
+                target_currencies.add(account.currency)
+
+        # Convert back to QuerySet for compatibility with existing code
+        target_currencies_qs = Currency.objects.filter(
+            id__in=[curr.id for curr in target_currencies]
+        )
+
+        # Get unique exchange currencies
+        exchange_currencies = set()
+
+        # From target_currencies
+        for currency in target_currencies:
+            if currency.exchange_currency:
+                exchange_currencies.add(currency.exchange_currency)
+
+        # From target_accounts
+        for account in service.target_accounts.all():
+            if account.exchange_currency:
+                exchange_currencies.add(account.exchange_currency)
+
+        return target_currencies_qs, exchange_currencies
+
+    @staticmethod
+    def _fetch_service_rates(service: ExchangeRateService) -> None:
+        """Fetch rates for a specific service"""
+        try:
+            provider = service.get_provider()
+
+            # Check if API key is required but missing
+            if provider.requires_api_key() and not service.api_key:
+                logger.error(f"API key required but not provided for {service.name}")
+                return
+
+            # Get unique currency pairs from both sources
+            target_currencies, exchange_currencies = (
+                ExchangeRateFetcher._get_unique_currency_pairs(service)
+            )
+
+            # Skip if no currencies to process
+            if not target_currencies or not exchange_currencies:
+                logger.info(f"No currency pairs to process for service {service.name}")
+                return
+
+            rates = provider.get_rates(target_currencies, exchange_currencies)
+
+            # Track processed currency pairs to avoid duplicates
+            processed_pairs = set()
+
+            for from_currency, to_currency, rate in rates:
+                # Create a unique identifier for this currency pair
+                pair_key = (from_currency.id, to_currency.id)
+                if pair_key in processed_pairs:
+                    continue
+
+                if provider.rates_inverted:
+                    # If rates are inverted, we need to swap currencies
+                    ExchangeRate.objects.create(
+                        from_currency=to_currency,
+                        to_currency=from_currency,
+                        rate=rate,
+                        date=timezone.now(),
+                    )
+                    processed_pairs.add((to_currency.id, from_currency.id))
+                else:
+                    # If rates are not inverted, we can use them as is
+                    ExchangeRate.objects.create(
+                        from_currency=from_currency,
+                        to_currency=to_currency,
+                        rate=rate,
+                        date=timezone.now(),
+                    )
+                    processed_pairs.add((from_currency.id, to_currency.id))
+
+            service.last_fetch = timezone.now()
+            service.save()
+
+        except Exception as e:
+            logger.error(f"Error fetching rates for {service.name}: {e}")

app/apps/currencies/exchange_rates/providers.py

@@ -0,0 +1,152 @@
+import logging
+import time
+
+import requests
+from decimal import Decimal
+from typing import Tuple, List
+
+from django.db.models import QuerySet
+
+from apps.currencies.models import Currency
+from apps.currencies.exchange_rates.base import ExchangeRateProvider
+
+logger = logging.getLogger(__name__)
+
+
+class SynthFinanceProvider(ExchangeRateProvider):
+    """Implementation for Synth Finance API (synthfinance.com)"""
+
+    BASE_URL = "https://api.synthfinance.com/rates/live"
+    rates_inverted = False  # SynthFinance returns non-inverted rates
+
+    def __init__(self, api_key: str = None):
+        super().__init__(api_key)
+        self.session = requests.Session()
+        self.session.headers.update({"Authorization": f"Bearer {self.api_key}"})
+
+    def get_rates(
+        self, target_currencies: QuerySet, exchange_currencies: set
+    ) -> List[Tuple[Currency, Currency, Decimal]]:
+        results = []
+        currency_groups = {}
+        for currency in target_currencies:
+            if currency.exchange_currency in exchange_currencies:
+                group = currency_groups.setdefault(currency.exchange_currency.code, [])
+                group.append(currency)
+
+        for base_currency, currencies in currency_groups.items():
+            try:
+                to_currencies = ",".join(
+                    currency.code
+                    for currency in currencies
+                    if currency.code != base_currency
+                )
+                response = self.session.get(
+                    f"{self.BASE_URL}",
+                    params={"from": base_currency, "to": to_currencies},
+                )
+                response.raise_for_status()
+                data = response.json()
+                rates = data["data"]["rates"]
+
+                for currency in currencies:
+                    if currency.code == base_currency:
+                        rate = Decimal("1")
+                    else:
+                        rate = Decimal(str(rates[currency.code]))
+                    # Return the rate as is, without inversion
+                    results.append((currency.exchange_currency, currency, rate))
+
+                credits_used = data["meta"]["credits_used"]
+                credits_remaining = data["meta"]["credits_remaining"]
+                logger.info(
+                    f"Synth Finance API call: {credits_used} credits used, {credits_remaining} remaining"
+                )
+            except requests.RequestException as e:
+                logger.error(
+                    f"Error fetching rates from Synth Finance API for base {base_currency}: {e}"
+                )
+            except KeyError as e:
+                logger.error(
+                    f"Unexpected response structure from Synth Finance API for base {base_currency}: {e}"
+                )
+            except Exception as e:
+                logger.error(
+                    f"Unexpected error processing Synth Finance data for base {base_currency}: {e}"
+                )
+        return results
+
+
+class CoinGeckoFreeProvider(ExchangeRateProvider):
+    """Implementation for CoinGecko Free API"""
+
+    BASE_URL = "https://api.coingecko.com/api/v3"
+    rates_inverted = True
+
+    def __init__(self, api_key: str):
+        super().__init__(api_key)
+        self.session = requests.Session()
+        self.session.headers.update({"x-cg-demo-api-key": api_key})
+
+    @classmethod
+    def requires_api_key(cls) -> bool:
+        return True
+
+    def get_rates(
+        self, target_currencies: QuerySet, exchange_currencies: set
+    ) -> List[Tuple[Currency, Currency, Decimal]]:
+        results = []
+        all_currencies = set(currency.code.lower() for currency in target_currencies)
+        all_currencies.update(currency.code.lower() for currency in exchange_currencies)
+
+        try:
+            response = self.session.get(
+                f"{self.BASE_URL}/simple/price",
+                params={
+                    "ids": ",".join(all_currencies),
+                    "vs_currencies": ",".join(all_currencies),
+                },
+            )
+            response.raise_for_status()
+            rates_data = response.json()
+
+            for target_currency in target_currencies:
+                if target_currency.exchange_currency in exchange_currencies:
+                    try:
+                        rate = Decimal(
+                            str(
+                                rates_data[target_currency.code.lower()][
+                                    target_currency.exchange_currency.code.lower()
+                                ]
+                            )
+                        )
+                        # The rate is already inverted, so we don't need to invert it again
+                        results.append(
+                            (target_currency.exchange_currency, target_currency, rate)
+                        )
+                    except KeyError:
+                        logger.error(
+                            f"Rate not found for {target_currency.code} or {target_currency.exchange_currency.code}"
+                        )
+                    except Exception as e:
+                        logger.error(
+                            f"Error calculating rate for {target_currency.code}: {e}"
+                        )
+
+            time.sleep(1)  # CoinGecko allows 10-30 calls/minute for free tier
+        except requests.RequestException as e:
+            logger.error(f"Error fetching rates from CoinGecko API: {e}")
+
+        return results
+
+
+class CoinGeckoProProvider(CoinGeckoFreeProvider):
+    """Implementation for CoinGecko Pro API"""
+
+    BASE_URL = "https://pro-api.coingecko.com/api/v3/simple/price"
+    rates_inverted = True
+
+    def __init__(self, api_key: str):
+        super().__init__(api_key)
+        self.session = requests.Session()
+        self.session.headers.update({"x-cg-pro-api-key": api_key})

app/apps/currencies/forms.py

@@ -1,14 +1,16 @@
+from crispy_bootstrap5.bootstrap5 import Switch
 from crispy_forms.bootstrap import FormActions
 from crispy_forms.helper import FormHelper
-from crispy_forms.layout import Layout
+from crispy_forms.layout import Layout, Row, Column
 from django import forms
 from django.forms import CharField
 from django.utils.translation import gettext_lazy as _
 
 from apps.common.widgets.crispy.submit import NoClassSubmit
+from apps.common.widgets.datepicker import AirDateTimePickerInput
 from apps.common.widgets.decimal import ArbitraryDecimalDisplayNumberInput
-from apps.currencies.models import Currency, ExchangeRate
 from apps.common.widgets.tom_select import TomSelect
+from apps.currencies.models import Currency, ExchangeRate, ExchangeRateService
 
 
 class CurrencyForm(forms.ModelForm):
@@ -64,9 +66,7 @@ class CurrencyForm(forms.ModelForm):
 
 class ExchangeRateForm(forms.ModelForm):
     date = forms.DateTimeField(
-        widget=forms.DateTimeInput(
-            attrs={"type": "datetime-local"}, format="%Y-%m-%dT%H:%M"
-        )
+        label=_("Date"),
     )
 
     class Meta:
@@ -82,6 +82,58 @@ class ExchangeRateForm(forms.ModelForm):
         self.helper.layout = Layout("date", "from_currency", "to_currency", "rate")
 
         self.fields["rate"].widget = ArbitraryDecimalDisplayNumberInput()
+        self.fields["date"].widget = AirDateTimePickerInput(clear_button=False)
+
+        if self.instance and self.instance.pk:
+            self.helper.layout.append(
+                FormActions(
+                    NoClassSubmit(
+                        "submit", _("Update"), css_class="btn btn-outline-primary w-100"
+                    ),
+                ),
+            )
+        else:
+            self.helper.layout.append(
+                FormActions(
+                    NoClassSubmit(
+                        "submit", _("Add"), css_class="btn btn-outline-primary w-100"
+                    ),
+                ),
+            )
+
+
+class ExchangeRateServiceForm(forms.ModelForm):
+    class Meta:
+        model = ExchangeRateService
+        fields = [
+            "name",
+            "service_type",
+            "is_active",
+            "api_key",
+            "interval_type",
+            "fetch_interval",
+            "target_currencies",
+            "target_accounts",
+        ]
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self.helper = FormHelper()
+        self.helper.form_tag = False
+        self.helper.form_method = "post"
+        self.helper.layout = Layout(
+            "name",
+            "service_type",
+            Switch("is_active"),
+            "api_key",
+            Row(
+                Column("interval_type", css_class="form-group col-md-6"),
+                Column("fetch_interval", css_class="form-group col-md-6"),
+            ),
+            "target_currencies",
+            "target_accounts",
+        )
 
         if self.instance and self.instance.pk:
             self.helper.layout.append(

app/apps/currencies/migrations/0007_exchangerateservice.py

@@ -0,0 +1,32 @@
+# Generated by Django 5.1.5 on 2025-02-02 20:35
+
+import django.core.validators
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('currencies', '0006_currency_exchange_currency'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ExchangeRateService',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=255, unique=True, verbose_name='Service Name')),
+                ('service_type', models.CharField(choices=[('synth_finance', 'Synth Finance'), ('coingecko', 'CoinGecko')], max_length=255, verbose_name='Service Type')),
+                ('is_active', models.BooleanField(default=True, verbose_name='Active')),
+                ('api_key', models.CharField(blank=True, help_text='API key for the service (if required)', max_length=255, null=True, verbose_name='API Key')),
+                ('fetch_interval_hours', models.PositiveIntegerField(default=24, validators=[django.core.validators.MinValueValidator(1)], verbose_name='Fetch Interval (hours)')),
+                ('last_fetch', models.DateTimeField(blank=True, null=True, verbose_name='Last Successful Fetch')),
+                ('target_currencies', models.ManyToManyField(help_text='Select currencies to fetch exchange rates for. Rates will be fetched for each currency against their exchange_currency.', related_name='exchange_services', to='currencies.currency', verbose_name='Target Currencies')),
+            ],
+            options={
+                'verbose_name': 'Exchange Rate Service',
+                'verbose_name_plural': 'Exchange Rate Services',
+                'ordering': ['name'],
+            },
+        ),
+    ]

app/apps/currencies/migrations/0008_exchangerateservice_target_accounts_and_more.py

@@ -0,0 +1,24 @@
+# Generated by Django 5.1.5 on 2025-02-03 01:52
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('accounts', '0008_alter_account_name'),
+        ('currencies', '0007_exchangerateservice'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='exchangerateservice',
+            name='target_accounts',
+            field=models.ManyToManyField(help_text="Select accounts to fetch exchange rates for. Rates will be fetched for each account's currency against their set exchange currency.", related_name='exchange_services', to='accounts.account', verbose_name='Target Accounts'),
+        ),
+        migrations.AlterField(
+            model_name='exchangerateservice',
+            name='target_currencies',
+            field=models.ManyToManyField(help_text='Select currencies to fetch exchange rates for. Rates will be fetched for each currency against their set exchange currency.', related_name='exchange_services', to='currencies.currency', verbose_name='Target Currencies'),
+        ),
+    ]

app/apps/currencies/migrations/0009_alter_exchangerateservice_target_accounts_and_more.py

@@ -0,0 +1,24 @@
+# Generated by Django 5.1.5 on 2025-02-03 01:55
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('accounts', '0008_alter_account_name'),
+        ('currencies', '0008_exchangerateservice_target_accounts_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='exchangerateservice',
+            name='target_accounts',
+            field=models.ManyToManyField(blank=True, help_text="Select accounts to fetch exchange rates for. Rates will be fetched for each account's currency against their set exchange currency.", related_name='exchange_services', to='accounts.account', verbose_name='Target Accounts'),
+        ),
+        migrations.AlterField(
+            model_name='exchangerateservice',
+            name='target_currencies',
+            field=models.ManyToManyField(blank=True, help_text='Select currencies to fetch exchange rates for. Rates will be fetched for each currency against their set exchange currency.', related_name='exchange_services', to='currencies.currency', verbose_name='Target Currencies'),
+        ),
+    ]

app/apps/currencies/migrations/0010_alter_currency_code.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.1.5 on 2025-02-03 03:17
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('currencies', '0009_alter_exchangerateservice_target_accounts_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='currency',
+            name='code',
+            field=models.CharField(max_length=255, verbose_name='Currency Code'),
+        ),
+    ]

app/apps/currencies/migrations/0011_remove_exchangerateservice_fetch_interval_hours_and_more.py

@@ -0,0 +1,32 @@
+# Generated by Django 5.1.5 on 2025-02-07 02:02
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('currencies', '0010_alter_currency_code'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='exchangerateservice',
+            name='fetch_interval_hours',
+        ),
+        migrations.AddField(
+            model_name='exchangerateservice',
+            name='fetch_interval',
+            field=models.CharField(default='24', max_length=1000, verbose_name='Interval'),
+        ),
+        migrations.AddField(
+            model_name='exchangerateservice',
+            name='interval_type',
+            field=models.CharField(choices=[('on', 'On'), ('every', 'Every X hours'), ('not_on', 'Not on')], default='every', max_length=255, verbose_name='Interval Type'),
+        ),
+        migrations.AlterField(
+            model_name='exchangerateservice',
+            name='service_type',
+            field=models.CharField(choices=[('synth_finance', 'Synth Finance'), ('coingecko_free', 'CoinGecko (Demo/Free)'), ('coingecko_pro', 'CoinGecko (Pro)')], max_length=255, verbose_name='Service Type'),
+        ),
+    ]

app/apps/currencies/models.py

@@ -1,11 +1,18 @@
+import logging
+from typing import Set
+
 from django.core.exceptions import ValidationError
 from django.core.validators import MaxValueValidator, MinValueValidator
 from django.db import models
 from django.utils.translation import gettext_lazy as _
 
+logger = logging.getLogger(__name__)
+
 
 class Currency(models.Model):
-    code = models.CharField(max_length=10, unique=True, verbose_name=_("Currency Code"))
+    code = models.CharField(
+        max_length=255, unique=False, verbose_name=_("Currency Code")
+    )
     name = models.CharField(max_length=50, verbose_name=_("Currency Name"), unique=True)
     decimal_places = models.PositiveIntegerField(
         default=2,
@@ -72,7 +79,161 @@ class ExchangeRate(models.Model):
 
     def clean(self):
         super().clean()
-        if self.from_currency == self.to_currency:
+        # Check if the attributes exist before comparing them
+        if hasattr(self, "from_currency") and hasattr(self, "to_currency"):
+            if self.from_currency == self.to_currency:
+                raise ValidationError(
+                    {"to_currency": _("From and To currencies cannot be the same.")}
+                )
+
+
+class ExchangeRateService(models.Model):
+    """Configuration for exchange rate services"""
+
+    class ServiceType(models.TextChoices):
+        SYNTH_FINANCE = "synth_finance", "Synth Finance"
+        COINGECKO_FREE = "coingecko_free", "CoinGecko (Demo/Free)"
+        COINGECKO_PRO = "coingecko_pro", "CoinGecko (Pro)"
+
+    class IntervalType(models.TextChoices):
+        ON = "on", _("On")
+        EVERY = "every", _("Every X hours")
+        NOT_ON = "not_on", _("Not on")
+
+    name = models.CharField(max_length=255, unique=True, verbose_name=_("Service Name"))
+    service_type = models.CharField(
+        max_length=255, choices=ServiceType.choices, verbose_name=_("Service Type")
+    )
+    is_active = models.BooleanField(default=True, verbose_name=_("Active"))
+    api_key = models.CharField(
+        max_length=255,
+        blank=True,
+        null=True,
+        verbose_name=_("API Key"),
+        help_text=_("API key for the service (if required)"),
+    )
+    interval_type = models.CharField(
+        max_length=255,
+        choices=IntervalType.choices,
+        verbose_name=_("Interval Type"),
+        default=IntervalType.EVERY,
+    )
+    fetch_interval = models.CharField(
+        max_length=1000, verbose_name=_("Interval"), default="24"
+    )
+    last_fetch = models.DateTimeField(
+        null=True, blank=True, verbose_name=_("Last Successful Fetch")
+    )
+
+    target_currencies = models.ManyToManyField(
+        Currency,
+        verbose_name=_("Target Currencies"),
+        help_text=_(
+            "Select currencies to fetch exchange rates for. Rates will be fetched for each currency against their set exchange currency."
+        ),
+        related_name="exchange_services",
+        blank=True,
+    )
+
+    target_accounts = models.ManyToManyField(
+        "accounts.Account",
+        verbose_name=_("Target Accounts"),
+        help_text=_(
+            "Select accounts to fetch exchange rates for. Rates will be fetched for each account's currency against their set exchange currency."
+        ),
+        related_name="exchange_services",
+        blank=True,
+    )
+
+    class Meta:
+        verbose_name = _("Exchange Rate Service")
+        verbose_name_plural = _("Exchange Rate Services")
+        ordering = ["name"]
+
+    def __str__(self):
+        return self.name
+
+    def get_provider(self):
+        from apps.currencies.exchange_rates.fetcher import PROVIDER_MAPPING
+
+        provider_class = PROVIDER_MAPPING[self.service_type]
+        return provider_class(self.api_key)
+
+    @staticmethod
+    def _parse_hour_ranges(interval_str: str) -> Set[int]:
+        """
+        Parse hour ranges and individual hours from string.
+
+        Valid formats:
+        - Single hours: "1,5,9"
+        - Ranges: "1-5"
+        - Mixed: "1-5,8,10-12"
+
+        Returns set of hours.
+        """
+        hours = set()
+
+        for part in interval_str.strip().split(","):
+            part = part.strip()
+            if "-" in part:
+                start, end = part.split("-")
+                start, end = int(start), int(end)
+                if not (0 <= start <= 23 and 0 <= end <= 23):
+                    raise ValueError("Hours must be between 0 and 23")
+                if start > end:
+                    raise ValueError(f"Invalid range: {start}-{end}")
+                hours.update(range(start, end + 1))
+            else:
+                hour = int(part)
+                if not 0 <= hour <= 23:
+                    raise ValueError("Hours must be between 0 and 23")
+                hours.add(hour)
+
+        return hours
+
+    def clean(self):
+        super().clean()
+        try:
+            if self.interval_type == self.IntervalType.EVERY:
+                if not self.fetch_interval.isdigit():
+                    raise ValidationError(
+                        {
+                            "fetch_interval": _(
+                                "'Every X hours' interval type requires a positive integer."
+                            )
+                        }
+                    )
+                hours = int(self.fetch_interval)
+                if hours < 0 or hours > 23:
+                    raise ValidationError(
+                        {
+                            "fetch_interval": _(
+                                "'Every X hours' interval must be between 0 and 23."
+                            )
+                        }
+                    )
+            else:
+                try:
+                    # Parse and validate hour ranges
+                    hours = self._parse_hour_ranges(self.fetch_interval)
+                    # Store in normalized format (optional)
+                    self.fetch_interval = ",".join(str(h) for h in sorted(hours))
+                except ValueError as e:
+                    raise ValidationError(
+                        {
+                            "fetch_interval": _(
+                                "Invalid hour format. Use comma-separated hours (0-23) "
+                                "and/or ranges (e.g., '1-5,8,10-12')."
+                            )
+                        }
+                    )
+        except ValidationError:
+            raise
+        except Exception as e:
             raise ValidationError(
-                {"to_currency": _("From and To currencies cannot be the same.")}
+                {
+                    "fetch_interval": _(
+                        "Invalid format. Please check the requirements for your selected interval type."
+                    )
+                }
             )

app/apps/currencies/tasks.py

@@ -0,0 +1,30 @@
+import logging
+
+from procrastinate.contrib.django import app
+
+from apps.currencies.exchange_rates.fetcher import ExchangeRateFetcher
+
+logger = logging.getLogger(__name__)
+
+
+@app.periodic(cron="0 * * * *")  # Run every hour
+@app.task(name="automatic_fetch_exchange_rates")
+def automatic_fetch_exchange_rates(timestamp=None):
+    """Fetch exchange rates for all due services"""
+    fetcher = ExchangeRateFetcher()
+
+    try:
+        fetcher.fetch_due_rates()
+    except Exception as e:
+        logger.error(e, exc_info=True)
+
+
+@app.task(name="manual_fetch_exchange_rates")
+def manual_fetch_exchange_rates(timestamp=None):
+    """Fetch exchange rates for all due services"""
+    fetcher = ExchangeRateFetcher()
+
+    try:
+        fetcher.fetch_due_rates(force=True)
+    except Exception as e:
+        logger.error(e, exc_info=True)

app/apps/currencies/urls.py

@@ -34,4 +34,34 @@ urlpatterns = [
         views.exchange_rate_delete,
         name="exchange_rate_delete",
     ),
+    path(
+        "automatic-exchange-rates/",
+        views.exchange_rates_services_index,
+        name="automatic_exchange_rates_index",
+    ),
+    path(
+        "automatic-exchange-rates/list/",
+        views.exchange_rates_services_list,
+        name="automatic_exchange_rates_list",
+    ),
+    path(
+        "automatic-exchange-rates/add/",
+        views.exchange_rate_service_add,
+        name="automatic_exchange_rate_add",
+    ),
+    path(
+        "automatic-exchange-rates/force-fetch/",
+        views.exchange_rate_service_force_fetch,
+        name="automatic_exchange_rate_force_fetch",
+    ),
+    path(
+        "automatic-exchange-rates/<int:pk>/edit/",
+        views.exchange_rate_service_edit,
+        name="automatic_exchange_rate_edit",
+    ),
+    path(
+        "automatic-exchange-rates/<int:pk>/delete/",
+        views.exchange_rate_service_delete,
+        name="automatic_exchange_rate_delete",
+    ),
 ]

app/apps/currencies/views/__init__.py

@@ -1,2 +1,3 @@
 from .currencies import *
 from .exchange_rates import *
+from .exchange_rates_services import *

app/apps/currencies/views/currencies.py

@@ -2,9 +2,7 @@ from django.contrib import messages
 from django.contrib.auth.decorators import login_required
 from django.http import HttpResponse
 from django.shortcuts import render, get_object_or_404
-from django.urls import reverse
 from django.utils.translation import gettext_lazy as _
-from django.views.decorators.csrf import csrf_exempt
 from django.views.decorators.http import require_http_methods
 
 from apps.common.decorators.htmx import only_htmx
@@ -89,7 +87,6 @@ def currency_edit(request, pk):
 
 @only_htmx
 @login_required
-@csrf_exempt
 @require_http_methods(["DELETE"])
 def currency_delete(request, pk):
     currency = get_object_or_404(Currency, id=pk)

app/apps/currencies/views/exchange_rates.py

@@ -1,12 +1,11 @@
 from django.contrib import messages
 from django.contrib.auth.decorators import login_required
 from django.core.paginator import Paginator
-from django.db.models import F, CharField, Value
+from django.db.models import CharField, Value
 from django.db.models.functions import Concat
 from django.http import HttpResponse
 from django.shortcuts import render, get_object_or_404
 from django.utils.translation import gettext_lazy as _
-from django.views.decorators.csrf import csrf_exempt
 from django.views.decorators.http import require_http_methods
 
 from apps.common.decorators.htmx import only_htmx
@@ -28,17 +27,17 @@ def exchange_rates_index(request):
 @require_http_methods(["GET"])
 def exchange_rates_list(request):
     pairings = (
-        ExchangeRate.objects.values("from_currency__code", "to_currency__code")
+        ExchangeRate.objects.values("from_currency__name", "to_currency__name")
         .distinct()
         .annotate(
             pair=Concat(
-                "from_currency__code",
+                "from_currency__name",
                 Value(" x "),
-                "to_currency__code",
+                "to_currency__name",
                 output_field=CharField(),
             )
         )
-        .values_list("pair", "from_currency__code", "to_currency__code")
+        .values_list("pair", "from_currency__name", "to_currency__name")
     )
 
     return render(
@@ -57,7 +56,7 @@ def exchange_rates_list_pair(request):
 
     if from_currency and to_currency:
         exchange_rates = ExchangeRate.objects.filter(
-            from_currency__code=from_currency, to_currency__code=to_currency
+            from_currency__name=from_currency, to_currency__name=to_currency
         ).order_by("-date")
     else:
         exchange_rates = ExchangeRate.objects.all().order_by("-date")
@@ -135,7 +134,6 @@ def exchange_rate_edit(request, pk):
 
 @only_htmx
 @login_required
-@csrf_exempt
 @require_http_methods(["DELETE"])
 def exchange_rate_delete(request, pk):
     exchange_rate = get_object_or_404(ExchangeRate, id=pk)

app/apps/currencies/views/exchange_rates_services.py

@@ -0,0 +1,122 @@
+from django.contrib import messages
+from django.contrib.auth.decorators import login_required
+from django.db.models import CharField, Value
+from django.db.models.functions import Concat
+from django.http import HttpResponse
+from django.shortcuts import render, get_object_or_404
+from django.utils.translation import gettext_lazy as _
+from django.views.decorators.http import require_http_methods
+
+from apps.common.decorators.htmx import only_htmx
+from apps.currencies.forms import ExchangeRateForm, ExchangeRateServiceForm
+from apps.currencies.models import ExchangeRate, ExchangeRateService
+from apps.currencies.tasks import manual_fetch_exchange_rates
+
+
+@login_required
+@require_http_methods(["GET"])
+def exchange_rates_services_index(request):
+    return render(
+        request,
+        "exchange_rates_services/pages/index.html",
+    )
+
+
+@only_htmx
+@login_required
+@require_http_methods(["GET"])
+def exchange_rates_services_list(request):
+    services = ExchangeRateService.objects.all()
+
+    return render(
+        request,
+        "exchange_rates_services/fragments/list.html",
+        {"services": services},
+    )
+
+
+@only_htmx
+@login_required
+@require_http_methods(["GET", "POST"])
+def exchange_rate_service_add(request):
+    if request.method == "POST":
+        form = ExchangeRateServiceForm(request.POST)
+        if form.is_valid():
+            form.save()
+            messages.success(request, _("Service added successfully"))
+
+            return HttpResponse(
+                status=204,
+                headers={
+                    "HX-Trigger": "updated, hide_offcanvas",
+                },
+            )
+    else:
+        form = ExchangeRateServiceForm()
+
+    return render(
+        request,
+        "exchange_rates_services/fragments/add.html",
+        {"form": form},
+    )
+
+
+@only_htmx
+@login_required
+@require_http_methods(["GET", "POST"])
+def exchange_rate_service_edit(request, pk):
+    service = get_object_or_404(ExchangeRateService, id=pk)
+
+    if request.method == "POST":
+        form = ExchangeRateServiceForm(request.POST, instance=service)
+        if form.is_valid():
+            form.save()
+            messages.success(request, _("Service updated successfully"))
+
+            return HttpResponse(
+                status=204,
+                headers={
+                    "HX-Trigger": "updated, hide_offcanvas",
+                },
+            )
+    else:
+        form = ExchangeRateServiceForm(instance=service)
+
+    return render(
+        request,
+        "exchange_rates_services/fragments/edit.html",
+        {"form": form, "service": service},
+    )
+
+
+@only_htmx
+@login_required
+@require_http_methods(["DELETE"])
+def exchange_rate_service_delete(request, pk):
+    service = get_object_or_404(ExchangeRateService, id=pk)
+
+    service.delete()
+
+    messages.success(request, _("Service deleted successfully"))
+
+    return HttpResponse(
+        status=204,
+        headers={
+            "HX-Trigger": "updated, hide_offcanvas",
+        },
+    )
+
+
+@only_htmx
+@login_required
+@require_http_methods(["GET"])
+def exchange_rate_service_force_fetch(request):
+    manual_fetch_exchange_rates.defer()
+    messages.success(request, _("Services queued successfully"))
+
+    return HttpResponse(
+        status=204,
+        headers={
+            "HX-Trigger": "toasts",
+        },
+    )

app/apps/dca/forms.py

@@ -1,13 +1,22 @@
-from crispy_forms.bootstrap import FormActions
-from django import forms
+from crispy_bootstrap5.bootstrap5 import Switch, BS5Accordion
+from crispy_forms.bootstrap import FormActions, AccordionGroup
 from crispy_forms.helper import FormHelper
-from crispy_forms.layout import Layout, Row, Column
+from crispy_forms.layout import Layout, Row, Column, HTML
+from django import forms
 from django.utils.translation import gettext_lazy as _
 
+from apps.accounts.models import Account
+from apps.common.widgets.crispy.submit import NoClassSubmit
+from apps.common.widgets.datepicker import AirDatePickerInput
+from apps.common.widgets.decimal import ArbitraryDecimalDisplayNumberInput
 from apps.common.widgets.tom_select import TomSelect
 from apps.dca.models import DCAStrategy, DCAEntry
-from apps.common.widgets.decimal import ArbitraryDecimalDisplayNumberInput
-from apps.common.widgets.crispy.submit import NoClassSubmit
+from apps.common.widgets.tom_select import TransactionSelect
+from apps.transactions.models import Transaction, TransactionTag, TransactionCategory
+from apps.common.fields.forms.dynamic_select import (
+    DynamicModelChoiceField,
+    DynamicModelMultipleChoiceField,
+)
 
 
 class DCAStrategyForm(forms.ModelForm):
@@ -52,6 +61,75 @@ class DCAStrategyForm(forms.ModelForm):
 
 
 class DCAEntryForm(forms.ModelForm):
+    create_transaction = forms.BooleanField(
+        label=_("Create transaction"), initial=False, required=False
+    )
+
+    from_account = forms.ModelChoiceField(
+        queryset=Account.objects.filter(is_archived=False),
+        label=_("From Account"),
+        widget=TomSelect(clear_button=False, group_by="group"),
+        required=False,
+    )
+    to_account = forms.ModelChoiceField(
+        queryset=Account.objects.filter(is_archived=False),
+        label=_("To Account"),
+        widget=TomSelect(clear_button=False, group_by="group"),
+        required=False,
+    )
+
+    from_category = DynamicModelChoiceField(
+        create_field="name",
+        model=TransactionCategory,
+        required=False,
+        label=_("Category"),
+        queryset=TransactionCategory.objects.filter(active=True),
+    )
+    to_category = DynamicModelChoiceField(
+        create_field="name",
+        model=TransactionCategory,
+        required=False,
+        label=_("Category"),
+        queryset=TransactionCategory.objects.filter(active=True),
+    )
+
+    from_tags = DynamicModelMultipleChoiceField(
+        model=TransactionTag,
+        to_field_name="name",
+        create_field="name",
+        required=False,
+        label=_("Tags"),
+        queryset=TransactionTag.objects.filter(active=True),
+    )
+    to_tags = DynamicModelMultipleChoiceField(
+        model=TransactionTag,
+        to_field_name="name",
+        create_field="name",
+        required=False,
+        label=_("Tags"),
+        queryset=TransactionTag.objects.filter(active=True),
+    )
+
+    expense_transaction = DynamicModelChoiceField(
+        model=Transaction,
+        to_field_name="id",
+        label=_("Expense Transaction"),
+        required=False,
+        queryset=Transaction.objects.none(),
+        widget=TransactionSelect(clear_button=True, income=False, expense=True),
+        help_text=_("Type to search for a transaction to link to this entry"),
+    )
+
+    income_transaction = DynamicModelChoiceField(
+        model=Transaction,
+        to_field_name="id",
+        label=_("Income Transaction"),
+        required=False,
+        queryset=Transaction.objects.none(),
+        widget=TransactionSelect(clear_button=True, income=True, expense=False),
+        help_text=_("Type to search for a transaction to link to this entry"),
+    )
+
     class Meta:
         model = DCAEntry
         fields = [
@@ -59,14 +137,19 @@ class DCAEntryForm(forms.ModelForm):
             "amount_paid",
             "amount_received",
             "notes",
+            "expense_transaction",
+            "income_transaction",
         ]
         widgets = {
-            "date": forms.DateInput(attrs={"type": "date"}, format="%Y-%m-%d"),
             "notes": forms.Textarea(attrs={"rows": 3}),
         }
 
     def __init__(self, *args, **kwargs):
+        strategy = kwargs.pop("strategy", None)
         super().__init__(*args, **kwargs)
+
+        self.strategy = strategy if strategy else self.instance.strategy
+
         self.helper = FormHelper()
         self.helper.form_tag = False
         self.helper.layout = Layout(
@@ -75,18 +158,66 @@ class DCAEntryForm(forms.ModelForm):
                 Column("amount_paid", css_class="form-group col-md-6"),
                 Column("amount_received", css_class="form-group col-md-6"),
             ),
-            Row(
-                Column("expense_transaction", css_class="form-group col-md-6"),
-                Column("income_transaction", css_class="form-group col-md-6"),
-            ),
             "notes",
+            BS5Accordion(
+                AccordionGroup(
+                    _("Create transaction"),
+                    Switch("create_transaction"),
+                    Row(
+                        Column(
+                            Row(
+                                Column(
+                                    "from_account",
+                                    css_class="form-group col-md-6 mb-0",
+                                ),
+                                css_class="form-row",
+                            ),
+                            Row(
+                                Column(
+                                    "from_category",
+                                    css_class="form-group col-md-6 mb-0",
+                                ),
+                                Column(
+                                    "from_tags", css_class="form-group col-md-6 mb-0"
+                                ),
+                                css_class="form-row",
+                            ),
+                        ),
+                        css_class="p-1 mx-1 my-3 border rounded-3",
+                    ),
+                    Row(
+                        Column(
+                            Row(
+                                Column(
+                                    "to_account",
+                                    css_class="form-group col-md-6 mb-0",
+                                ),
+                                css_class="form-row",
+                            ),
+                            Row(
+                                Column(
+                                    "to_category", css_class="form-group col-md-6 mb-0"
+                                ),
+                                Column("to_tags", css_class="form-group col-md-6 mb-0"),
+                                css_class="form-row",
+                            ),
+                        ),
+                        css_class="p-1 mx-1 my-3 border rounded-3",
+                    ),
+                    active=False,
+                ),
+                AccordionGroup(
+                    _("Link transaction"),
+                    "income_transaction",
+                    "expense_transaction",
+                ),
+                flush=False,
+                always_open=False,
+                css_class="mb-3",
+            ),
         )
 
         if self.instance and self.instance.pk:
-            # decimal_places = self.instance.account.currency.decimal_places
-            # self.fields["amount"].widget = ArbitraryDecimalDisplayNumberInput(
-            #     decimal_places=decimal_places
-            # )
             self.helper.layout.append(
                 FormActions(
                     NoClassSubmit(
@@ -95,7 +226,6 @@ class DCAEntryForm(forms.ModelForm):
                 ),
             )
         else:
-            # self.fields["amount"].widget = ArbitraryDecimalDisplayNumberInput()
             self.helper.layout.append(
                 FormActions(
                     NoClassSubmit(
@@ -106,3 +236,119 @@ class DCAEntryForm(forms.ModelForm):
 
         self.fields["amount_paid"].widget = ArbitraryDecimalDisplayNumberInput()
         self.fields["amount_received"].widget = ArbitraryDecimalDisplayNumberInput()
+        self.fields["date"].widget = AirDatePickerInput(clear_button=False)
+
+        expense_transaction = None
+        income_transaction = None
+        if self.instance and self.instance.pk:
+            # Edit mode - get from instance
+            expense_transaction = self.instance.expense_transaction
+            income_transaction = self.instance.income_transaction
+        elif self.data.get("expense_transaction"):
+            # Form validation - get from submitted data
+            try:
+                expense_transaction = Transaction.objects.get(
+                    id=self.data["expense_transaction"]
+                )
+                income_transaction = Transaction.objects.get(
+                    id=self.data["income_transaction"]
+                )
+            except Transaction.DoesNotExist:
+                pass
+
+            # If we have a current transaction, ensure it's in the queryset
+        if income_transaction:
+            self.fields["income_transaction"].queryset = Transaction.objects.filter(
+                id=income_transaction.id
+            )
+        if expense_transaction:
+            self.fields["expense_transaction"].queryset = Transaction.objects.filter(
+                id=expense_transaction.id
+            )
+
+    def clean(self):
+        cleaned_data = super().clean()
+
+        if cleaned_data.get("create_transaction"):
+            from_account = cleaned_data.get("from_account")
+            to_account = cleaned_data.get("to_account")
+
+            if not from_account and not to_account:
+                raise forms.ValidationError(
+                    {
+                        "from_account": _("You must provide an account."),
+                        "to_account": _("You must provide an account."),
+                    }
+                )
+            elif not from_account and to_account:
+                raise forms.ValidationError(
+                    {"from_account": _("You must provide an account.")}
+                )
+            elif not to_account and from_account:
+                raise forms.ValidationError(
+                    {"to_account": _("You must provide an account.")}
+                )
+
+            if from_account == to_account:
+                raise forms.ValidationError(
+                    _("From and To accounts must be different.")
+                )
+
+        return cleaned_data
+
+    def save(self, **kwargs):
+        instance = super().save(commit=False)
+
+        if self.cleaned_data.get("create_transaction"):
+            from_account = self.cleaned_data["from_account"]
+            to_account = self.cleaned_data["to_account"]
+            from_amount = instance.amount_paid
+            to_amount = instance.amount_received
+            date = instance.date
+            description = _("DCA for %(strategy_name)s") % {
+                "strategy_name": self.strategy.name
+            }
+            from_category = self.cleaned_data.get("from_category")
+            to_category = self.cleaned_data.get("to_category")
+            notes = self.cleaned_data.get("notes")
+
+            # Create "From" transaction
+            from_transaction = Transaction.objects.create(
+                account=from_account,
+                type=Transaction.Type.EXPENSE,
+                is_paid=True,
+                date=date,
+                amount=from_amount,
+                description=description,
+                category=from_category,
+                notes=notes,
+            )
+            from_transaction.tags.set(self.cleaned_data.get("from_tags", []))
+
+            # Create "To" transaction
+            to_transaction = Transaction.objects.create(
+                account=to_account,
+                type=Transaction.Type.INCOME,
+                is_paid=True,
+                date=date,
+                amount=to_amount,
+                description=description,
+                category=to_category,
+                notes=notes,
+            )
+            to_transaction.tags.set(self.cleaned_data.get("to_tags", []))
+
+            instance.expense_transaction = from_transaction
+            instance.income_transaction = to_transaction
+        else:
+            if instance.expense_transaction:
+                instance.expense_transaction.amount = instance.amount_paid
+                instance.expense_transaction.save()
+            if instance.income_transaction:
+                instance.income_transaction.amount = instance.amount_received
+                instance.income_transaction.save()
+
+        instance.strategy = self.strategy
+        instance.save()
+
+        return instance

app/apps/dca/views.py

@@ -6,12 +6,11 @@ from django.db.models.functions import TruncMonth
 from django.http import HttpResponse
 from django.shortcuts import render, get_object_or_404
 from django.utils.translation import gettext_lazy as _
-from django.views.decorators.csrf import csrf_exempt
 from django.views.decorators.http import require_http_methods
 
 from apps.common.decorators.htmx import only_htmx
-from apps.dca.models import DCAStrategy, DCAEntry
 from apps.dca.forms import DCAEntryForm, DCAStrategyForm
+from apps.dca.models import DCAStrategy, DCAEntry
 
 
 @login_required
@@ -82,7 +81,6 @@ def strategy_edit(request, strategy_id):
 
 @only_htmx
 @login_required
-@csrf_exempt
 @require_http_methods(["DELETE"])
 def strategy_delete(request, strategy_id):
     dca_strategy = get_object_or_404(DCAStrategy, id=strategy_id)
@@ -157,11 +155,9 @@ def strategy_detail(request, strategy_id):
 def strategy_entry_add(request, strategy_id):
     strategy = get_object_or_404(DCAStrategy, id=strategy_id)
     if request.method == "POST":
-        form = DCAEntryForm(request.POST)
+        form = DCAEntryForm(request.POST, strategy=strategy)
         if form.is_valid():
-            entry = form.save(commit=False)
-            entry.strategy = strategy
-            entry.save()
+            entry = form.save()
             messages.success(request, _("Entry added successfully"))
 
             return HttpResponse(
@@ -171,7 +167,7 @@ def strategy_entry_add(request, strategy_id):
                 },
             )
     else:
-        form = DCAEntryForm()
+        form = DCAEntryForm(strategy=strategy)
 
     return render(
         request,
@@ -209,7 +205,6 @@ def strategy_entry_edit(request, strategy_id, entry_id):
 
 @only_htmx
 @login_required
-@csrf_exempt
 @require_http_methods(["DELETE"])
 def strategy_entry_delete(request, entry_id, strategy_id):
     dca_entry = get_object_or_404(DCAEntry, id=entry_id, strategy__id=strategy_id)

app/apps/export_app/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.

app/apps/export_app/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class ExportConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "apps.export_app"

app/apps/export_app/forms.py

@@ -0,0 +1,189 @@
+from crispy_forms.bootstrap import FormActions
+from crispy_forms.helper import FormHelper
+from crispy_forms.layout import Layout, HTML
+from django import forms
+from django.utils.translation import gettext_lazy as _
+
+from apps.common.widgets.crispy.submit import NoClassSubmit
+
+
+class ExportForm(forms.Form):
+    accounts = forms.BooleanField(
+        required=False,
+        widget=forms.CheckboxInput(),
+        label=_("Accounts"),
+        initial=True,
+    )
+    currencies = forms.BooleanField(
+        required=False,
+        widget=forms.CheckboxInput(),
+        label=_("Currencies"),
+        initial=True,
+    )
+    transactions = forms.BooleanField(
+        required=False,
+        widget=forms.CheckboxInput(),
+        label=_("Transactions"),
+        initial=True,
+    )
+    categories = forms.BooleanField(
+        required=False,
+        widget=forms.CheckboxInput(),
+        label=_("Categories"),
+        initial=True,
+    )
+    tags = forms.BooleanField(
+        required=False,
+        widget=forms.CheckboxInput(),
+        label=_("Tags"),
+        initial=False,
+    )
+    entities = forms.BooleanField(
+        required=False,
+        widget=forms.CheckboxInput(),
+        label=_("Entities"),
+        initial=False,
+    )
+    recurring_transactions = forms.BooleanField(
+        required=False,
+        widget=forms.CheckboxInput(),
+        label=_("Recurring Transactions"),
+        initial=True,
+    )
+    installment_plans = forms.BooleanField(
+        required=False,
+        widget=forms.CheckboxInput(),
+        label=_("Installment Plans"),
+        initial=True,
+    )
+    exchange_rates = forms.BooleanField(
+        required=False,
+        widget=forms.CheckboxInput(),
+        label=_("Exchange Rates"),
+        initial=False,
+    )
+    exchange_rates_services = forms.BooleanField(
+        required=False,
+        widget=forms.CheckboxInput(),
+        label=_("Automatic Exchange Rates"),
+        initial=False,
+    )
+    rules = forms.BooleanField(
+        required=False,
+        widget=forms.CheckboxInput(),
+        label=_("Rules"),
+        initial=True,
+    )
+    dca = forms.BooleanField(
+        required=False,
+        widget=forms.CheckboxInput(),
+        label=_("DCA"),
+        initial=False,
+    )
+    import_profiles = forms.BooleanField(
+        required=False,
+        widget=forms.CheckboxInput(),
+        label=_("Import Profiles"),
+        initial=True,
+    )
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self.helper = FormHelper()
+        self.helper.form_tag = False
+        self.helper.form_method = "post"
+        self.helper.layout = Layout(
+            "accounts",
+            "currencies",
+            "transactions",
+            "categories",
+            "entities",
+            "tags",
+            "installment_plans",
+            "recurring_transactions",
+            "exchange_rates_services",
+            "exchange_rates",
+            "rules",
+            "dca",
+            "import_profiles",
+            FormActions(
+                NoClassSubmit(
+                    "submit", _("Export"), css_class="btn btn-outline-primary w-100"
+                ),
+            ),
+        )
+
+
+class RestoreForm(forms.Form):
+    zip_file = forms.FileField(
+        required=False,
+        help_text=_("Import a ZIP file exported from WYGIWYH"),
+        label=_("ZIP File"),
+    )
+    accounts = forms.FileField(required=False, label=_("Accounts"))
+    currencies = forms.FileField(required=False, label=_("Currencies"))
+    transactions_categories = forms.FileField(required=False, label=_("Categories"))
+    transactions_tags = forms.FileField(required=False, label=_("Tags"))
+    transactions_entities = forms.FileField(required=False, label=_("Entities"))
+    transactions = forms.FileField(required=False, label=_("Transactions"))
+    installment_plans = forms.FileField(required=False, label=_("Installment Plans"))
+    recurring_transactions = forms.FileField(
+        required=False, label=_("Recurring Transactions")
+    )
+    automatic_exchange_rates = forms.FileField(
+        required=False, label=_("Automatic Exchange Rates")
+    )
+    exchange_rates = forms.FileField(required=False, label=_("Exchange Rates"))
+    transaction_rules = forms.FileField(required=False, label=_("Transaction rules"))
+    transaction_rules_actions = forms.FileField(
+        required=False, label=_("Edit transaction action")
+    )
+    transaction_rules_update_or_create = forms.FileField(
+        required=False, label=_("Update or create transaction actions")
+    )
+    dca_strategies = forms.FileField(required=False, label=_("DCA Strategies"))
+    dca_entries = forms.FileField(required=False, label=_("DCA Entries"))
+    import_profiles = forms.FileField(required=False, label=_("Import Profiles"))
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self.helper = FormHelper()
+        self.helper.form_tag = False
+        self.helper.form_method = "post"
+        self.helper.layout = Layout(
+            "zip_file",
+            HTML("<hr />"),
+            "accounts",
+            "currencies",
+            "transactions",
+            "transactions_categories",
+            "transactions_entities",
+            "transactions_tags",
+            "installment_plans",
+            "recurring_transactions",
+            "automatic_exchange_rates",
+            "exchange_rates",
+            "transaction_rules",
+            "transaction_rules_actions",
+            "transaction_rules_update_or_create",
+            "dca_strategies",
+            "dca_entries",
+            "import_profiles",
+            FormActions(
+                NoClassSubmit(
+                    "submit", _("Restore"), css_class="btn btn-outline-primary w-100"
+                ),
+            ),
+        )
+
+    def clean(self):
+        cleaned_data = super().clean()
+        if not cleaned_data.get("zip_file") and not any(
+            cleaned_data.get(field) for field in self.fields if field != "zip_file"
+        ):
+            raise forms.ValidationError(
+                _("Please upload either a ZIP file or at least one CSV file")
+            )
+        return cleaned_data

app/apps/export_app/models.py

@@ -0,0 +1,3 @@
+from django.db import models
+
+# Create your models here.

app/apps/export_app/resources/accounts.py

@@ -0,0 +1,26 @@
+from import_export import fields, resources, widgets
+
+from apps.accounts.models import Account, AccountGroup
+from apps.export_app.widgets.foreign_key import AutoCreateForeignKeyWidget
+from apps.currencies.models import Currency
+
+
+class AccountResource(resources.ModelResource):
+    group = fields.Field(
+        attribute="group",
+        column_name="group",
+        widget=AutoCreateForeignKeyWidget(AccountGroup, "name"),
+    )
+    currency = fields.Field(
+        attribute="currency",
+        column_name="currency",
+        widget=widgets.ForeignKeyWidget(Currency, "name"),
+    )
+    exchange_currency = fields.Field(
+        attribute="exchange_currency",
+        column_name="exchange_currency",
+        widget=widgets.ForeignKeyWidget(Currency, "name"),
+    )
+
+    class Meta:
+        model = Account

app/apps/export_app/resources/currencies.py

@@ -0,0 +1,47 @@
+from import_export import fields, resources, widgets
+
+from apps.accounts.models import Account
+from apps.currencies.models import Currency, ExchangeRate, ExchangeRateService
+
+
+class CurrencyResource(resources.ModelResource):
+    exchange_currency = fields.Field(
+        attribute="exchange_currency",
+        column_name="exchange_currency",
+        widget=widgets.ForeignKeyWidget(Currency, "name"),
+    )
+
+    class Meta:
+        model = Currency
+
+
+class ExchangeRateResource(resources.ModelResource):
+    from_currency = fields.Field(
+        attribute="from_currency",
+        column_name="from_currency",
+        widget=widgets.ForeignKeyWidget(Currency, "name"),
+    )
+    to_currency = fields.Field(
+        attribute="to_currency",
+        column_name="to_currency",
+        widget=widgets.ForeignKeyWidget(Currency, "name"),
+    )
+
+    class Meta:
+        model = ExchangeRate
+
+
+class ExchangeRateServiceResource(resources.ModelResource):
+    target_currencies = fields.Field(
+        attribute="target_currencies",
+        column_name="target_currencies",
+        widget=widgets.ManyToManyWidget(Currency, field="name"),
+    )
+    target_accounts = fields.Field(
+        attribute="target_accounts",
+        column_name="target_accounts",
+        widget=widgets.ManyToManyWidget(Account, field="name"),
+    )
+
+    class Meta:
+        model = ExchangeRateService

app/apps/export_app/resources/dca.py

@@ -0,0 +1,26 @@
+from import_export import fields, resources
+from import_export.widgets import ForeignKeyWidget
+
+from apps.dca.models import DCAStrategy, DCAEntry
+from apps.currencies.models import Currency
+
+
+class DCAStrategyResource(resources.ModelResource):
+    target_currency = fields.Field(
+        attribute="target_currency",
+        column_name="target_currency",
+        widget=ForeignKeyWidget(Currency, "name"),
+    )
+    payment_currency = fields.Field(
+        attribute="payment_currency",
+        column_name="payment_currency",
+        widget=ForeignKeyWidget(Currency, "name"),
+    )
+
+    class Meta:
+        model = DCAStrategy
+
+
+class DCAEntryResource(resources.ModelResource):
+    class Meta:
+        model = DCAEntry

app/apps/export_app/resources/import_app.py

@@ -0,0 +1,8 @@
+from import_export import resources
+
+from apps.import_app.models import ImportProfile
+
+
+class ImportProfileResource(resources.ModelResource):
+    class Meta:
+        model = ImportProfile

app/apps/export_app/resources/rules.py

@@ -0,0 +1,25 @@
+from import_export import fields, resources
+from import_export.widgets import ForeignKeyWidget
+
+from apps.export_app.widgets.foreign_key import AutoCreateForeignKeyWidget
+from apps.export_app.widgets.many_to_many import AutoCreateManyToManyWidget
+from apps.rules.models import (
+    TransactionRule,
+    TransactionRuleAction,
+    UpdateOrCreateTransactionRuleAction,
+)
+
+
+class TransactionRuleResource(resources.ModelResource):
+    class Meta:
+        model = TransactionRule
+
+
+class TransactionRuleActionResource(resources.ModelResource):
+    class Meta:
+        model = TransactionRuleAction
+
+
+class UpdateOrCreateTransactionRuleResource(resources.ModelResource):
+    class Meta:
+        model = UpdateOrCreateTransactionRuleAction

app/apps/export_app/resources/transactions.py

@@ -0,0 +1,124 @@
+from import_export import fields, resources
+from import_export.widgets import ForeignKeyWidget
+
+from apps.accounts.models import Account
+from apps.export_app.widgets.foreign_key import AutoCreateForeignKeyWidget
+from apps.export_app.widgets.many_to_many import AutoCreateManyToManyWidget
+from apps.export_app.widgets.string import EmptyStringToNoneField
+from apps.transactions.models import (
+    Transaction,
+    TransactionCategory,
+    TransactionTag,
+    TransactionEntity,
+    RecurringTransaction,
+    InstallmentPlan,
+)
+
+
+class TransactionResource(resources.ModelResource):
+    account = fields.Field(
+        attribute="account",
+        column_name="account",
+        widget=ForeignKeyWidget(Account, "name"),
+    )
+
+    category = fields.Field(
+        attribute="category",
+        column_name="category",
+        widget=AutoCreateForeignKeyWidget(TransactionCategory, "name"),
+    )
+
+    tags = fields.Field(
+        attribute="tags",
+        column_name="tags",
+        widget=AutoCreateManyToManyWidget(TransactionTag, field="name"),
+    )
+
+    entities = fields.Field(
+        attribute="entities",
+        column_name="entities",
+        widget=AutoCreateManyToManyWidget(TransactionEntity, field="name"),
+    )
+
+    internal_id = EmptyStringToNoneField(
+        column_name="internal_id", attribute="internal_id"
+    )
+
+    class Meta:
+        model = Transaction
+
+    def get_queryset(self):
+        return Transaction.all_objects.all()
+
+
+class TransactionTagResource(resources.ModelResource):
+    class Meta:
+        model = TransactionTag
+
+
+class TransactionEntityResource(resources.ModelResource):
+    class Meta:
+        model = TransactionEntity
+
+
+class TransactionCategoyResource(resources.ModelResource):
+    class Meta:
+        model = TransactionCategory
+
+
+class RecurringTransactionResource(resources.ModelResource):
+    account = fields.Field(
+        attribute="account",
+        column_name="account",
+        widget=ForeignKeyWidget(Account, "name"),
+    )
+
+    category = fields.Field(
+        attribute="category",
+        column_name="category",
+        widget=AutoCreateForeignKeyWidget(TransactionCategory, "name"),
+    )
+
+    tags = fields.Field(
+        attribute="tags",
+        column_name="tags",
+        widget=AutoCreateManyToManyWidget(TransactionTag, field="name"),
+    )
+
+    entities = fields.Field(
+        attribute="entities",
+        column_name="entities",
+        widget=AutoCreateManyToManyWidget(TransactionEntity, field="name"),
+    )
+
+    class Meta:
+        model = RecurringTransaction
+
+
+class InstallmentPlanResource(resources.ModelResource):
+    account = fields.Field(
+        attribute="account",
+        column_name="account",
+        widget=ForeignKeyWidget(Account, "name"),
+    )
+
+    category = fields.Field(
+        attribute="category",
+        column_name="category",
+        widget=AutoCreateForeignKeyWidget(TransactionCategory, "name"),
+    )
+
+    tags = fields.Field(
+        attribute="tags",
+        column_name="tags",
+        widget=AutoCreateManyToManyWidget(TransactionTag, field="name"),
+    )
+
+    entities = fields.Field(
+        attribute="entities",
+        column_name="entities",
+        widget=AutoCreateManyToManyWidget(TransactionEntity, field="name"),
+    )
+
+    class Meta:
+        model = InstallmentPlan

app/apps/export_app/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

app/apps/export_app/urls.py

@@ -0,0 +1,8 @@
+from django.urls import path
+import apps.export_app.views as views
+
+urlpatterns = [
+    path("export/", views.export_index, name="export_index"),
+    path("export/form/", views.export_form, name="export_form"),
+    path("export/restore/", views.import_form, name="restore_form"),
+]

app/apps/export_app/views.py

@@ -0,0 +1,285 @@
+import logging
+import zipfile
+from io import BytesIO, TextIOWrapper
+
+from django.contrib import messages
+from django.contrib.auth.decorators import login_required
+from django.db import transaction
+from django.http import HttpResponse
+from django.shortcuts import render
+from django.utils import timezone
+from django.utils.translation import gettext_lazy as _
+from django.views.decorators.http import require_http_methods
+from tablib import Dataset
+
+from apps.export_app.forms import ExportForm, RestoreForm
+from apps.export_app.resources.accounts import AccountResource
+from apps.export_app.resources.transactions import (
+    TransactionResource,
+    TransactionTagResource,
+    TransactionEntityResource,
+    TransactionCategoyResource,
+    InstallmentPlanResource,
+    RecurringTransactionResource,
+)
+from apps.export_app.resources.currencies import (
+    CurrencyResource,
+    ExchangeRateResource,
+    ExchangeRateServiceResource,
+)
+from apps.export_app.resources.rules import (
+    TransactionRuleResource,
+    TransactionRuleActionResource,
+    UpdateOrCreateTransactionRuleResource,
+)
+from apps.export_app.resources.dca import (
+    DCAStrategyResource,
+    DCAEntryResource,
+)
+from apps.export_app.resources.import_app import (
+    ImportProfileResource,
+)
+from apps.common.decorators.htmx import only_htmx
+
+logger = logging.getLogger()
+
+
+@login_required
+@require_http_methods(["GET"])
+def export_index(request):
+    return render(request, "export_app/pages/index.html")
+
+
+@login_required
+@require_http_methods(["GET", "POST"])
+def export_form(request):
+    timestamp = timezone.localtime(timezone.now()).strftime("%Y-%m-%dT%H-%M-%S")
+
+    if request.method == "POST":
+        form = ExportForm(request.POST)
+        if form.is_valid():
+            zip_buffer = BytesIO()
+
+            export_accounts = form.cleaned_data.get("accounts", False)
+            export_currencies = form.cleaned_data.get("currencies", False)
+            export_transactions = form.cleaned_data.get("transactions", False)
+            export_categories = form.cleaned_data.get("categories", False)
+            export_tags = form.cleaned_data.get("tags", False)
+            export_entities = form.cleaned_data.get("entities", False)
+            export_installment_plans = form.cleaned_data.get("installment_plans", False)
+            export_recurring_transactions = form.cleaned_data.get(
+                "recurring_transactions", False
+            )
+
+            export_exchange_rates_services = form.cleaned_data.get(
+                "exchange_rates_services", False
+            )
+            export_exchange_rates = form.cleaned_data.get("exchange_rates", False)
+            export_rules = form.cleaned_data.get("rules", False)
+            export_dca = form.cleaned_data.get("dca", False)
+            export_import_profiles = form.cleaned_data.get("import_profiles", False)
+
+            exports = []
+            if export_accounts:
+                exports.append((AccountResource().export(), "accounts"))
+            if export_currencies:
+                exports.append((CurrencyResource().export(), "currencies"))
+            if export_transactions:
+                exports.append((TransactionResource().export(), "transactions"))
+            if export_categories:
+                exports.append(
+                    (TransactionCategoyResource().export(), "transactions_categories")
+                )
+            if export_tags:
+                exports.append((TransactionTagResource().export(), "transactions_tags"))
+            if export_entities:
+                exports.append(
+                    (TransactionEntityResource().export(), "transactions_entities")
+                )
+            if export_installment_plans:
+                exports.append(
+                    (InstallmentPlanResource().export(), "installment_plans")
+                )
+            if export_recurring_transactions:
+                exports.append(
+                    (RecurringTransactionResource().export(), "recurring_transactions")
+                )
+            if export_exchange_rates_services:
+                exports.append(
+                    (ExchangeRateServiceResource().export(), "automatic_exchange_rates")
+                )
+            if export_exchange_rates:
+                exports.append((ExchangeRateResource().export(), "exchange_rates"))
+            if export_rules:
+                exports.append(
+                    (TransactionRuleResource().export(), "transaction_rules")
+                )
+                exports.append(
+                    (
+                        TransactionRuleActionResource().export(),
+                        "transaction_rules_actions",
+                    )
+                )
+                exports.append(
+                    (
+                        UpdateOrCreateTransactionRuleResource().export(),
+                        "transaction_rules_update_or_create",
+                    )
+                )
+            if export_dca:
+                exports.append((DCAStrategyResource().export(), "dca_strategies"))
+                exports.append(
+                    (
+                        DCAEntryResource().export(),
+                        "dca_entries",
+                    )
+                )
+            if export_import_profiles:
+                exports.append((ImportProfileResource().export(), "import_profiles"))
+
+            if len(exports) >= 2:
+                with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zip_file:
+                    for dataset, name in exports:
+                        zip_file.writestr(f"{name}.csv", dataset.csv)
+
+                response = HttpResponse(
+                    zip_buffer.getvalue(),
+                    content_type="application/zip",
+                    headers={
+                        "HX-Trigger": "hide_offcanvas, updated",
+                        "Content-Disposition": f'attachment; filename="{timestamp}_WYGIWYH_export.zip"',
+                    },
+                )
+                return response
+            elif len(exports) == 1:
+                dataset, name = exports[0]
+
+                response = HttpResponse(
+                    dataset.csv,
+                    content_type="text/csv",
+                    headers={
+                        "HX-Trigger": "hide_offcanvas, updated",
+                        "Content-Disposition": f'attachment; filename="{timestamp}_WYGIWYH_export_{name}.csv"',
+                    },
+                )
+                return response
+            else:
+                return HttpResponse(
+                    _("You have to select at least one export"),
+                )
+
+    else:
+        form = ExportForm()
+
+    return render(request, "export_app/fragments/export.html", context={"form": form})
+
+
+@only_htmx
+@login_required
+@require_http_methods(["GET", "POST"])
+def import_form(request):
+    if request.method == "POST":
+        form = RestoreForm(request.POST, request.FILES)
+        if form.is_valid():
+            try:
+                process_imports(request, form.cleaned_data)
+                messages.success(request, _("Data restored successfully"))
+                return HttpResponse(
+                    status=204,
+                    headers={
+                        "HX-Trigger": "hide_offcanvas, updated",
+                    },
+                )
+            except Exception as e:
+                logger.error("Error importing", exc_info=e)
+                messages.error(
+                    request,
+                    _(
+                        "There was an error restoring your data. Check the logs for more details."
+                    ),
+                )
+    else:
+        form = RestoreForm()
+
+    response = render(request, "export_app/fragments/restore.html", {"form": form})
+    response["HX-Trigger"] = "updated"
+    return response
+
+
+def process_imports(request, cleaned_data):
+    # Define import order to handle dependencies
+    import_order = [
+        ("currencies", CurrencyResource),
+        (
+            "currencies",
+            CurrencyResource,
+        ),  # We do a double pass because exchange_currency may not exist when currency is initially created
+        ("accounts", AccountResource),
+        ("transactions_categories", TransactionCategoyResource),
+        ("transactions_tags", TransactionTagResource),
+        ("transactions_entities", TransactionEntityResource),
+        ("automatic_exchange_rates", ExchangeRateServiceResource),
+        ("exchange_rates", ExchangeRateResource),
+        ("installment_plans", InstallmentPlanResource),
+        ("recurring_transactions", RecurringTransactionResource),
+        ("transactions", TransactionResource),
+        ("dca_strategies", DCAStrategyResource),
+        ("dca_entries", DCAEntryResource),
+        ("import_profiles", ImportProfileResource),
+        ("transaction_rules", TransactionRuleResource),
+        ("transaction_rules_actions", TransactionRuleActionResource),
+        ("transaction_rules_update_or_create", UpdateOrCreateTransactionRuleResource),
+    ]
+
+    def import_dataset(content, resource_class, field_name):
+        try:
+            # Create a new resource instance
+            resource = resource_class()
+
+            # Create dataset from CSV content
+            dataset = Dataset()
+            dataset.load(content, format="csv")
+
+            # Debug logging
+            logger.debug(f"Importing {field_name}")
+            logger.debug(f"Headers: {dataset.headers}")
+            logger.debug(f"First row: {dataset[0] if len(dataset) > 0 else 'No data'}")
+
+            # Perform the import
+            result = resource.import_data(
+                dataset,
+                dry_run=False,
+                raise_errors=True,
+                collect_failed_rows=True,
+                use_transactions=False,
+                skip_unchanged=True,
+            )
+
+            if result.has_errors():
+                raise ImportError(f"Failed rows: {result.failed_dataset}")
+
+            return result
+
+        except Exception as e:
+            logger.error(f"Error importing {field_name}: {str(e)}")
+            raise ImportError(f"Error importing {field_name}: {str(e)}")
+
+    with transaction.atomic():
+        if zip_file := cleaned_data.get("zip_file"):
+            # Process ZIP file
+            with zipfile.ZipFile(zip_file) as z:
+                for filename in z.namelist():
+                    name = filename.replace(".csv", "")
+                    with z.open(filename) as f:
+                        content = f.read().decode("utf-8")
+
+                        for field_name, resource_class in import_order:
+                            if name == field_name:
+                                import_dataset(content, resource_class, field_name)
+                                break
+        else:
+            # Process individual files
+            for field_name, resource_class in import_order:
+                if csv_file := cleaned_data.get(field_name):
+                    content = csv_file.read().decode("utf-8")
+                    import_dataset(content, resource_class, field_name)

app/apps/export_app/widgets/foreign_key.py

@@ -0,0 +1,11 @@
+from import_export.widgets import ForeignKeyWidget
+
+
+class AutoCreateForeignKeyWidget(ForeignKeyWidget):
+    def clean(self, value, row=None, *args, **kwargs):
+        if value:
+            try:
+                return super().clean(value, row, **kwargs)
+            except self.model.DoesNotExist:
+                return self.model.objects.create(name=value)
+        return None

app/apps/export_app/widgets/many_to_many.py

@@ -0,0 +1,21 @@
+from import_export.widgets import ManyToManyWidget
+
+
+class AutoCreateManyToManyWidget(ManyToManyWidget):
+    def clean(self, value, row=None, *args, **kwargs):
+        if not value:
+            return []
+
+        values = value.split(self.separator)
+        cleaned_values = []
+
+        for val in values:
+            val = val.strip()
+            if val:
+                try:
+                    obj = self.model.objects.get(**{self.field: val})
+                except self.model.DoesNotExist:
+                    obj = self.model.objects.create(name=val)
+                cleaned_values.append(obj)
+
+        return cleaned_values

app/apps/export_app/widgets/string.py

@@ -0,0 +1,7 @@
+from import_export import fields
+
+
+class EmptyStringToNoneField(fields.Field):
+    def clean(self, data, **kwargs):
+        value = super().clean(data)
+        return None if value == "" else value

app/apps/import_app/admin.py

@@ -0,0 +1,6 @@
+from django.contrib import admin
+from apps.import_app import models
+
+# Register your models here.
+admin.site.register(models.ImportRun)
+admin.site.register(models.ImportProfile)

app/apps/import_app/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class ImportConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "apps.import_app"

app/apps/import_app/forms.py

@@ -0,0 +1,64 @@
+from crispy_forms.bootstrap import FormActions
+from crispy_forms.helper import FormHelper
+from crispy_forms.layout import (
+    Layout,
+)
+from django import forms
+from django.utils.translation import gettext_lazy as _
+
+from apps.import_app.models import ImportProfile
+from apps.common.widgets.crispy.submit import NoClassSubmit
+
+
+class ImportProfileForm(forms.ModelForm):
+    class Meta:
+        model = ImportProfile
+        fields = [
+            "name",
+            "version",
+            "yaml_config",
+        ]
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self.helper = FormHelper()
+        self.helper.form_tag = False
+        self.helper.form_method = "post"
+        self.helper.layout = Layout("name", "version", "yaml_config")
+
+        if self.instance and self.instance.pk:
+            self.helper.layout.append(
+                FormActions(
+                    NoClassSubmit(
+                        "submit", _("Update"), css_class="btn btn-outline-primary w-100"
+                    ),
+                ),
+            )
+        else:
+            self.helper.layout.append(
+                FormActions(
+                    NoClassSubmit(
+                        "submit", _("Add"), css_class="btn btn-outline-primary w-100"
+                    ),
+                ),
+            )
+
+
+class ImportRunFileUploadForm(forms.Form):
+    file = forms.FileField(label=_("Select a file"))
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self.helper = FormHelper()
+        self.helper.form_tag = False
+        self.helper.form_method = "post"
+        self.helper.layout = Layout(
+            "file",
+            FormActions(
+                NoClassSubmit(
+                    "submit", _("Import"), css_class="btn btn-outline-primary w-100"
+                ),
+            ),
+        )

app/apps/import_app/migrations/0001_initial.py

@@ -0,0 +1,51 @@
+# Generated by Django 5.1.5 on 2025-01-19 00:44
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('currencies', '0006_currency_exchange_currency'),
+        ('transactions', '0028_transaction_internal_note'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ImportProfile',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=100)),
+                ('yaml_config', models.TextField(help_text='YAML configuration')),
+                ('version', models.IntegerField(choices=[(1, 'Version 1')], default=1, verbose_name='Version')),
+            ],
+            options={
+                'ordering': ['name'],
+            },
+        ),
+        migrations.CreateModel(
+            name='ImportRun',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('status', models.CharField(choices=[('QUEUED', 'Queued'), ('PROCESSING', 'Processing'), ('FAILED', 'Failed'), ('FINISHED', 'Finished')], default='QUEUED', max_length=10, verbose_name='Status')),
+                ('file_name', models.CharField(help_text='File name', max_length=10000)),
+                ('logs', models.TextField(blank=True)),
+                ('processed_rows', models.IntegerField(default=0)),
+                ('total_rows', models.IntegerField(default=0)),
+                ('successful_rows', models.IntegerField(default=0)),
+                ('skipped_rows', models.IntegerField(default=0)),
+                ('failed_rows', models.IntegerField(default=0)),
+                ('started_at', models.DateTimeField(null=True)),
+                ('finished_at', models.DateTimeField(null=True)),
+                ('categories', models.ManyToManyField(related_name='import_runs', to='transactions.transactioncategory')),
+                ('currencies', models.ManyToManyField(related_name='import_runs', to='currencies.currency')),
+                ('entities', models.ManyToManyField(related_name='import_runs', to='transactions.transactionentity')),
+                ('profile', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='import_app.importprofile')),
+                ('tags', models.ManyToManyField(related_name='import_runs', to='transactions.transactiontag')),
+                ('transactions', models.ManyToManyField(related_name='import_runs', to='transactions.transaction')),
+            ],
+        ),
+    ]

app/apps/import_app/migrations/0002_alter_importprofile_name_and_more.py

@@ -0,0 +1,23 @@
+# Generated by Django 5.1.5 on 2025-01-23 03:03
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('import_app', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='importprofile',
+            name='name',
+            field=models.CharField(max_length=100, unique=True, verbose_name='Name'),
+        ),
+        migrations.AlterField(
+            model_name='importprofile',
+            name='yaml_config',
+            field=models.TextField(verbose_name='YAML Configuration'),
+        ),
+    ]

app/apps/import_app/models.py

@@ -0,0 +1,87 @@
+import yaml
+
+from django.core.exceptions import ValidationError
+from django.db import models
+from django.utils.translation import gettext_lazy as _
+
+from apps.import_app.schemas import version_1
+
+
+class ImportProfile(models.Model):
+    class Versions(models.IntegerChoices):
+        VERSION_1 = 1, "Version 1"
+
+    name = models.CharField(max_length=100, verbose_name=_("Name"), unique=True)
+    yaml_config = models.TextField(verbose_name=_("YAML Configuration"))
+    version = models.IntegerField(
+        choices=Versions,
+        default=Versions.VERSION_1,
+        verbose_name=_("Version"),
+    )
+
+    def __str__(self):
+        return self.name
+
+    class Meta:
+        ordering = ["name"]
+
+    def get_version_display(self):
+        version_number = self.Versions(self.version).name.split("_")[1]
+        return _("Version {number}").format(number=version_number)
+
+    def clean(self):
+        if self.version and self.version == self.Versions.VERSION_1:
+            try:
+                yaml_data = yaml.safe_load(self.yaml_config)
+                version_1.ImportProfileSchema(**yaml_data)
+            except Exception as e:
+                raise ValidationError(
+                    {"yaml_config": _("Invalid YAML Configuration: ") + str(e)}
+                )
+
+
+class ImportRun(models.Model):
+    class Status(models.TextChoices):
+        QUEUED = "QUEUED", _("Queued")
+        PROCESSING = "PROCESSING", _("Processing")
+        FAILED = "FAILED", _("Failed")
+        FINISHED = "FINISHED", _("Finished")
+
+    status = models.CharField(
+        max_length=10,
+        choices=Status,
+        default=Status.QUEUED,
+        verbose_name=_("Status"),
+    )
+    profile = models.ForeignKey(
+        ImportProfile,
+        on_delete=models.CASCADE,
+    )
+    file_name = models.CharField(
+        max_length=10000,
+        help_text=_("File name"),
+    )
+    transactions = models.ManyToManyField(
+        "transactions.Transaction", related_name="import_runs"
+    )
+    tags = models.ManyToManyField(
+        "transactions.TransactionTag", related_name="import_runs"
+    )
+    categories = models.ManyToManyField(
+        "transactions.TransactionCategory", related_name="import_runs"
+    )
+    entities = models.ManyToManyField(
+        "transactions.TransactionEntity", related_name="import_runs"
+    )
+    currencies = models.ManyToManyField(
+        "currencies.Currency", related_name="import_runs"
+    )
+
+    logs = models.TextField(blank=True)
+    processed_rows = models.IntegerField(default=0)
+    total_rows = models.IntegerField(default=0)
+    successful_rows = models.IntegerField(default=0)
+    skipped_rows = models.IntegerField(default=0)
+    failed_rows = models.IntegerField(default=0)
+    started_at = models.DateTimeField(null=True)
+    finished_at = models.DateTimeField(null=True)

app/apps/import_app/schemas/__init__.py

@@ -0,0 +1 @@
+import apps.import_app.schemas.v1 as version_1

app/apps/import_app/schemas/v1.py

@@ -0,0 +1,441 @@
+from typing import Dict, List, Optional, Literal
+from pydantic import BaseModel, Field, model_validator, field_validator
+
+
+class CompareDeduplicationRule(BaseModel):
+    type: Literal["compare"]
+    fields: list[str] = Field(..., description="Compare fields for deduplication")
+    match_type: Literal["lax", "strict"] = "lax"
+
+
+class ReplaceTransformationRule(BaseModel):
+    type: Literal["replace", "regex"] = Field(
+        ..., description="Type of transformation: replace or regex"
+    )
+    pattern: str = Field(..., description="Pattern to match")
+    replacement: str = Field(..., description="Value to replace with")
+    exclusive: bool = Field(
+        default=False,
+        description="If it should match against the last transformation or the original value",
+    )
+
+
+class DateFormatTransformationRule(BaseModel):
+    type: Literal["date_format"] = Field(
+        ..., description="Type of transformation: date_format"
+    )
+    original_format: str = Field(..., description="Original date format")
+    new_format: str = Field(..., description="New date format to use")
+
+
+class HashTransformationRule(BaseModel):
+    fields: List[str]
+    type: Literal["hash"]
+
+
+class MergeTransformationRule(BaseModel):
+    fields: List[str]
+    type: Literal["merge"]
+    separator: str = Field(default=" ", description="Separator to use when merging")
+
+
+class SplitTransformationRule(BaseModel):
+    type: Literal["split"]
+    separator: str = Field(default=",", description="Separator to use when splitting")
+    index: int | None = Field(
+        default=0, description="Index to return as value. Empty to return all."
+    )
+
+
+class AddTransformationRule(BaseModel):
+    type: Literal["add"]
+    field: str = Field(..., description="Field to add to the source value")
+    absolute_values: bool = Field(
+        default=False, description="Use absolute values for addition"
+    )
+    thousand_separator: str = Field(
+        default="", description="Thousand separator character"
+    )
+    decimal_separator: str = Field(
+        default=".", description="Decimal separator character"
+    )
+
+
+class SubtractTransformationRule(BaseModel):
+    type: Literal["subtract"]
+    field: str = Field(..., description="Field to subtract from the source value")
+    absolute_values: bool = Field(
+        default=False, description="Use absolute values for subtraction"
+    )
+    thousand_separator: str = Field(
+        default="", description="Thousand separator character"
+    )
+    decimal_separator: str = Field(
+        default=".", description="Decimal separator character"
+    )
+
+
+class CSVImportSettings(BaseModel):
+    skip_errors: bool = Field(
+        default=False,
+        description="If True, errors during import will be logged and skipped",
+    )
+    file_type: Literal["csv"] = "csv"
+    delimiter: str = Field(default=",", description="CSV delimiter character")
+    encoding: str = Field(default="utf-8", description="File encoding")
+    skip_lines: int = Field(
+        default=0, description="Number of rows to skip at the beginning of the file"
+    )
+    trigger_transaction_rules: bool = True
+    importing: Literal[
+        "transactions", "accounts", "currencies", "categories", "tags", "entities"
+    ]
+
+
+class ExcelImportSettings(BaseModel):
+    skip_errors: bool = Field(
+        default=False,
+        description="If True, errors during import will be logged and skipped",
+    )
+    file_type: Literal["xls", "xlsx"]
+    trigger_transaction_rules: bool = True
+    importing: Literal[
+        "transactions", "accounts", "currencies", "categories", "tags", "entities"
+    ]
+    start_row: int = Field(default=1, description="Where your header is located")
+    sheets: list[str] | str = "*"
+
+
+class ColumnMapping(BaseModel):
+    source: Optional[str] | Optional[list[str]] = Field(
+        default=None,
+        description="CSV column header. If None, the field will be generated from transformations",
+    )
+    default: Optional[str] = None
+    required: bool = False
+    transformations: Optional[
+        List[
+            ReplaceTransformationRule
+            | DateFormatTransformationRule
+            | HashTransformationRule
+            | MergeTransformationRule
+            | SplitTransformationRule
+            | AddTransformationRule
+            | SubtractTransformationRule
+        ]
+    ] = Field(default_factory=list)
+
+
+class TransactionAccountMapping(ColumnMapping):
+    target: Literal["account"] = Field(..., description="Transaction field to map to")
+    type: Literal["id", "name"] = "name"
+    coerce_to: Literal["str|int"] = Field("str|int", frozen=True)
+
+
+class TransactionTypeMapping(ColumnMapping):
+    target: Literal["type"] = Field(..., description="Transaction field to map to")
+    detection_method: Literal["sign", "always_income", "always_expense"] = "sign"
+    coerce_to: Literal["transaction_type"] = Field("transaction_type", frozen=True)
+
+
+class TransactionIsPaidMapping(ColumnMapping):
+    target: Literal["is_paid"] = Field(..., description="Transaction field to map to")
+    detection_method: Literal["boolean", "always_paid", "always_unpaid"]
+    coerce_to: Literal["is_paid"] = Field("is_paid", frozen=True)
+
+
+class TransactionDateMapping(ColumnMapping):
+    target: Literal["date"] = Field(..., description="Transaction field to map to")
+    format: List[str] | str
+    coerce_to: Literal["date"] = Field("date", frozen=True)
+
+
+class TransactionReferenceDateMapping(ColumnMapping):
+    target: Literal["reference_date"] = Field(
+        ..., description="Transaction field to map to"
+    )
+    format: List[str] | str
+    coerce_to: Literal["date"] = Field("date", frozen=True)
+
+
+class TransactionAmountMapping(ColumnMapping):
+    target: Literal["amount"] = Field(..., description="Transaction field to map to")
+    coerce_to: Literal["positive_decimal"] = Field("positive_decimal", frozen=True)
+
+
+class TransactionDescriptionMapping(ColumnMapping):
+    target: Literal["description"] = Field(
+        ..., description="Transaction field to map to"
+    )
+    coerce_to: Literal["str"] = Field("str", frozen=True)
+
+
+class TransactionNotesMapping(ColumnMapping):
+    target: Literal["notes"] = Field(..., description="Transaction field to map to")
+    coerce_to: Literal["str"] = Field("str", frozen=True)
+
+
+class TransactionTagsMapping(ColumnMapping):
+    target: Literal["tags"] = Field(..., description="Transaction field to map to")
+    type: Literal["id", "name"] = "name"
+    create: bool = Field(
+        default=True, description="Create new tags if they doesn't exist"
+    )
+    coerce_to: Literal["list"] = Field("list", frozen=True)
+
+
+class TransactionEntitiesMapping(ColumnMapping):
+    target: Literal["entities"] = Field(..., description="Transaction field to map to")
+    type: Literal["id", "name"] = "name"
+    create: bool = Field(
+        default=True, description="Create new entities if they doesn't exist"
+    )
+    coerce_to: Literal["list"] = Field("list", frozen=True)
+
+
+class TransactionCategoryMapping(ColumnMapping):
+    target: Literal["category"] = Field(..., description="Transaction field to map to")
+    create: bool = Field(
+        default=True, description="Create category if it doesn't exist"
+    )
+    type: Literal["id", "name"] = "name"
+    coerce_to: Literal["str|int"] = Field("str|int", frozen=True)
+
+
+class TransactionInternalNoteMapping(ColumnMapping):
+    target: Literal["internal_note"] = Field(
+        ..., description="Transaction field to map to"
+    )
+    coerce_to: Literal["str"] = Field("str", frozen=True)
+
+
+class TransactionInternalIDMapping(ColumnMapping):
+    target: Literal["internal_id"] = Field(
+        ..., description="Transaction field to map to"
+    )
+    coerce_to: Literal["str"] = Field("str", frozen=True)
+
+
+class CategoryNameMapping(ColumnMapping):
+    target: Literal["category_name"] = Field(
+        ..., description="Category field to map to"
+    )
+    coerce_to: Literal["str"] = Field("str", frozen=True)
+
+
+class CategoryMuteMapping(ColumnMapping):
+    target: Literal["category_mute"] = Field(
+        ..., description="Category field to map to"
+    )
+    coerce_to: Literal["bool"] = Field("bool", frozen=True)
+
+
+class CategoryActiveMapping(ColumnMapping):
+    target: Literal["category_active"] = Field(
+        ..., description="Category field to map to"
+    )
+    coerce_to: Literal["bool"] = Field("bool", frozen=True)
+
+
+class TagNameMapping(ColumnMapping):
+    target: Literal["tag_name"] = Field(..., description="Tag field to map to")
+    coerce_to: Literal["str"] = Field("str", frozen=True)
+
+
+class TagActiveMapping(ColumnMapping):
+    target: Literal["tag_active"] = Field(..., description="Tag field to map to")
+    coerce_to: Literal["bool"] = Field("bool", frozen=True)
+
+
+class EntityNameMapping(ColumnMapping):
+    target: Literal["entity_name"] = Field(..., description="Entity field to map to")
+    coerce_to: Literal["str"] = Field("str", frozen=True)
+
+
+class EntityActiveMapping(ColumnMapping):
+    target: Literal["entity_active"] = Field(..., description="Entity field to map to")
+    coerce_to: Literal["bool"] = Field("bool", frozen=True)
+
+
+class AccountNameMapping(ColumnMapping):
+    target: Literal["account_name"] = Field(..., description="Account field to map to")
+    coerce_to: Literal["str"] = Field("str", frozen=True)
+
+
+class AccountGroupMapping(ColumnMapping):
+    target: Literal["account_group"] = Field(..., description="Account field to map to")
+    type: Literal["id", "name"]
+    coerce_to: Literal["str|int"] = Field("str|int", frozen=True)
+
+
+class AccountCurrencyMapping(ColumnMapping):
+    target: Literal["account_currency"] = Field(
+        ..., description="Account field to map to"
+    )
+    type: Literal["id", "name", "code"]
+    coerce_to: Literal["str|int"] = Field("str|int", frozen=True)
+
+
+class AccountExchangeCurrencyMapping(ColumnMapping):
+    target: Literal["account_exchange_currency"] = Field(
+        ..., description="Account field to map to"
+    )
+    type: Literal["id", "name", "code"]
+    coerce_to: Literal["str|int"] = Field("str|int", frozen=True)
+
+
+class AccountIsAssetMapping(ColumnMapping):
+    target: Literal["account_is_asset"] = Field(
+        ..., description="Account field to map to"
+    )
+    coerce_to: Literal["bool"] = Field("bool", frozen=True)
+
+
+class AccountIsArchivedMapping(ColumnMapping):
+    target: Literal["account_is_archived"] = Field(
+        ..., description="Account field to map to"
+    )
+    coerce_to: Literal["bool"] = Field("bool", frozen=True)
+
+
+class CurrencyCodeMapping(ColumnMapping):
+    target: Literal["currency_code"] = Field(
+        ..., description="Currency field to map to"
+    )
+    coerce_to: Literal["str"] = Field("str", frozen=True)
+
+
+class CurrencyNameMapping(ColumnMapping):
+    target: Literal["currency_name"] = Field(
+        ..., description="Currency field to map to"
+    )
+    coerce_to: Literal["str"] = Field("str", frozen=True)
+
+
+class CurrencyDecimalPlacesMapping(ColumnMapping):
+    target: Literal["currency_decimal_places"] = Field(
+        ..., description="Currency field to map to"
+    )
+    coerce_to: Literal["int"] = Field("int", frozen=True)
+
+
+class CurrencyPrefixMapping(ColumnMapping):
+    target: Literal["currency_prefix"] = Field(
+        ..., description="Currency field to map to"
+    )
+    coerce_to: Literal["str"] = Field("str", frozen=True)
+
+
+class CurrencySuffixMapping(ColumnMapping):
+    target: Literal["currency_suffix"] = Field(
+        ..., description="Currency field to map to"
+    )
+    coerce_to: Literal["str"] = Field("str", frozen=True)
+
+
+class CurrencyExchangeMapping(ColumnMapping):
+    target: Literal["currency_exchange"] = Field(
+        ..., description="Currency field to map to"
+    )
+    type: Literal["id", "name", "code"]
+    coerce_to: Literal["str|int"] = Field("str|int", frozen=True)
+
+
+class ImportProfileSchema(BaseModel):
+    settings: CSVImportSettings | ExcelImportSettings
+    mapping: Dict[
+        str,
+        TransactionAccountMapping
+        | TransactionTypeMapping
+        | TransactionIsPaidMapping
+        | TransactionDateMapping
+        | TransactionReferenceDateMapping
+        | TransactionAmountMapping
+        | TransactionDescriptionMapping
+        | TransactionNotesMapping
+        | TransactionTagsMapping
+        | TransactionEntitiesMapping
+        | TransactionCategoryMapping
+        | TransactionInternalNoteMapping
+        | TransactionInternalIDMapping
+        | CategoryNameMapping
+        | CategoryMuteMapping
+        | CategoryActiveMapping
+        | TagNameMapping
+        | TagActiveMapping
+        | EntityNameMapping
+        | EntityActiveMapping
+        | AccountNameMapping
+        | AccountGroupMapping
+        | AccountCurrencyMapping
+        | AccountExchangeCurrencyMapping
+        | AccountIsAssetMapping
+        | AccountIsArchivedMapping
+        | CurrencyCodeMapping
+        | CurrencyNameMapping
+        | CurrencyDecimalPlacesMapping
+        | CurrencyPrefixMapping
+        | CurrencySuffixMapping
+        | CurrencyExchangeMapping,
+    ]
+    deduplication: List[CompareDeduplicationRule] = Field(
+        default_factory=list,
+        description="Rules for deduplicating records during import",
+    )
+
+    @model_validator(mode="after")
+    def validate_mappings(self) -> "ImportProfileSchema":
+        import_type = self.settings.importing
+
+        # Define allowed mapping types for each import type
+        allowed_mappings = {
+            "transactions": (
+                TransactionAccountMapping,
+                TransactionTypeMapping,
+                TransactionIsPaidMapping,
+                TransactionDateMapping,
+                TransactionReferenceDateMapping,
+                TransactionAmountMapping,
+                TransactionDescriptionMapping,
+                TransactionNotesMapping,
+                TransactionTagsMapping,
+                TransactionEntitiesMapping,
+                TransactionCategoryMapping,
+                TransactionInternalNoteMapping,
+                TransactionInternalIDMapping,
+            ),
+            "accounts": (
+                AccountNameMapping,
+                AccountGroupMapping,
+                AccountCurrencyMapping,
+                AccountExchangeCurrencyMapping,
+                AccountIsAssetMapping,
+                AccountIsArchivedMapping,
+            ),
+            "currencies": (
+                CurrencyCodeMapping,
+                CurrencyNameMapping,
+                CurrencyDecimalPlacesMapping,
+                CurrencyPrefixMapping,
+                CurrencySuffixMapping,
+                CurrencyExchangeMapping,
+            ),
+            "categories": (
+                CategoryNameMapping,
+                CategoryMuteMapping,
+                CategoryActiveMapping,
+            ),
+            "tags": (TagNameMapping, TagActiveMapping),
+            "entities": (EntityNameMapping, EntityActiveMapping),
+        }
+
+        allowed_types = allowed_mappings[import_type]
+
+        for field_name, mapping in self.mapping.items():
+            if not isinstance(mapping, allowed_types):
+                raise ValueError(
+                    f"Mapping type '{type(mapping).__name__}' is not allowed when importing {import_type}. "
+                    f"Allowed types are: {', '.join(t.__name__ for t in allowed_types)}"
+                )
+
+        return self

app/apps/import_app/services/__init__.py

@@ -0,0 +1,3 @@
+from apps.import_app.services.v1 import ImportService as ImportServiceV1
+
+from apps.import_app.services.presets import PresetService

app/apps/import_app/services/presets.py

@@ -0,0 +1,45 @@
+import json
+from pathlib import Path
+
+from apps.import_app.models import ImportProfile
+
+
+class PresetService:
+    PRESET_PATH = "/usr/src/app/import_presets"
+
+    @classmethod
+    def get_all_presets(cls):
+        presets = []
+
+        for folder in Path(cls.PRESET_PATH).iterdir():
+            if folder.is_dir():
+                manifest_path = folder / "manifest.json"
+                config_path = folder / "config.yml"
+
+                if manifest_path.exists() and config_path.exists():
+                    with open(manifest_path) as f:
+                        manifest = json.load(f)
+
+                    with open(config_path) as f:
+                        config = json.dumps(f.read())
+
+                    try:
+                        preset = {
+                            "name": manifest.get("name", folder.name),
+                            "description": manifest.get("description", ""),
+                            "message": json.dumps(manifest.get("message", "")),
+                            "authors": manifest.get("author", "").split(","),
+                            "schema_version": (int(manifest.get("schema_version", 1))),
+                            "folder_name": folder.name,
+                            "config": config,
+                        }
+
+                        ImportProfile.Versions(
+                            preset["schema_version"]
+                        )  # Check if schema version is valid
+                    except Exception as e:
+                        pass
+                    else:
+                        presets.append(preset)
+
+        return presets

app/apps/import_app/services/v1.py

@@ -0,0 +1,867 @@
+import csv
+import hashlib
+import logging
+import os
+import re
+from datetime import datetime, date
+from decimal import Decimal, InvalidOperation
+from typing import Dict, Any, Literal, Union
+
+import openpyxl
+import xlrd
+import yaml
+from cachalot.api import cachalot_disabled
+from django.utils import timezone
+from openpyxl.utils.exceptions import InvalidFileException
+
+from apps.accounts.models import Account, AccountGroup
+from apps.currencies.models import Currency
+from apps.import_app.models import ImportRun, ImportProfile
+from apps.import_app.schemas import version_1
+from apps.transactions.models import (
+    Transaction,
+    TransactionCategory,
+    TransactionTag,
+    TransactionEntity,
+)
+from apps.rules.signals import transaction_created
+from apps.import_app.schemas.v1 import (
+    TransactionCategoryMapping,
+    TransactionAccountMapping,
+    TransactionTagsMapping,
+    TransactionEntitiesMapping,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class ImportService:
+    TEMP_DIR = "/usr/src/app/temp"
+
+    def __init__(self, import_run: ImportRun):
+        self.import_run: ImportRun = import_run
+        self.profile: ImportProfile = import_run.profile
+        self.config: version_1.ImportProfileSchema = self._load_config()
+        self.settings: version_1.CSVImportSettings | version_1.ExcelImportSettings = (
+            self.config.settings
+        )
+        self.deduplication: list[version_1.CompareDeduplicationRule] = (
+            self.config.deduplication
+        )
+        self.mapping: Dict[str, version_1.ColumnMapping] = self.config.mapping
+
+        # Ensure temp directory exists
+        os.makedirs(self.TEMP_DIR, exist_ok=True)
+
+    def _load_config(self) -> version_1.ImportProfileSchema:
+        yaml_data = yaml.safe_load(self.profile.yaml_config)
+        try:
+            config = version_1.ImportProfileSchema(**yaml_data)
+        except Exception as e:
+            self._log("error", f"Fatal error processing YAML config: {str(e)}")
+            self._update_status("FAILED")
+            raise e
+        else:
+            return config
+
+    def _log(self, level: str, message: str, **kwargs) -> None:
+        """Add a log entry to the import run logs"""
+        timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+        # Format additional context if present
+        context = ""
+        if kwargs:
+            context = " - " + ", ".join(f"{k}={v}" for k, v in kwargs.items())
+
+        log_line = f"[{timestamp}] {level.upper()}: {message}{context}\n"
+
+        # Append to existing logs
+        self.import_run.logs += log_line
+        self.import_run.save(update_fields=["logs"])
+
+        if level == "info":
+            logger.info(log_line)
+        elif level == "warning":
+            logger.warning(log_line)
+        elif level == "error":
+            logger.error(log_line, exc_info=True)
+
+    def _update_totals(
+        self,
+        field: Literal["total", "processed", "successful", "skipped", "failed"],
+        value: int,
+    ) -> None:
+        if field == "total":
+            self.import_run.total_rows = value
+            self.import_run.save(update_fields=["total_rows"])
+        elif field == "processed":
+            self.import_run.processed_rows = value
+            self.import_run.save(update_fields=["processed_rows"])
+        elif field == "successful":
+            self.import_run.successful_rows = value
+            self.import_run.save(update_fields=["successful_rows"])
+        elif field == "skipped":
+            self.import_run.skipped_rows = value
+            self.import_run.save(update_fields=["skipped_rows"])
+        elif field == "failed":
+            self.import_run.failed_rows = value
+            self.import_run.save(update_fields=["failed_rows"])
+
+    def _increment_totals(
+        self,
+        field: Literal["total", "processed", "successful", "skipped", "failed"],
+        value: int,
+    ) -> None:
+        if field == "total":
+            self.import_run.total_rows = self.import_run.total_rows + value
+            self.import_run.save(update_fields=["total_rows"])
+        elif field == "processed":
+            self.import_run.processed_rows = self.import_run.processed_rows + value
+            self.import_run.save(update_fields=["processed_rows"])
+        elif field == "successful":
+            self.import_run.successful_rows = self.import_run.successful_rows + value
+            self.import_run.save(update_fields=["successful_rows"])
+        elif field == "skipped":
+            self.import_run.skipped_rows = self.import_run.skipped_rows + value
+            self.import_run.save(update_fields=["skipped_rows"])
+        elif field == "failed":
+            self.import_run.failed_rows = self.import_run.failed_rows + value
+            self.import_run.save(update_fields=["failed_rows"])
+
+    def _update_status(
+        self, new_status: Literal["PROCESSING", "FAILED", "FINISHED"]
+    ) -> None:
+        if new_status == "PROCESSING":
+            self.import_run.status = ImportRun.Status.PROCESSING
+        elif new_status == "FAILED":
+            self.import_run.status = ImportRun.Status.FAILED
+        elif new_status == "FINISHED":
+            self.import_run.status = ImportRun.Status.FINISHED
+
+        self.import_run.save(update_fields=["status"])
+
+    def _transform_value(
+        self,
+        value: str,
+        mapping: version_1.ColumnMapping,
+        row: Dict[str, str] = None,
+        mapped_data: Dict[str, Any] = None,
+    ) -> Any:
+        transformed = value
+
+        for transform in mapping.transformations:
+            if transform.type == "hash":
+                # Collect all values to be hashed
+                values_to_hash = []
+                for field in transform.fields:
+                    if field in row:
+                        values_to_hash.append(str(row[field]))
+                    elif (
+                        field.startswith("__")
+                        and mapped_data
+                        and field[2:] in mapped_data
+                    ):
+                        values_to_hash.append(str(mapped_data[field[2:]]))
+                if values_to_hash:
+                    concatenated = "|".join(values_to_hash)
+                    transformed = hashlib.sha256(concatenated.encode()).hexdigest()
+
+            elif transform.type == "replace":
+                if transform.exclusive:
+                    transformed = value.replace(
+                        transform.pattern, transform.replacement
+                    )
+                else:
+                    transformed = transformed.replace(
+                        transform.pattern, transform.replacement
+                    )
+
+            elif transform.type == "regex":
+                if transform.exclusive:
+                    transformed = re.sub(
+                        transform.pattern, transform.replacement, value
+                    )
+                else:
+                    transformed = re.sub(
+                        transform.pattern, transform.replacement, transformed
+                    )
+
+            elif transform.type == "date_format":
+                transformed = datetime.strptime(
+                    transformed, transform.original_format
+                ).strftime(transform.new_format)
+
+            elif transform.type == "merge":
+                values_to_merge = []
+                for field in transform.fields:
+                    if field in row:
+                        values_to_merge.append(str(row[field]))
+                    elif (
+                        field.startswith("__")
+                        and mapped_data
+                        and field[2:] in mapped_data
+                    ):
+                        values_to_merge.append(str(mapped_data[field[2:]]))
+                transformed = transform.separator.join(values_to_merge)
+
+            elif transform.type == "split":
+                parts = transformed.split(transform.separator)
+                if transform.index is not None:
+                    transformed = parts[transform.index] if parts else ""
+                else:
+                    transformed = parts
+
+            elif transform.type in ["add", "subtract"]:
+                try:
+                    source_value = Decimal(transformed)
+
+                    # First check row data, then mapped data if not found
+                    field_value = row.get(transform.field)
+                    if field_value is None and transform.field.startswith("__"):
+                        field_value = mapped_data.get(transform.field[2:])
+
+                    if field_value is None:
+                        raise KeyError(
+                            f"Field '{transform.field}' not found in row or mapped data"
+                        )
+
+                    field_value = self._prepare_numeric_value(
+                        str(field_value),
+                        transform.thousand_separator,
+                        transform.decimal_separator,
+                    )
+
+                    if transform.absolute_values:
+                        source_value = abs(source_value)
+                        field_value = abs(field_value)
+
+                    if transform.type == "add":
+                        transformed = str(source_value + field_value)
+                    else:  # subtract
+                        transformed = str(source_value - field_value)
+                except (InvalidOperation, KeyError, AttributeError) as e:
+                    logger.warning(
+                        f"Error in {transform.type} transformation: {e}. Values: {transformed}, {transform.field}"
+                    )
+        return transformed
+
+    def _create_transaction(self, data: Dict[str, Any]) -> Transaction:
+        tags = []
+        entities = []
+        # Handle related objects first
+        if "category" in data:
+            if "category" in data:
+                category_name = data.pop("category")
+                category_mapping = next(
+                    (
+                        m
+                        for m in self.mapping.values()
+                        if isinstance(m, TransactionCategoryMapping)
+                        and m.target == "category"
+                    ),
+                    None,
+                )
+
+                try:
+                    if category_mapping:
+                        if category_mapping.type == "id":
+                            category = TransactionCategory.objects.get(id=category_name)
+                        else:  # name
+                            if getattr(category_mapping, "create", False):
+                                category, _ = TransactionCategory.objects.get_or_create(
+                                    name=category_name
+                                )
+                            else:
+                                category = TransactionCategory.objects.filter(
+                                    name=category_name
+                                ).first()
+
+                        if category:
+                            data["category"] = category
+                            self.import_run.categories.add(category)
+                except (TransactionCategory.DoesNotExist, ValueError):
+                    # Ignore if category doesn't exist and create is False or not set
+                    data["category"] = None
+
+        if "account" in data:
+            account_id = data.pop("account")
+            account_mapping = next(
+                (
+                    m
+                    for m in self.mapping.values()
+                    if isinstance(m, TransactionAccountMapping)
+                    and m.target == "account"
+                ),
+                None,
+            )
+
+            try:
+                if account_mapping and account_mapping.type == "id":
+                    account = Account.objects.filter(id=account_id).first()
+                else:  # name
+                    account = Account.objects.filter(name=account_id).first()
+
+                if account:
+                    data["account"] = account
+            except ValueError:
+                # Ignore if account doesn't exist
+                pass
+
+        if "tags" in data:
+            tag_names = data.pop("tags")
+            tags_mapping = next(
+                (
+                    m
+                    for m in self.mapping.values()
+                    if isinstance(m, TransactionTagsMapping) and m.target == "tags"
+                ),
+                None,
+            )
+
+            for tag_name in tag_names:
+                try:
+                    if tags_mapping:
+                        if tags_mapping.type == "id":
+                            tag = TransactionTag.objects.filter(id=tag_name).first()
+                        else:  # name
+                            if getattr(tags_mapping, "create", False):
+                                tag, _ = TransactionTag.objects.get_or_create(
+                                    name=tag_name.strip()
+                                )
+                            else:
+                                tag = TransactionTag.objects.filter(
+                                    name=tag_name.strip()
+                                ).first()
+
+                        if tag:
+                            tags.append(tag)
+                            self.import_run.tags.add(tag)
+                except ValueError:
+                    # Ignore if tag doesn't exist and create is False or not set
+                    continue
+
+        if "entities" in data:
+            entity_names = data.pop("entities")
+            entities_mapping = next(
+                (
+                    m
+                    for m in self.mapping.values()
+                    if isinstance(m, TransactionEntitiesMapping)
+                    and m.target == "entities"
+                ),
+                None,
+            )
+
+            for entity_name in entity_names:
+                try:
+                    if entities_mapping:
+                        if entities_mapping.type == "id":
+                            entity = TransactionTag.objects.filter(
+                                id=entity_name
+                            ).first()
+                        else:  # name
+                            if getattr(entities_mapping, "create", False):
+                                entity, _ = TransactionEntity.objects.get_or_create(
+                                    name=entity_name.strip()
+                                )
+                            else:
+                                entity = TransactionEntity.objects.filter(
+                                    name=entity_name.strip()
+                                ).first()
+
+                        if entity:
+                            entities.append(entity)
+                            self.import_run.entities.add(entity)
+                except ValueError:
+                    # Ignore if entity doesn't exist and create is False or not set
+                    continue
+
+        # Create the transaction
+        new_transaction = Transaction.objects.create(**data)
+        self.import_run.transactions.add(new_transaction)
+
+        # Add many-to-many relationships
+        if tags:
+            new_transaction.tags.set(tags)
+        if entities:
+            new_transaction.entities.set(entities)
+
+        if self.settings.trigger_transaction_rules:
+            transaction_created.send(sender=new_transaction)
+
+        return new_transaction
+
+    def _create_account(self, data: Dict[str, Any]) -> Account:
+        if "group" in data:
+            group_name = data.pop("group")
+            group, _ = AccountGroup.objects.get_or_create(name=group_name)
+            data["group"] = group
+
+        # Handle currency references
+        if "currency" in data:
+            currency = Currency.objects.get(code=data["currency"])
+            data["currency"] = currency
+            self.import_run.currencies.add(currency)
+
+        if "exchange_currency" in data:
+            exchange_currency = Currency.objects.get(code=data["exchange_currency"])
+            data["exchange_currency"] = exchange_currency
+            self.import_run.currencies.add(exchange_currency)
+
+        return Account.objects.create(**data)
+
+    def _create_currency(self, data: Dict[str, Any]) -> Currency:
+        # Handle exchange currency reference
+        if "exchange_currency" in data:
+            exchange_currency = Currency.objects.get(code=data["exchange_currency"])
+            data["exchange_currency"] = exchange_currency
+            self.import_run.currencies.add(exchange_currency)
+
+        currency = Currency.objects.create(**data)
+        self.import_run.currencies.add(currency)
+        return currency
+
+    def _create_category(self, data: Dict[str, Any]) -> TransactionCategory:
+        category = TransactionCategory.objects.create(**data)
+        self.import_run.categories.add(category)
+        return category
+
+    def _create_tag(self, data: Dict[str, Any]) -> TransactionTag:
+        tag = TransactionTag.objects.create(**data)
+        self.import_run.tags.add(tag)
+        return tag
+
+    def _create_entity(self, data: Dict[str, Any]) -> TransactionEntity:
+        entity = TransactionEntity.objects.create(**data)
+        self.import_run.entities.add(entity)
+        return entity
+
+    def _check_duplicate_transaction(self, transaction_data: Dict[str, Any]) -> bool:
+        for rule in self.deduplication:
+            if rule.type == "compare":
+                query = Transaction.all_objects.all().values("id")
+
+                # Build query conditions for each field in the rule
+                for field in rule.fields:
+                    if field in transaction_data:
+                        if rule.match_type == "strict":
+                            query = query.filter(**{field: transaction_data[field]})
+                        else:  # lax matching
+                            query = query.filter(
+                                **{f"{field}__iexact": transaction_data[field]}
+                            )
+
+                # If we found any matching transaction, it's a duplicate
+                if query.exists():
+                    return True
+
+        return False
+
+    def _coerce_type(
+        self, value: str, mapping: version_1.ColumnMapping
+    ) -> Union[str, int, bool, Decimal, datetime, list, None]:
+        if not value:
+            return None
+
+        coerce_to = mapping.coerce_to
+
+        return self._coerce_single_type(value, coerce_to, mapping)
+
+    @staticmethod
+    def _coerce_single_type(
+        value: str, coerce_to: str, mapping: version_1.ColumnMapping
+    ) -> Union[str, int, bool, Decimal, datetime.date, list]:
+        if coerce_to == "str":
+            return str(value)
+        elif coerce_to == "int":
+            return int(value)
+        elif coerce_to == "str|int":
+            if hasattr(mapping, "type") and mapping.type == "id":
+                return int(value)
+            elif hasattr(mapping, "type") and mapping.type in ["name", "code"]:
+                return str(value)
+            else:
+                return str(value)
+        elif coerce_to == "bool":
+            return value.lower() in ["true", "1", "yes", "y", "on"]
+        elif coerce_to == "positive_decimal":
+            return abs(Decimal(value))
+        elif coerce_to == "date":
+            if isinstance(
+                mapping,
+                (
+                    version_1.TransactionDateMapping,
+                    version_1.TransactionReferenceDateMapping,
+                ),
+            ):
+                if isinstance(value, datetime):
+                    return value.date()
+                elif isinstance(value, date):
+                    return value
+
+                formats = (
+                    mapping.format
+                    if isinstance(mapping.format, list)
+                    else [mapping.format]
+                )
+                for fmt in formats:
+                    try:
+                        return datetime.strptime(value, fmt).date()
+                    except ValueError:
+                        continue
+                raise ValueError(
+                    f"Could not parse date '{value}' with any of the provided formats"
+                )
+            else:
+                raise ValueError(
+                    "Date coercion is only supported for TransactionDateMapping and TransactionReferenceDateMapping"
+                )
+        elif coerce_to == "list":
+            return (
+                value
+                if isinstance(value, list)
+                else [item.strip() for item in value.split(",") if item.strip()]
+            )
+        elif coerce_to == "transaction_type":
+            if isinstance(mapping, version_1.TransactionTypeMapping):
+                if mapping.detection_method == "sign":
+                    return (
+                        Transaction.Type.EXPENSE
+                        if value.startswith("-")
+                        else Transaction.Type.INCOME
+                    )
+                elif mapping.detection_method == "always_income":
+                    return Transaction.Type.INCOME
+                elif mapping.detection_method == "always_expense":
+                    return Transaction.Type.EXPENSE
+            raise ValueError("Invalid transaction type detection method")
+        elif coerce_to == "is_paid":
+            if isinstance(mapping, version_1.TransactionIsPaidMapping):
+                if mapping.detection_method == "boolean":
+                    return value.lower() in ["true", "1", "yes", "y", "on"]
+                elif mapping.detection_method == "always_paid":
+                    return True
+                elif mapping.detection_method == "always_unpaid":
+                    return False
+            raise ValueError("Invalid is_paid detection method")
+        else:
+            raise ValueError(f"Unsupported coercion type: {coerce_to}")
+
+    def _map_row(self, row: Dict[str, str]) -> Dict[str, Any]:
+        mapped_data = {}
+        for field, mapping in self.mapping.items():
+            value = None
+            if isinstance(mapping.source, str):
+                if mapping.source in row:
+                    value = row[mapping.source]
+                elif (
+                    mapping.source.startswith("__")
+                    and mapping.source[2:] in mapped_data
+                ):
+                    value = mapped_data[mapping.source[2:]]
+            elif isinstance(mapping.source, list):
+                for source in mapping.source:
+                    if source in row:
+                        value = row[source]
+                        break
+                    elif source.startswith("__") and source[2:] in mapped_data:
+                        value = mapped_data[source[2:]]
+                        break
+
+            if value is None:
+                value = mapping.default
+
+            if mapping.transformations:
+                value = self._transform_value(value, mapping, row, mapped_data)
+
+            value = self._coerce_type(value, mapping)
+
+            if mapping.required and value is None:
+                raise ValueError(f"Required field {field} is missing")
+
+            if value is not None:
+                target = mapping.target
+                if self.settings.importing == "transactions":
+                    mapped_data[target] = value
+                else:
+                    field_name = target.split("_", 1)[1]
+                    mapped_data[field_name] = value
+
+        return mapped_data
+
+    @staticmethod
+    def _prepare_numeric_value(
+        value: str, thousand_separator: str, decimal_separator: str
+    ) -> Decimal:
+        # Remove thousand separators
+        if thousand_separator:
+            value = value.replace(thousand_separator, "")
+
+        # Replace decimal separator with dot
+        if decimal_separator != ".":
+            value = value.replace(decimal_separator, ".")
+
+        return Decimal(value)
+
+    def _process_row(self, row: Dict[str, str], row_number: int) -> None:
+        try:
+            mapped_data = self._map_row(row)
+
+            if mapped_data:
+                # Handle different import types
+                if self.settings.importing == "transactions":
+                    if self.deduplication and self._check_duplicate_transaction(
+                        mapped_data
+                    ):
+                        self._increment_totals("skipped", 1)
+                        self._log("info", f"Skipped duplicate row {row_number}")
+                        return
+                    self._create_transaction(mapped_data)
+                elif self.settings.importing == "accounts":
+                    self._create_account(mapped_data)
+                elif self.settings.importing == "currencies":
+                    self._create_currency(mapped_data)
+                elif self.settings.importing == "categories":
+                    self._create_category(mapped_data)
+                elif self.settings.importing == "tags":
+                    self._create_tag(mapped_data)
+                elif self.settings.importing == "entities":
+                    self._create_entity(mapped_data)
+
+                self._increment_totals("successful", value=1)
+                self._log("info", f"Successfully processed row {row_number}")
+
+            self._increment_totals("processed", value=1)
+
+        except Exception as e:
+            if not self.settings.skip_errors:
+                self._log("error", f"Fatal error processing row {row_number}: {str(e)}")
+                self._update_status("FAILED")
+                raise
+            else:
+                self._log("warning", f"Error processing row {row_number}: {str(e)}")
+                self._increment_totals("failed", value=1)
+
+            logger.error(f"Fatal error processing row {row_number}", exc_info=e)
+
+    def _process_csv(self, file_path):
+        # First pass: count rows
+        with open(file_path, "r", encoding=self.settings.encoding) as csv_file:
+            # Skip specified number of rows
+            for _ in range(self.settings.skip_lines):
+                next(csv_file)
+
+            reader = csv.DictReader(csv_file, delimiter=self.settings.delimiter)
+            self._update_totals("total", value=sum(1 for _ in reader))
+
+        with open(file_path, "r", encoding=self.settings.encoding) as csv_file:
+            # Skip specified number of rows
+            for _ in range(self.settings.skip_lines):
+                next(csv_file)
+            if self.settings.skip_lines:
+                self._log("info", f"Skipped {self.settings.skip_lines} initial lines")
+
+            reader = csv.DictReader(csv_file, delimiter=self.settings.delimiter)
+
+            self._log("info", f"Starting import with {self.import_run.total_rows} rows")
+
+            for row_number, row in enumerate(reader, start=1):
+                self._process_row(row, row_number)
+
+    def _process_excel(self, file_path):
+        try:
+            if self.settings.file_type == "xlsx":
+                workbook = openpyxl.load_workbook(
+                    file_path, read_only=True, data_only=True
+                )
+                sheets_to_process = (
+                    workbook.sheetnames
+                    if self.settings.sheets == "*"
+                    else (
+                        self.settings.sheets
+                        if isinstance(self.settings.sheets, list)
+                        else [self.settings.sheets]
+                    )
+                )
+
+                # Calculate total rows
+                total_rows = sum(
+                    max(0, workbook[sheet_name].max_row - self.settings.start_row)
+                    for sheet_name in sheets_to_process
+                    if sheet_name in workbook.sheetnames
+                )
+                self._update_totals("total", value=total_rows)
+
+                # Process sheets
+                for sheet_name in sheets_to_process:
+                    if sheet_name not in workbook.sheetnames:
+                        self._log(
+                            "warning",
+                            f"Sheet '{sheet_name}' not found in the Excel file. Skipping.",
+                        )
+                        continue
+
+                    sheet = workbook[sheet_name]
+                    self._log("info", f"Processing sheet: {sheet_name}")
+                    headers = [
+                        str(cell.value or "") for cell in sheet[self.settings.start_row]
+                    ]
+
+                    for row_number, row in enumerate(
+                        sheet.iter_rows(
+                            min_row=self.settings.start_row + 1, values_only=True
+                        ),
+                        start=1,
+                    ):
+                        try:
+                            row_data = {
+                                key: str(value) if value is not None else None
+                                for key, value in zip(headers, row)
+                            }
+                            self._process_row(row_data, row_number)
+                        except Exception as e:
+                            if self.settings.skip_errors:
+                                self._log(
+                                    "warning",
+                                    f"Error processing row {row_number} in sheet '{sheet_name}': {str(e)}",
+                                )
+                                self._increment_totals("failed", value=1)
+                            else:
+                                raise
+
+                workbook.close()
+
+            else:  # xls
+                workbook = xlrd.open_workbook(file_path)
+                sheets_to_process = (
+                    workbook.sheet_names()
+                    if self.settings.sheets == "*"
+                    else (
+                        self.settings.sheets
+                        if isinstance(self.settings.sheets, list)
+                        else [self.settings.sheets]
+                    )
+                )
+                # Calculate total rows
+                total_rows = sum(
+                    max(
+                        0,
+                        workbook.sheet_by_name(sheet_name).nrows
+                        - self.settings.start_row,
+                    )
+                    for sheet_name in sheets_to_process
+                    if sheet_name in workbook.sheet_names()
+                )
+                self._update_totals("total", value=total_rows)
+                # Process sheets
+                for sheet_name in sheets_to_process:
+                    if sheet_name not in workbook.sheet_names():
+                        self._log(
+                            "warning",
+                            f"Sheet '{sheet_name}' not found in the Excel file. Skipping.",
+                        )
+                        continue
+                    sheet = workbook.sheet_by_name(sheet_name)
+                    self._log("info", f"Processing sheet: {sheet_name}")
+                    headers = [
+                        str(sheet.cell_value(self.settings.start_row - 1, col) or "")
+                        for col in range(sheet.ncols)
+                    ]
+                    for row_number in range(self.settings.start_row, sheet.nrows):
+                        try:
+                            row_data = {}
+                            for col, key in enumerate(headers):
+                                cell_type = sheet.cell_type(row_number, col)
+                                cell_value = sheet.cell_value(row_number, col)
+
+                                if cell_type == xlrd.XL_CELL_DATE:
+                                    # Convert Excel date to Python datetime
+                                    try:
+                                        python_date = datetime(
+                                            *xlrd.xldate_as_tuple(
+                                                cell_value, workbook.datemode
+                                            )
+                                        )
+                                        row_data[key] = python_date
+                                    except Exception:
+                                        # If date conversion fails, use the original value
+                                        row_data[key] = (
+                                            str(cell_value)
+                                            if cell_value is not None
+                                            else None
+                                        )
+                                elif cell_value is None:
+                                    row_data[key] = None
+                                else:
+                                    row_data[key] = str(cell_value)
+
+                            self._process_row(
+                                row_data, row_number - self.settings.start_row + 1
+                            )
+                        except Exception as e:
+                            if self.settings.skip_errors:
+                                self._log(
+                                    "warning",
+                                    f"Error processing row {row_number} in sheet '{sheet_name}': {str(e)}",
+                                )
+                                self._increment_totals("failed", value=1)
+                            else:
+                                raise
+
+        except (InvalidFileException, xlrd.XLRDError) as e:
+            raise ValueError(
+                f"Invalid {self.settings.file_type.upper()} file format: {str(e)}"
+            )
+
+    def _validate_file_path(self, file_path: str) -> str:
+        """
+        Validates that the file path is within the allowed temporary directory.
+        Returns the absolute path.
+        """
+        abs_path = os.path.abspath(file_path)
+        if not abs_path.startswith(self.TEMP_DIR):
+            raise ValueError(f"Invalid file path. File must be in {self.TEMP_DIR}")
+        return abs_path
+
+    def process_file(self, file_path: str):
+        with cachalot_disabled():
+            # Validate and get absolute path
+            file_path = self._validate_file_path(file_path)
+
+            self._update_status("PROCESSING")
+            self.import_run.started_at = timezone.now()
+            self.import_run.save(update_fields=["started_at"])
+
+            self._log("info", "Starting import process")
+
+            try:
+                if isinstance(self.settings, version_1.CSVImportSettings):
+                    self._process_csv(file_path)
+                elif isinstance(self.settings, version_1.ExcelImportSettings):
+                    self._process_excel(file_path)
+
+                self._update_status("FINISHED")
+                self._log(
+                    "info",
+                    f"Import completed successfully. "
+                    f"Successful: {self.import_run.successful_rows}, "
+                    f"Failed: {self.import_run.failed_rows}, "
+                    f"Skipped: {self.import_run.skipped_rows}",
+                )
+
+            except Exception as e:
+                self._update_status("FAILED")
+                self._log("error", f"Import failed: {str(e)}")
+                raise Exception("Import failed")
+
+            finally:
+                self._log("info", "Cleaning up temporary files")
+                try:
+                    if os.path.exists(file_path):
+                        os.remove(file_path)
+                        self._log("info", f"Deleted temporary file: {file_path}")
+                except OSError as e:
+                    self._log("warning", f"Failed to delete temporary file: {str(e)}")
+
+                self.import_run.finished_at = timezone.now()
+                self.import_run.save(update_fields=["finished_at"])

app/apps/import_app/tasks.py

@@ -0,0 +1,18 @@
+import logging
+
+from procrastinate.contrib.django import app
+
+from apps.import_app.models import ImportRun
+from apps.import_app.services import ImportServiceV1
+
+logger = logging.getLogger(__name__)
+
+
+@app.task(name="process_import")
+def process_import(import_run_id: int, file_path: str):
+    try:
+        import_run = ImportRun.objects.get(id=import_run_id)
+        import_service = ImportServiceV1(import_run)
+        import_service.process_file(file_path)
+    except ImportRun.DoesNotExist:
+        raise ValueError(f"ImportRun with id {import_run_id} not found")

app/apps/import_app/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

app/apps/import_app/urls.py

@@ -0,0 +1,56 @@
+from django.urls import path
+import apps.import_app.views as views
+
+urlpatterns = [
+    path("import/", views.import_view, name="import"),
+    path(
+        "import/presets/",
+        views.import_presets_list,
+        name="import_presets_list",
+    ),
+    path(
+        "import/profiles/",
+        views.import_profile_index,
+        name="import_profiles_index",
+    ),
+    path(
+        "import/profiles/list/",
+        views.import_profile_list,
+        name="import_profiles_list",
+    ),
+    path(
+        "import/profiles/<int:profile_id>/delete/",
+        views.import_profile_delete,
+        name="import_profile_delete",
+    ),
+    path(
+        "import/profiles/add/",
+        views.import_profile_add,
+        name="import_profiles_add",
+    ),
+    path(
+        "import/profiles/<int:profile_id>/edit/",
+        views.import_profile_edit,
+        name="import_profile_edit",
+    ),
+    path(
+        "import/profiles/<int:profile_id>/runs/list/",
+        views.import_runs_list,
+        name="import_profile_runs_list",
+    ),
+    path(
+        "import/profiles/<int:profile_id>/runs/<int:run_id>/log/",
+        views.import_run_log,
+        name="import_run_log",
+    ),
+    path(
+        "import/profiles/<int:profile_id>/runs/<int:run_id>/delete/",
+        views.import_run_delete,
+        name="import_run_delete",
+    ),
+    path(
+        "import/profiles/<int:profile_id>/runs/add/",
+        views.import_run_add,
+        name="import_run_add",
+    ),
+]

app/apps/import_app/views.py

@@ -0,0 +1,227 @@
+import shutil
+
+from django.contrib import messages
+from django.contrib.auth.decorators import login_required
+from django.core.files.storage import FileSystemStorage
+from django.http import HttpResponse
+from django.shortcuts import render, get_object_or_404
+from django.utils.translation import gettext_lazy as _
+from django.views.decorators.http import require_http_methods
+
+from apps.common.decorators.htmx import only_htmx
+from apps.import_app.forms import ImportRunFileUploadForm, ImportProfileForm
+from apps.import_app.models import ImportRun, ImportProfile
+from apps.import_app.services import PresetService
+from apps.import_app.tasks import process_import
+
+
+def import_view(request):
+    import_profile = ImportProfile.objects.get(id=2)
+    shutil.copyfile(
+        "/usr/src/app/apps/import_app/teste2.csv", "/usr/src/app/temp/teste2.csv"
+    )
+    ir = ImportRun.objects.create(profile=import_profile, file_name="teste.csv")
+    process_import.defer(
+        import_run_id=ir.id,
+        file_path="/usr/src/app/temp/teste2.csv",
+    )
+    return HttpResponse("Hello, world. You're at the polls page.")
+
+
+@login_required
+@require_http_methods(["GET"])
+def import_presets_list(request):
+    presets = PresetService.get_all_presets()
+    return render(
+        request,
+        "import_app/fragments/profiles/list_presets.html",
+        {"presets": presets},
+    )
+
+
+@login_required
+@require_http_methods(["GET", "POST"])
+def import_profile_index(request):
+    return render(
+        request,
+        "import_app/pages/profiles_index.html",
+    )
+
+
+@only_htmx
+@login_required
+@require_http_methods(["GET", "POST"])
+def import_profile_list(request):
+    profiles = ImportProfile.objects.all()
+
+    return render(
+        request,
+        "import_app/fragments/profiles/list.html",
+        {"profiles": profiles},
+    )
+
+
+@only_htmx
+@login_required
+@require_http_methods(["GET", "POST"])
+def import_profile_add(request):
+    message = request.POST.get("message", None)
+
+    if request.method == "POST" and request.POST.get("submit"):
+        form = ImportProfileForm(request.POST)
+
+        if form.is_valid():
+            form.save()
+            messages.success(request, _("Import Profile added successfully"))
+
+            return HttpResponse(
+                status=204,
+                headers={
+                    "HX-Trigger": "updated, hide_offcanvas",
+                },
+            )
+    else:
+        form = ImportProfileForm(
+            initial={
+                "name": request.POST.get("name"),
+                "version": int(request.POST.get("version", 1)),
+                "yaml_config": request.POST.get("yaml_config"),
+            }
+        )
+
+    return render(
+        request,
+        "import_app/fragments/profiles/add.html",
+        {"form": form, "message": message},
+    )
+
+
+@only_htmx
+@login_required
+@require_http_methods(["GET", "POST"])
+def import_profile_edit(request, profile_id):
+    profile = get_object_or_404(ImportProfile, id=profile_id)
+
+    if request.method == "POST":
+        form = ImportProfileForm(request.POST, instance=profile)
+
+        if form.is_valid():
+            form.save()
+            messages.success(request, _("Import Profile update successfully"))
+
+            return HttpResponse(
+                status=204,
+                headers={
+                    "HX-Trigger": "updated, hide_offcanvas",
+                },
+            )
+    else:
+        form = ImportProfileForm(instance=profile)
+
+    return render(
+        request,
+        "import_app/fragments/profiles/edit.html",
+        {"form": form, "profile": profile},
+    )
+
+
+@only_htmx
+@login_required
+@require_http_methods(["DELETE"])
+def import_profile_delete(request, profile_id):
+    profile = ImportProfile.objects.get(id=profile_id)
+
+    profile.delete()
+
+    messages.success(request, _("Import Profile deleted successfully"))
+
+    return HttpResponse(
+        status=204,
+        headers={
+            "HX-Trigger": "updated",
+        },
+    )
+
+
+@only_htmx
+@login_required
+@require_http_methods(["GET", "POST"])
+def import_runs_list(request, profile_id):
+    profile = ImportProfile.objects.get(id=profile_id)
+
+    runs = ImportRun.objects.filter(profile=profile).order_by("-id")
+
+    return render(
+        request,
+        "import_app/fragments/runs/list.html",
+        {"profile": profile, "runs": runs},
+    )
+
+
+@only_htmx
+@login_required
+@require_http_methods(["GET", "POST"])
+def import_run_log(request, profile_id, run_id):
+    run = ImportRun.objects.get(profile__id=profile_id, id=run_id)
+
+    return render(
+        request,
+        "import_app/fragments/runs/log.html",
+        {"run": run},
+    )
+
+
+@only_htmx
+@login_required
+@require_http_methods(["GET", "POST"])
+def import_run_add(request, profile_id):
+    profile = ImportProfile.objects.get(id=profile_id)
+
+    if request.method == "POST":
+        form = ImportRunFileUploadForm(request.POST, request.FILES)
+
+        if form.is_valid():
+            uploaded_file = request.FILES["file"]
+            fs = FileSystemStorage(location="/usr/src/app/temp")
+            filename = fs.save(uploaded_file.name, uploaded_file)
+            file_path = fs.path(filename)
+
+            import_run = ImportRun.objects.create(profile=profile, file_name=filename)
+
+            # Defer the procrastinate task
+            process_import.defer(import_run_id=import_run.id, file_path=file_path)
+
+            messages.success(request, _("Import Run queued successfully"))
+
+            return HttpResponse(
+                status=204,
+                headers={
+                    "HX-Trigger": "updated, hide_offcanvas",
+                },
+            )
+    else:
+        form = ImportRunFileUploadForm()
+
+    return render(
+        request,
+        "import_app/fragments/runs/add.html",
+        {"form": form, "profile": profile},
+    )
+
+
+@only_htmx
+@login_required
+@require_http_methods(["DELETE"])
+def import_run_delete(request, profile_id, run_id):
+    run = ImportRun.objects.get(profile__id=profile_id, id=run_id)
+
+    run.delete()
+
+    messages.success(request, _("Run deleted successfully"))
+
+    return HttpResponse(
+        status=204,
+        headers={
+            "HX-Trigger": "updated",
+        },
+    )