Merge pull request #149

feat(import:v1): add "add" and "subtract" transformations

app/apps/import_app/schemas/v1.py

@@ -47,6 +47,34 @@ class SplitTransformationRule(BaseModel):
     )
 
 
+class AddTransformationRule(BaseModel):
+    type: Literal["add"]
+    field: str = Field(..., description="Field to add to the source value")
+    absolute_values: bool = Field(
+        default=False, description="Use absolute values for addition"
+    )
+    thousand_separator: str = Field(
+        default="", description="Thousand separator character"
+    )
+    decimal_separator: str = Field(
+        default=".", description="Decimal separator character"
+    )
+
+
+class SubtractTransformationRule(BaseModel):
+    type: Literal["subtract"]
+    field: str = Field(..., description="Field to subtract from the source value")
+    absolute_values: bool = Field(
+        default=False, description="Use absolute values for subtraction"
+    )
+    thousand_separator: str = Field(
+        default="", description="Thousand separator character"
+    )
+    decimal_separator: str = Field(
+        default=".", description="Decimal separator character"
+    )
+
+
 class CSVImportSettings(BaseModel):
     skip_errors: bool = Field(
         default=False,
@@ -78,6 +106,8 @@ class ColumnMapping(BaseModel):
             | HashTransformationRule
             | MergeTransformationRule
             | SplitTransformationRule
+            | AddTransformationRule
+            | SubtractTransformationRule
         ]
     ] = Field(default_factory=list)
 

app/apps/import_app/services/v1.py

@@ -4,10 +4,9 @@ import logging
 import os
 import re
 from datetime import datetime
-from decimal import Decimal
+from decimal import Decimal, InvalidOperation
 from typing import Dict, Any, Literal, Union
 
-import cachalot.api
 import yaml
 from cachalot.api import cachalot_disabled
 from django.utils import timezone
@@ -129,8 +128,8 @@ class ImportService:
 
         self.import_run.save(update_fields=["status"])
 
-    @staticmethod
     def _transform_value(
+        self,
         value: str,
         mapping: version_1.ColumnMapping,
         row: Dict[str, str] = None,
@@ -195,7 +194,38 @@ class ImportService:
                     transformed = parts[transform.index] if parts else ""
                 else:
                     transformed = parts
+            elif transform.type in ["add", "subtract"]:
+                try:
+                    source_value = Decimal(transformed)
 
+                    # First check row data, then mapped data if not found
+                    field_value = row.get(transform.field)
+                    if field_value is None and transform.field.startswith("__"):
+                        field_value = mapped_data.get(transform.field[2:])
+
+                    if field_value is None:
+                        raise KeyError(
+                            f"Field '{transform.field}' not found in row or mapped data"
+                        )
+
+                    field_value = self._prepare_numeric_value(
+                        str(field_value),
+                        transform.thousand_separator,
+                        transform.decimal_separator,
+                    )
+
+                    if transform.absolute_values:
+                        source_value = abs(source_value)
+                        field_value = abs(field_value)
+
+                    if transform.type == "add":
+                        transformed = str(source_value + field_value)
+                    else:  # subtract
+                        transformed = str(source_value - field_value)
+                except (InvalidOperation, KeyError, AttributeError) as e:
+                    logger.warning(
+                        f"Error in {transform.type} transformation: {e}. Values: {transformed}, {transform.field}"
+                    )
         return transformed
 
     def _create_transaction(self, data: Dict[str, Any]) -> Transaction:
@@ -537,6 +567,20 @@ class ImportService:
 
         return mapped_data
 
+    @staticmethod
+    def _prepare_numeric_value(
+        value: str, thousand_separator: str, decimal_separator: str
+    ) -> Decimal:
+        # Remove thousand separators
+        if thousand_separator:
+            value = value.replace(thousand_separator, "")
+
+        # Replace decimal separator with dot
+        if decimal_separator != ".":
+            value = value.replace(decimal_separator, ".")
+
+        return Decimal(value)
+
     def _process_row(self, row: Dict[str, str], row_number: int) -> None:
         try:
             mapped_data = self._map_row(row)
@@ -652,4 +696,3 @@ class ImportService:
 
                 self.import_run.finished_at = timezone.now()
                 self.import_run.save(update_fields=["finished_at"])
-        cachalot.api.invalidate()

app/apps/import_app/tasks.py

@@ -1,6 +1,5 @@
 import logging
 
-import cachalot.api
 from procrastinate.contrib.django import app
 
 from apps.import_app.models import ImportRun
@@ -15,7 +14,5 @@ def process_import(import_run_id: int, file_path: str):
         import_run = ImportRun.objects.get(id=import_run_id)
         import_service = ImportServiceV1(import_run)
         import_service.process_file(file_path)
-        cachalot.api.invalidate()
     except ImportRun.DoesNotExist:
-        cachalot.api.invalidate()
         raise ValueError(f"ImportRun with id {import_run_id} not found")