Merge branch 'eitchtee:main' into main

.env.example

@@ -23,3 +23,5 @@ WEB_CONCURRENCY=4
 ENABLE_SOFT_DELETE=false
 # If ENABLE_SOFT_DELETE is true, transactions deleted for more than KEEP_DELETED_TRANSACTIONS_FOR days will be truly deleted. Set to 0 to keep all.
 KEEP_DELETED_TRANSACTIONS_FOR=365
+
+TASK_WORKERS=1 # This only work if you're using the single container option. Increase to have more open queues via procrastinate, you probably don't need to increase this.

.github/workflows/release.yml

@@ -3,6 +3,8 @@ name: Release Pipeline
 on:
   release:
     types: [created]
+  push:
+    branches: [ main ]
 
 env:
   IMAGE_NAME: wygiwyh
@@ -29,7 +31,21 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Build and push image
+      - name: Build and push nightly image
+        if: github.event_name == 'push'
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./docker/prod/django/Dockerfile
+          push: true
+          provenance: false
+          tags: ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.IMAGE_NAME }}:nightly
+          platforms: linux/amd64,linux/arm64
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Build and push release image
+        if: github.event_name == 'release'
         uses: docker/build-push-action@v6
         with:
           context: .

app/WYGIWYH/settings.py

@@ -222,7 +222,7 @@ SESSION_COOKIE_SECURE = os.getenv("HTTPS_ENABLED", "false").lower() == "true"
 
 DEBUG_TOOLBAR_CONFIG = {
     "ROOT_TAG_EXTRA_ATTRS": "hx-preserve",
-    "SHOW_TOOLBAR_CALLBACK": lambda r: False,  # disables it}
+    # "SHOW_TOOLBAR_CALLBACK": lambda r: False,  # disables it
 }
 DEBUG_TOOLBAR_PANELS = [
     "debug_toolbar.panels.history.HistoryPanel",

app/apps/common/tasks.py

@@ -1,5 +1,8 @@
 import logging
 
+from asgiref.sync import sync_to_async
+from django.core import management
+
 from procrastinate import builtin_tasks
 from procrastinate.contrib.django import app
 
@@ -24,3 +27,16 @@ async def remove_old_jobs(context, timestamp):
             exc_info=True,
         )
         raise e
+
+
+@app.periodic(cron="0 6 1 * *")
+@app.task(queueing_lock="remove_expired_sessions")
+async def remove_expired_sessions(timestamp=None):
+    """Cleanup expired sessions by using Django management command."""
+    try:
+        await sync_to_async(management.call_command)("clearsessions", verbosity=0)
+    except Exception:
+        logger.error(
+            "Error while executing 'remove_expired_sessions' task",
+            exc_info=True,
+        )

app/apps/import_app/schemas/v1.py

@@ -65,7 +65,7 @@ class CSVImportSettings(BaseModel):
 
 
 class ColumnMapping(BaseModel):
-    source: Optional[str] = Field(
+    source: Optional[str] | Optional[list[str]] = Field(
         default=None,
         description="CSV column header. If None, the field will be generated from transformations",
     )

app/apps/import_app/services/v1.py

@@ -486,8 +486,18 @@ class ImportService:
         mapped_data = {}
 
         for field, mapping in self.mapping.items():
-            # If source is None, use None as the initial value
+            value = None
-            value = row.get(mapping.source) if mapping.source else None
+
+            if isinstance(mapping.source, str):
+                value = row.get(mapping.source)
+            elif isinstance(mapping.source, list):
+                for source in mapping.source:
+                    value = row.get(source)
+                    if value is not None:
+                        break
+            else:
+                # If source is None, use None as the initial value
+                value = None
 
             # Use default_value if value is None
             if value is None:

app/apps/monthly_overview/views.py

@@ -30,6 +30,8 @@ def index(request):
 @login_required
 @require_http_methods(["GET"])
 def monthly_overview(request, month: int, year: int):
+    order = request.session.get("monthly_transactions_order", "default")
+
     if month < 1 or month > 12:
         from django.http import Http404
 
@@ -54,6 +56,7 @@ def monthly_overview(request, month: int, year: int):
             "previous_month": previous_month,
             "previous_year": previous_year,
             "filter": f,
+            "order": order,
         },
     )
 
@@ -62,7 +65,12 @@ def monthly_overview(request, month: int, year: int):
 @login_required
 @require_http_methods(["GET"])
 def transactions_list(request, month: int, year: int):
-    order = request.GET.get("order")
+    order = request.session.get("monthly_transactions_order", "default")
+
+    if "order" in request.GET:
+        order = request.GET["order"]
+        if order != request.session.get("monthly_transactions_order", "default"):
+            request.session["monthly_transactions_order"] = order
 
     f = TransactionsFilter(request.GET)
     transactions_filtered = (

app/apps/transactions/views/transactions.py

@@ -313,15 +313,23 @@ def transaction_pay(request, transaction_id):
 @login_required
 @require_http_methods(["GET"])
 def transaction_all_index(request):
+    order = request.session.get("all_transactions_order", "default")
     f = TransactionsFilter(request.GET)
-    return render(request, "transactions/pages/transactions.html", {"filter": f})
+    return render(
+        request, "transactions/pages/transactions.html", {"filter": f, "order": order}
+    )
 
 
 @only_htmx
 @login_required
 @require_http_methods(["GET"])
 def transaction_all_list(request):
-    order = request.GET.get("order")
+    order = request.session.get("all_transactions_order", "default")
+
+    if "order" in request.GET:
+        order = request.GET["order"]
+        if order != request.session.get("all_transactions_order", "default"):
+            request.session["all_transactions_order"] = order
 
     transactions = Transaction.objects.prefetch_related(
         "account",

app/templates/monthly_overview/pages/overview.html

@@ -113,9 +113,9 @@
           <div class="text-sm-end" _="on change trigger updated on window">
             <label for="order">{% translate "Order by" %}</label>
             <select class="tw-border-0 focus-visible:tw-outline-0 w-full pe-2 tw-leading-normal text-bg-tertiary tw-font-medium rounded" name="order" id="order">
-              <option value="default">{% translate 'Default' %}</option>
+              <option value="default" {% if order == 'default' %}selected{% endif %}>{% translate 'Default' %}</option>
-              <option value="older">{% translate 'Oldest first' %}</option>
+              <option value="older" {% if order == 'older' %}selected{% endif %}>{% translate 'Oldest first' %}</option>
-              <option value="newer">{% translate 'Newest first' %}</option>
+              <option value="newer" {% if order == 'newer' %}selected{% endif %}>{% translate 'Newest first' %}</option>
             </select>
           </div>
         </div>

app/templates/transactions/pages/transactions.html

@@ -32,9 +32,9 @@
         <div class="tw-content-center" _="on change trigger updated on window">
           <label for="order">{% translate "Order by" %}</label>
           <select class="tw-border-0 focus-visible:tw-outline-0 w-full pe-2 tw-leading-normal text-bg-tertiary tw-font-medium rounded" name="order" id="order">
-            <option value="default">{% translate 'Default' %}</option>
+            <option value="default" {% if order == 'default' %}selected{% endif %}>{% translate 'Default' %}</option>
-            <option value="older">{% translate 'Oldest first' %}</option>
+            <option value="older" {% if order == 'older' %}selected{% endif %}>{% translate 'Oldest first' %}</option>
-            <option value="newer">{% translate 'Newest first' %}</option>
+            <option value="newer" {% if order == 'newer' %}selected{% endif %}>{% translate 'Newest first' %}</option>
           </select>
         </div>
       </div>

docker-compose.dev.yml

@@ -3,13 +3,13 @@ volumes:
   wygiwyh_temp:
 
 services:
-  web: &django
+  web:
     build:
       context: .
       dockerfile: ./docker/dev/django/Dockerfile
     image: wygiwyh_dev_server
     container_name: wygiwyh_dev_server
-    command: /start
+    command: /start-supervisor
     volumes:
       - ./app/:/usr/src/app/:z
       - ./frontend/:/usr/src/frontend:z
@@ -54,12 +54,12 @@ services:
       - '${SQL_PORT}:5432'
     restart: unless-stopped
 
-  procrastinate:
+#  procrastinate:
-    <<: *django
+#    <<: *django
-    image: wygiwyh_dev_procrastinate
+#    image: wygiwyh_dev_procrastinate
-    container_name: wygiwyh_dev_procrastinate
+#    container_name: wygiwyh_dev_procrastinate
-    depends_on:
+#    depends_on:
-      - db
+#      - db
-    ports: [ ]
+#    ports: [ ]
-    command: /start-procrastinate
+#    command: /start-procrastinate
-    restart: unless-stopped
+#    restart: unless-stopped

docker-compose.prod.yml

@@ -2,15 +2,13 @@ services:
   web:
     image: eitchtee/wygiwyh:latest
     container_name: ${SERVER_NAME}
-    command: /start
+    command: /start-single
     ports:
       - "${OUTBOUND_PORT}:8000"
     env_file:
       - .env
     depends_on:
       - db
-    volumes:
-      - wygiwyh_temp:/usr/src/app/temp/
     restart: unless-stopped
 
   db:
@@ -23,18 +21,3 @@ services:
       - POSTGRES_USER=${SQL_USER}
       - POSTGRES_PASSWORD=${SQL_PASSWORD}
       - POSTGRES_DB=${SQL_DATABASE}
-
-  procrastinate:
-    image: eitchtee/wygiwyh:latest
-    container_name: ${PROCRASTINATE_NAME}
-    depends_on:
-      - db
-    env_file:
-      - .env
-    volumes:
-      - wygiwyh_temp:/usr/src/app/temp/
-    command: /start-procrastinate
-    restart: unless-stopped
-
-volumes:
-  wygiwyh_temp:

docker/dev/django/Dockerfile

@@ -18,7 +18,7 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
 COPY --from=python-build-stage /usr/src/app/wheels /wheels/
 
 RUN apt-get update && \
-    apt-get install --no-install-recommends -y gettext && \
+    apt-get install --no-install-recommends -y gettext supervisor && \
     rm -rf /var/lib/apt/lists/* && \
     pip install --upgrade pip && \
     pip install --no-cache-dir --no-index --find-links=/wheels/ /wheels/* && \
@@ -26,9 +26,15 @@ RUN apt-get update && \
 
 COPY ./docker/dev/django/start /start
 COPY ./docker/dev/procrastinate/start /start-procrastinate
+COPY ./docker/dev/supervisord/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+COPY ./docker/dev/supervisord/supervisord.conf /etc/supervisord.conf
+COPY ./docker/dev/supervisord/start /start-supervisor
+
 RUN sed -i 's/\r$//g' /start && \
     chmod +x /start && \
     sed -i 's/\r$//g' /start-procrastinate && \
-    chmod +x /start-procrastinate
+    chmod +x /start-procrastinate && \
+    sed -i 's/\r$//g' /start-supervisor && \
+    chmod +x /start-supervisor
 
 COPY ./app .

docker/dev/supervisord/start

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+export TASK_WORKERS=${TASK_WORKERS:=1}
+
+exec supervisord -c /etc/supervisor/conf.d/supervisord.conf

docker/dev/supervisord/supervisord.conf

@@ -0,0 +1,39 @@
+[supervisord]
+nodaemon=true
+logfile=/dev/null
+logfile_maxbytes=0
+pidfile=/tmp/supervisord.pid
+user=root
+
+[supervisorctl]
+serverurl=unix:///run/supervisord.sock
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[unix_http_server]
+file=/run/supervisord.sock
+chmod=0700
+
+[program:web]
+directory=/usr/src/app
+command=/bin/bash /start
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/fd/2
+stderr_logfile_maxbytes=0
+autorestart=true
+startretries=5
+
+[program:procrastinate]
+directory=/usr/src/app
+command=/bin/bash /start-procrastinate
+process_name=%(program_name)s_%(process_num)02d
+numprocs=%(ENV_TASK_WORKERS)s
+numprocs_start=1
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/fd/2
+stderr_logfile_maxbytes=0
+autorestart=true
+startretries=5

docker/prod/django/Dockerfile

@@ -31,7 +31,7 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
 COPY --from=python-build-stage /usr/src/app/wheels /wheels/
 RUN --mount=type=cache,target=/root/.cache/apt \
     apt-get update && \
-    apt-get install --no-install-recommends -y gettext && \
+    apt-get install --no-install-recommends -y gettext supervisor && \
     rm -rf /var/lib/apt/lists/* && \
     pip install --upgrade pip && \
     pip install --no-cache-dir --no-index --find-links=/wheels/ /wheels/* && \
@@ -39,10 +39,15 @@ RUN --mount=type=cache,target=/root/.cache/apt \
 
 COPY --chown=app:app ./docker/prod/django/start /start
 COPY --chown=app:app ./docker/prod/procrastinate/start /start-procrastinate
+COPY --chown=app:app ./docker/prod/supervisord/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+COPY --chown=app:app ./docker/prod/supervisord/supervisord.conf /etc/supervisord.conf
+COPY --chown=app:app ./docker/prod/supervisord/start /start-single
 RUN sed -i 's/\r$//g' /start && \
     chmod +x /start && \
     sed -i 's/\r$//g' /start-procrastinate && \
-    chmod +x /start-procrastinate
+    chmod +x /start-procrastinate && \
+    sed -i 's/\r$//g' /start-single && \
+    chmod +x /start-single
 
 COPY --chown=app:app ./app .
 

docker/prod/supervisord/start

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+export TASK_WORKERS=${TASK_WORKERS:=1}
+
+exec supervisord -c /etc/supervisor/conf.d/supervisord.conf

docker/prod/supervisord/supervisord.conf

@@ -0,0 +1,37 @@
+[supervisord]
+nodaemon=true
+logfile=/dev/null
+logfile_maxbytes=0
+pidfile=/tmp/supervisord.pid
+
+[supervisorctl]
+serverurl=unix:///tmp/supervisord.sock
+
+[unix_http_server]
+file=/tmp/supervisord.sock
+chmod=0700
+
+[program:web]
+user=app
+directory=/usr/src/app
+command=/bin/bash /start
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/fd/2
+stderr_logfile_maxbytes=0
+autorestart=true
+startretries=5
+
+[program:procrastinate]
+user=app
+directory=/usr/src/app
+command=/bin/bash /start-procrastinate
+process_name=%(program_name)s_%(process_num)02d
+numprocs=%(ENV_TASK_WORKERS)s
+numprocs_start=1
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/fd/2
+stderr_logfile_maxbytes=0
+autorestart=true
+startretries=5