Implements #2006 - run reports and scripts in the background

netbox/extras/api/nested_serializers.py

@@ -1,13 +1,14 @@
 from rest_framework import serializers
 
-from extras import models
-from utilities.api import WritableNestedSerializer
+from extras import choices, models
+from users.api.nested_serializers import NestedUserSerializer
+from utilities.api import ChoiceField, WritableNestedSerializer
 
 __all__ = [
     'NestedConfigContextSerializer',
     'NestedExportTemplateSerializer',
     'NestedGraphSerializer',
-    'NestedReportResultSerializer',
+    'NestedJobResultSerializer',
     'NestedTagSerializer',
 ]
 
@@ -44,13 +45,13 @@ class NestedTagSerializer(WritableNestedSerializer):
         fields = ['id', 'url', 'name', 'slug', 'color']
 
 
-class NestedReportResultSerializer(serializers.ModelSerializer):
-    url = serializers.HyperlinkedIdentityField(
-        view_name='extras-api:report-detail',
-        lookup_field='report',
-        lookup_url_kwarg='pk'
+class NestedJobResultSerializer(serializers.ModelSerializer):
+    url = serializers.HyperlinkedIdentityField(view_name='extras-api:jobresult-detail')
+    status = ChoiceField(choices=choices.JobResultStatusChoices)
+    user = NestedUserSerializer(
+        read_only=True
     )
 
     class Meta:
-        model = models.ReportResult
-        fields = ['url', 'created', 'user', 'failed']
+        model = models.JobResult
+        fields = ['url', 'created', 'completed', 'user', 'status']

netbox/extras/api/serializers.py

@@ -11,7 +11,7 @@ from dcim.models import Device, DeviceRole, Platform, Rack, Region, Site
 from extras.choices import *
 from extras.constants import *
 from extras.models import (
-    ConfigContext, ExportTemplate, Graph, ImageAttachment, ObjectChange, ReportResult, Tag,
+    ConfigContext, ExportTemplate, Graph, ImageAttachment, ObjectChange, JobResult, Tag,
 )
 from extras.utils import FeatureQuery
 from tenancy.api.nested_serializers import NestedTenantSerializer, NestedTenantGroupSerializer
@@ -232,27 +232,46 @@ class ConfigContextSerializer(ValidatedModelSerializer):
         ]
 
 
+#
+# Job Results
+#
+
+class JobResultSerializer(serializers.ModelSerializer):
+    url = serializers.HyperlinkedIdentityField(view_name='extras-api:jobresult-detail')
+    user = NestedUserSerializer(
+        read_only=True
+    )
+    status = ChoiceField(choices=JobResultStatusChoices, read_only=True)
+    obj_type = ContentTypeField(
+        read_only=True
+    )
+
+    class Meta:
+        model = JobResult
+        fields = [
+            'id', 'url', 'created', 'completed', 'name', 'obj_type', 'status', 'user', 'data', 'job_id',
+        ]
+
+
 #
 # Reports
 #
 
-class ReportResultSerializer(serializers.ModelSerializer):
-
-    class Meta:
-        model = ReportResult
-        fields = ['created', 'user', 'failed', 'data']
-
-
 class ReportSerializer(serializers.Serializer):
+    url = serializers.HyperlinkedIdentityField(
+        view_name='extras-api:report-detail',
+        lookup_field='full_name',
+        lookup_url_kwarg='pk'
+    )
     module = serializers.CharField(max_length=255)
     name = serializers.CharField(max_length=255)
     description = serializers.CharField(max_length=255, required=False)
     test_methods = serializers.ListField(child=serializers.CharField(max_length=255))
-    result = NestedReportResultSerializer()
+    result = NestedJobResultSerializer()
 
 
 class ReportDetailSerializer(ReportSerializer):
-    result = ReportResultSerializer()
+    result = JobResultSerializer()
 
 
 #
@@ -260,10 +279,16 @@ class ReportDetailSerializer(ReportSerializer):
 #
 
 class ScriptSerializer(serializers.Serializer):
+    url = serializers.HyperlinkedIdentityField(
+        view_name='extras-api:script-detail',
+        lookup_field='full_name',
+        lookup_url_kwarg='pk'
+    )
     id = serializers.SerializerMethodField(read_only=True)
     name = serializers.SerializerMethodField(read_only=True)
     description = serializers.SerializerMethodField(read_only=True)
     vars = serializers.SerializerMethodField(read_only=True)
+    result = NestedJobResultSerializer()
 
     def get_id(self, instance):
         return '{}.{}'.format(instance.__module__, instance.__name__)
@@ -280,6 +305,10 @@ class ScriptSerializer(serializers.Serializer):
         }
 
 
+class ScriptDetailSerializer(ScriptSerializer):
+    result = JobResultSerializer()
+
+
 class ScriptInputSerializer(serializers.Serializer):
     data = serializers.JSONField()
     commit = serializers.BooleanField()

netbox/extras/api/urls.py

@@ -41,5 +41,8 @@ router.register('scripts', views.ScriptViewSet, basename='script')
 # Change logging
 router.register('object-changes', views.ObjectChangeViewSet)
 
+# Job Results
+router.register('job-results', views.JobResultViewSet)
+
 app_name = 'extras-api'
 urlpatterns = router.urls

netbox/extras/api/views.py

@@ -10,8 +10,9 @@ from rest_framework.response import Response
 from rest_framework.viewsets import ReadOnlyModelViewSet, ViewSet
 
 from extras import filters
+from extras.choices import JobResultStatusChoices
 from extras.models import (
-    ConfigContext, CustomFieldChoice, ExportTemplate, Graph, ImageAttachment, ObjectChange, ReportResult, Tag,
+    ConfigContext, CustomFieldChoice, ExportTemplate, Graph, ImageAttachment, ObjectChange, JobResult, Tag,
 )
 from extras.reports import get_report, get_reports
 from extras.scripts import get_script, get_scripts, run_script
@@ -165,13 +166,21 @@ class ReportViewSet(ViewSet):
         Compile all reports and their related results (if any). Result data is deferred in the list view.
         """
         report_list = []
+        report_content_type = ContentType.objects.get(app_label='extras', model='report')
+        results = {
+            r.name: r
+            for r in JobResult.objects.filter(
+                obj_type=report_content_type,
+                status__in=JobResultStatusChoices.TERMINAL_STATE_CHOICES
+            ).defer('data')
+        }
 
         # Iterate through all available Reports.
         for module_name, reports in get_reports():
             for report in reports:
 
-                # Attach the relevant ReportResult (if any) to each Report.
-                report.result = ReportResult.objects.filter(report=report.full_name).defer('data').first()
+                # Attach the relevant JobResult (if any) to each Report.
+                report.result = results.get(report.full_name, None)
                 report_list.append(report)
 
         serializer = serializers.ReportSerializer(report_list, many=True, context={
@@ -185,27 +194,41 @@ class ReportViewSet(ViewSet):
         Retrieve a single Report identified as "<module>.<report>".
         """
 
-        # Retrieve the Report and ReportResult, if any.
+        # Retrieve the Report and JobResult, if any.
         report = self._retrieve_report(pk)
-        report.result = ReportResult.objects.filter(report=report.full_name).first()
+        report_content_type = ContentType.objects.get(app_label='extras', model='report')
+        report.result = JobResult.objects.filter(
+            obj_type=report_content_type,
+            name=report.full_name,
+            status__in=JobResultStatusChoices.TERMINAL_STATE_CHOICES
+        ).first()
 
-        serializer = serializers.ReportDetailSerializer(report)
+        serializer = serializers.ReportDetailSerializer(report, context={
+            'request': request
+        })
 
         return Response(serializer.data)
 
     @action(detail=True, methods=['post'])
     def run(self, request, pk):
         """
-        Run a Report and create a new ReportResult, overwriting any previous result for the Report.
+        Run a Report identified as "<module>.<script>" and return the pending JobResult as the result
         """
 
         # Check that the user has permission to run reports.
         if not request.user.has_perm('extras.add_reportresult'):
             raise PermissionDenied("This user does not have permission to run reports.")
 
-        # Retrieve and run the Report. This will create a new ReportResult.
+        # Retrieve and run the Report. This will create a new JobResult.
         report = self._retrieve_report(pk)
-        report.run()
+        report_content_type = ContentType.objects.get(app_label='extras', model='report')
+        job_result = JobResult.enqueue_job(
+            run_report,
+            report.full_name,
+            report_content_type,
+            request.user
+        )
+        report.result = job_result
 
         serializer = serializers.ReportDetailSerializer(report)
 
@@ -231,23 +254,42 @@ class ScriptViewSet(ViewSet):
 
     def list(self, request):
 
+        script_content_type = ContentType.objects.get(app_label='extras', model='script')
+        results = {
+            r.name: r
+            for r in JobResult.objects.filter(
+                obj_type=script_content_type,
+                status__in=JobResultStatusChoices.TERMINAL_STATE_CHOICES
+            ).defer('data').order_by('created')
+        }
+
         flat_list = []
         for script_list in get_scripts().values():
             flat_list.extend(script_list.values())
 
+        # Attach JobResult objects to each script (if any)
+        for script in flat_list:
+            script.result = results.get(script.full_name, None)
+
         serializer = serializers.ScriptSerializer(flat_list, many=True, context={'request': request})
 
         return Response(serializer.data)
 
     def retrieve(self, request, pk):
         script = self._get_script(pk)
-        serializer = serializers.ScriptSerializer(script, context={'request': request})
+        script_content_type = ContentType.objects.get(app_label='extras', model='script')
+        script.result = JobResult.objects.filter(
+            obj_type=script_content_type,
+            name=script.full_name,
+            status__in=JobResultStatusChoices.TERMINAL_STATE_CHOICES
+        ).first()
+        serializer = serializers.ScriptDetailSerializer(script, context={'request': request})
 
         return Response(serializer.data)
 
     def post(self, request, pk):
         """
-        Run a Script identified as "<module>.<script>".
+        Run a Script identified as "<module>.<script>" and return the pending JobResult as the result
         """
         script = self._get_script(pk)()
         input_serializer = serializers.ScriptInputSerializer(data=request.data)
@@ -255,10 +297,21 @@ class ScriptViewSet(ViewSet):
         if input_serializer.is_valid():
             data = input_serializer.data['data']
             commit = input_serializer.data['commit']
-            script.output, execution_time = run_script(script, data, request, commit)
-            output_serializer = serializers.ScriptOutputSerializer(script)
 
-            return Response(output_serializer.data)
+            script_content_type = ContentType.objects.get(app_label='extras', model='script')
+            job_result = JobResult.enqueue_job(
+                run_script,
+                script.full_name,
+                script_content_type,
+                request.user,
+                data=form.cleaned_data,
+                request=copy_safe_request(request),
+                commit=commit
+            )
+            script.result = job_result
+            serializer = serializers.ScriptDetailSerializer(script)
+
+            return Response(serializer.data)
 
         return Response(input_serializer.errors, status=status.HTTP_400_BAD_REQUEST)
 
@@ -274,3 +327,16 @@ class ObjectChangeViewSet(ReadOnlyModelViewSet):
     queryset = ObjectChange.objects.prefetch_related('user')
     serializer_class = serializers.ObjectChangeSerializer
     filterset_class = filters.ObjectChangeFilterSet
+
+
+#
+# Job Results
+#
+
+class JobResultViewSet(ReadOnlyModelViewSet):
+    """
+    Retrieve a list of job results
+    """
+    queryset = JobResult.objects.prefetch_related('user')
+    serializer_class = serializers.JobResultSerializer
+    filterset_class = filters.JobResultFilterSet