Initial POC for #21025

netbox/dcim/api/views.py

@@ -12,7 +12,7 @@ from dcim import filtersets
 from dcim.constants import CABLE_TRACE_SVG_DEFAULT_WIDTH
 from dcim.models import *
 from dcim.svg import CableTraceSVG
-from extras.api.mixins import ConfigContextQuerySetMixin, RenderConfigMixin
+from extras.api.mixins import RenderConfigMixin
 from netbox.api.authentication import IsAuthenticatedOrLoginNotRequired
 from netbox.api.metadata import ContentTypeMetadata
 from netbox.api.pagination import StripCountAnnotationsPaginator
@@ -398,12 +398,7 @@ class PlatformViewSet(MPTTLockedMixin, NetBoxModelViewSet):
 # Devices/modules
 #
 
-class DeviceViewSet(
-    SequentialBulkCreatesMixin,
-    ConfigContextQuerySetMixin,
-    RenderConfigMixin,
-    NetBoxModelViewSet
-):
+class DeviceViewSet(SequentialBulkCreatesMixin, RenderConfigMixin, NetBoxModelViewSet):
     queryset = Device.objects.prefetch_related(
         'parent_bay',  # Referenced by DeviceSerializer.get_parent_device()
     )

netbox/dcim/migrations/0227_device_config_context_data.py

@@ -0,0 +1,21 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('dcim', '0226_modulebay_rebuild_tree'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='device',
+            name='config_context_data',
+            field=models.JSONField(blank=True, editable=False, null=True),
+        ),
+        migrations.AddField(
+            model_name='module',
+            name='config_context_data',
+            field=models.JSONField(blank=True, editable=False, null=True),
+        ),
+    ]

netbox/dcim/views.py

@@ -2683,7 +2683,7 @@ class DeviceInventoryView(DeviceComponentsView):
 
 @register_model_view(Device, 'configcontext', path='config-context')
 class DeviceConfigContextView(ObjectConfigContextView):
-    queryset = Device.objects.annotate_config_context_data()
+    queryset = Device.objects.all()
     base_template = 'dcim/device/base.html'
     tab = ViewTab(
         label=_('Config Context'),

netbox/extras/api/mixins.py

@@ -10,34 +10,11 @@ from netbox.api.renderers import TextRenderer
 from .serializers import ConfigTemplateSerializer
 
 __all__ = (
-    'ConfigContextQuerySetMixin',
     'ConfigTemplateRenderMixin',
     'RenderConfigMixin',
 )
 
 
-class ConfigContextQuerySetMixin:
-    """
-    Used by views that work with config context models (device and virtual machine).
-    Provides a get_queryset() method which deals with adding the config context
-    data annotation or not.
-    """
-    def get_queryset(self):
-        """
-        Build the proper queryset based on the request context
-
-        If the `brief` query param equates to True or the `exclude` query param
-        includes `config_context` as a value, return the base queryset.
-
-        Else, return the queryset annotated with config context data
-        """
-        queryset = super().get_queryset()
-        request = self.get_serializer_context()['request']
-        if self.brief or 'config_context' in request.query_params.get('exclude', []):
-            return queryset
-        return queryset.annotate_config_context_data()
-
-
 class ConfigTemplateRenderMixin:
     """
     Provides a method to return a rendered ConfigTemplate as REST API data.

netbox/extras/graphql/mixins.py

@@ -22,19 +22,7 @@ if TYPE_CHECKING:
 @strawberry.type
 class ConfigContextMixin:
 
-    @classmethod
-    def get_queryset(cls, queryset, info: Info, **kwargs):
-        queryset = super().get_queryset(queryset, info, **kwargs)
-
-        # If `config_context` is requested, call annotate_config_context_data() on the queryset
-        selected = {f.name for f in info.selected_fields[0].selections}
-        if 'config_context' in selected and hasattr(queryset, 'annotate_config_context_data'):
-            return queryset.annotate_config_context_data()
-
-        return queryset
-
-    # Ensure `local_context_data` is fetched when `config_context` is requested
-    @strawberry_django.field(only=['local_context_data'])
+    @strawberry_django.field(only=['config_context_data', 'local_context_data'])
     def config_context(self) -> strawberry.scalars.JSON:
         return self.get_config_context()
 

netbox/extras/management/commands/rebuild_config_context.py

@@ -0,0 +1,40 @@
+from django.core.management.base import BaseCommand
+from django.db import connection
+
+
+class Command(BaseCommand):
+    help = 'Rebuild pre-rendered config context data for all devices and/or virtual machines'
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--devices-only',
+            action='store_true',
+            help='Only rebuild config context data for devices',
+        )
+        parser.add_argument(
+            '--vms-only',
+            action='store_true',
+            help='Only rebuild config context data for virtual machines',
+        )
+
+    def handle(self, *args, **options):
+        devices_only = options['devices_only']
+        vms_only = options['vms_only']
+
+        with connection.cursor() as cursor:
+            if not vms_only:
+                self.stdout.write('Rebuilding config context data for devices...')
+                cursor.execute(
+                    'UPDATE dcim_device SET config_context_data = compute_config_context_for_device(id)'
+                )
+                self.stdout.write(self.style.SUCCESS(f'  Updated {cursor.rowcount} devices'))
+
+            if not devices_only:
+                self.stdout.write('Rebuilding config context data for virtual machines...')
+                cursor.execute(
+                    'UPDATE virtualization_virtualmachine '
+                    'SET config_context_data = compute_config_context_for_vm(id)'
+                )
+                self.stdout.write(self.style.SUCCESS(f'  Updated {cursor.rowcount} virtual machines'))
+
+        self.stdout.write(self.style.SUCCESS('Done.'))

netbox/extras/models/configs.py

@@ -225,6 +225,11 @@ class ConfigContextModel(models.Model):
             "Local config context data takes precedence over source contexts in the final rendered config context"
         )
     )
+    config_context_data = models.JSONField(
+        blank=True,
+        null=True,
+        editable=False,
+    )
 
     class Meta:
         abstract = True
@@ -234,19 +239,21 @@ class ConfigContextModel(models.Model):
         Compile all config data, overwriting lower-weight values with higher-weight values where a collision occurs.
         Return the rendered configuration context for a device or VM.
         """
-        data = {}
+        # Use pre-rendered cached field if available
+        if self.config_context_data is not None:
+            return self.config_context_data
 
-        if not hasattr(self, 'config_context_data'):
-            # The annotation is not available, so we fall back to manually querying for the config context objects
-            config_context_data = ConfigContext.objects.get_for_object(self, aggregate_data=True) or []
+        # Fall back to annotation if queryset was annotated
+        data = {}
+        if hasattr(self, '_annotated_config_context_data'):
+            config_context_data = self._annotated_config_context_data or []
         else:
-            # The attribute may exist, but the annotated value could be None if there is no config context data
-            config_context_data = self.config_context_data or []
+            # Last resort: compute on-the-fly
+            config_context_data = ConfigContext.objects.get_for_object(self, aggregate_data=True) or []
 
         for context in config_context_data:
             data = deepmerge(data, context)
 
-        # If the object has local config context data defined, merge it last
         if self.local_context_data:
             data = deepmerge(data, self.local_context_data)
 

netbox/extras/querysets.py

@@ -90,7 +90,7 @@ class ConfigContextModelQuerySet(RestrictedQuerySet):
         """
         from extras.models import ConfigContext
         return self.annotate(
-            config_context_data=Subquery(
+            _annotated_config_context_data=Subquery(
                 ConfigContext.objects.filter(
                     self._get_config_context_filters()
                 ).annotate(

netbox/extras/tests/test_models.py

@@ -206,6 +206,7 @@ class ConfigContextTest(TestCase):
             "b": 456,
             "c": 777
         }
+        device.refresh_from_db()
         self.assertEqual(device.get_config_context(), expected_data)
 
     def test_name_ordering_after_weight(self):
@@ -235,6 +236,7 @@ class ConfigContextTest(TestCase):
             "b": 456,
             "c": 789
         }
+        device.refresh_from_db()
         self.assertEqual(device.get_config_context(), expected_data)
 
     def test_schema_validation(self):
@@ -303,6 +305,7 @@ class ConfigContextTest(TestCase):
         )
         ConfigContext.objects.bulk_create([context1, context2, context3, context4])
 
+        device.refresh_from_db()
         annotated_queryset = Device.objects.filter(name=device.name).annotate_config_context_data()
         self.assertEqual(device.get_config_context(), annotated_queryset[0].get_config_context())
 
@@ -666,7 +669,7 @@ class ConfigContextTest(TestCase):
         self.assertFalse(queryset.query.distinct)
 
         # Check that tag subqueries DO use DISTINCT by inspecting the annotation
-        config_annotation = queryset.query.annotations.get('config_context_data')
+        config_annotation = queryset.query.annotations.get('_annotated_config_context_data')
         self.assertIsNotNone(config_annotation)
 
         def find_tag_subqueries(where_node):

netbox/virtualization/api/views.py

@@ -1,7 +1,7 @@
 from django.db.models import Sum
 from rest_framework.routers import APIRootView
 
-from extras.api.mixins import ConfigContextQuerySetMixin, RenderConfigMixin
+from extras.api.mixins import RenderConfigMixin
 from netbox.api.viewsets import NetBoxModelViewSet
 from utilities.query_functions import CollateAsChar
 from virtualization import filtersets
@@ -48,7 +48,7 @@ class ClusterViewSet(NetBoxModelViewSet):
 # Virtual machines
 #
 
-class VirtualMachineViewSet(ConfigContextQuerySetMixin, RenderConfigMixin, NetBoxModelViewSet):
+class VirtualMachineViewSet(RenderConfigMixin, NetBoxModelViewSet):
     queryset = VirtualMachine.objects.all()
     filterset_class = filtersets.VirtualMachineFilterSet
 

netbox/virtualization/migrations/0053_virtualmachine_config_context_data.py

@@ -0,0 +1,16 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('virtualization', '0052_gfk_indexes'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='virtualmachine',
+            name='config_context_data',
+            field=models.JSONField(blank=True, editable=False, null=True),
+        ),
+    ]

netbox/virtualization/views.py

@@ -487,7 +487,7 @@ class VirtualMachineVirtualDisksView(generic.ObjectChildrenView):
 
 @register_model_view(VirtualMachine, 'configcontext', path='config-context')
 class VirtualMachineConfigContextView(ObjectConfigContextView):
-    queryset = VirtualMachine.objects.annotate_config_context_data()
+    queryset = VirtualMachine.objects.all()
     base_template = 'virtualization/virtualmachine.html'
     tab = ViewTab(
         label=_('Config Context'),