Closes #10560: New global search (#10676)

* Initial work on new search backend

* Clean up search backends

* Return only the most relevant result per object

* Clear any pre-existing cached entries on cache()

* #6003: Implement global search functionality for custom field values

* Tweak field weights & document guidance

* Extend search() to accept a lookup type

* Move get_registry() out of SearchBackend

* Enforce object permissions when returning search results

* Add indexers for remaining models

* Avoid calling remove() on non-cacheable objects

* Use new search backend by default

* Extend search backend to filter by object type

* Clean up search view form

* Enable specifying lookup logic

* Add indexes for value field

* Remove object type selector from search bar

* Introduce SearchTable and enable HTMX for results

* Enable pagination

* Remove legacy search backend

* Cleanup

* Use a UUID for CachedValue primary key

* Refactoring search methods

* Define max search results limit

* Extend reindex command to support specifying particular models

* Add clear() and size to SearchBackend

* Optimize bulk caching performance

* Highlight matched portion of field value

* Performance improvements for reindexing

* Started on search tests

* Cleanup & docs

* Documentation updates

* Clean up SearchIndex

* Flatten search registry to register by app_label.model_name

* Clean up search backend classes

* Clean up RestrictedGenericForeignKey and RestrictedPrefetch

* Resolve migrations conflict

netbox/extras/api/serializers.py

@@ -92,8 +92,8 @@ class CustomFieldSerializer(ValidatedModelSerializer):
         model = CustomField
         fields = [
             'id', 'url', 'display', 'content_types', 'type', 'object_type', 'data_type', 'name', 'label', 'group_name',
-            'description', 'required', 'filter_logic', 'ui_visibility', 'default', 'weight', 'validation_minimum',
-            'validation_maximum', 'validation_regex', 'choices', 'created', 'last_updated',
+            'description', 'required', 'search_weight', 'filter_logic', 'ui_visibility', 'default', 'weight',
+            'validation_minimum', 'validation_maximum', 'validation_regex', 'choices', 'created', 'last_updated',
         ]
 
     def get_data_type(self, obj):

netbox/extras/filtersets.py

@@ -73,8 +73,8 @@ class CustomFieldFilterSet(BaseFilterSet):
     class Meta:
         model = CustomField
         fields = [
-            'id', 'content_types', 'name', 'group_name', 'required', 'filter_logic', 'ui_visibility', 'weight',
-            'description',
+            'id', 'content_types', 'name', 'group_name', 'required', 'search_weight', 'filter_logic', 'ui_visibility',
+            'weight', 'description',
         ]
 
     def search(self, queryset, name, value):

netbox/extras/forms/bulk_import.py

@@ -46,8 +46,8 @@ class CustomFieldCSVForm(CSVModelForm):
     class Meta:
         model = CustomField
         fields = (
-            'name', 'label', 'group_name', 'type', 'content_types', 'object_type', 'required', 'description', 'weight',
-            'filter_logic', 'default', 'choices', 'weight', 'validation_minimum', 'validation_maximum',
+            'name', 'label', 'group_name', 'type', 'content_types', 'object_type', 'required', 'description',
+            'search_weight', 'filter_logic', 'default', 'choices', 'weight', 'validation_minimum', 'validation_maximum',
             'validation_regex', 'ui_visibility',
         )
 

netbox/extras/forms/models.py

@@ -41,9 +41,9 @@ class CustomFieldForm(BootstrapMixin, forms.ModelForm):
 
     fieldsets = (
         ('Custom Field', (
-            'content_types', 'name', 'label', 'group_name', 'type', 'object_type', 'weight', 'required', 'description',
+            'content_types', 'name', 'label', 'group_name', 'type', 'object_type', 'required', 'description',
         )),
-        ('Behavior', ('filter_logic', 'ui_visibility')),
+        ('Behavior', ('search_weight', 'filter_logic', 'ui_visibility', 'weight')),
         ('Values', ('default', 'choices')),
         ('Validation', ('validation_minimum', 'validation_maximum', 'validation_regex')),
     )

netbox/extras/management/commands/reindex.py

@@ -0,0 +1,77 @@
+from django.contrib.contenttypes.models import ContentType
+from django.core.management.base import BaseCommand, CommandError
+
+from extras.registry import registry
+from netbox.search.backends import search_backend
+
+
+class Command(BaseCommand):
+    help = 'Reindex objects for search'
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            'args',
+            metavar='app_label[.ModelName]',
+            nargs='*',
+            help='One or more apps or models to reindex',
+        )
+
+    def _get_indexers(self, *model_names):
+        indexers = {}
+
+        # No models specified; pull in all registered indexers
+        if not model_names:
+            for idx in registry['search'].values():
+                indexers[idx.model] = idx
+
+        # Return only indexers for the specified models
+        else:
+            for label in model_names:
+                try:
+                    app_label, model_name = label.lower().split('.')
+                except ValueError:
+                    raise CommandError(
+                        f"Invalid model: {label}. Model names must be in the format <app_label>.<model_name>."
+                    )
+                try:
+                    idx = registry['search'][f'{app_label}.{model_name}']
+                    indexers[idx.model] = idx
+                except KeyError:
+                    raise CommandError(f"No indexer registered for {label}")
+
+        return indexers
+
+    def handle(self, *model_labels, **kwargs):
+
+        # Determine which models to reindex
+        indexers = self._get_indexers(*model_labels)
+        if not indexers:
+            raise CommandError("No indexers found!")
+        self.stdout.write(f'Reindexing {len(indexers)} models.')
+
+        # Clear all cached values for the specified models
+        self.stdout.write('Clearing cached values... ', ending='')
+        self.stdout.flush()
+        content_types = [
+            ContentType.objects.get_for_model(model) for model in indexers.keys()
+        ]
+        deleted_count = search_backend.clear(content_types)
+        self.stdout.write(f'{deleted_count} entries deleted.')
+
+        # Index models
+        self.stdout.write('Indexing models')
+        for model, idx in indexers.items():
+            app_label = model._meta.app_label
+            model_name = model._meta.model_name
+            self.stdout.write(f'  {app_label}.{model_name}... ', ending='')
+            self.stdout.flush()
+            i = search_backend.cache(model.objects.iterator(), remove_existing=False)
+            if i:
+                self.stdout.write(f'{i} entries cached.')
+            else:
+                self.stdout.write(f'None found.')
+
+        msg = f'Completed.'
+        if total_count := search_backend.size:
+            msg += f' Total entries: {total_count}'
+        self.stdout.write(msg, self.style.SUCCESS)

netbox/extras/migrations/0079_change_jobresult_order.py

@@ -1,17 +0,0 @@
-# Generated by Django 4.1.1 on 2022-10-09 18:37
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('extras', '0078_unique_constraints'),
-    ]
-
-    operations = [
-        migrations.AlterModelOptions(
-            name='jobresult',
-            options={'ordering': ['-created']},
-        ),
-    ]

netbox/extras/migrations/0079_jobresult_scheduled_time.py

@@ -1,12 +1,10 @@
-# Generated by Django 4.1.1 on 2022-10-16 09:52
-
 from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('extras', '0079_change_jobresult_order'),
+        ('extras', '0078_unique_constraints'),
     ]
 
     operations = [
@@ -15,4 +13,8 @@ class Migration(migrations.Migration):
             name='scheduled_time',
             field=models.DateTimeField(blank=True, null=True),
         ),
+        migrations.AlterModelOptions(
+            name='jobresult',
+            options={'ordering': ['-created']},
+        ),
     ]

netbox/extras/migrations/0080_search.py

@@ -0,0 +1,35 @@
+from django.db import migrations, models
+import django.db.models.deletion
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('contenttypes', '0002_remove_content_type_name'),
+        ('extras', '0079_jobresult_scheduled_time'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='customfield',
+            name='search_weight',
+            field=models.PositiveSmallIntegerField(default=1000),
+        ),
+        migrations.CreateModel(
+            name='CachedValue',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('timestamp', models.DateTimeField(auto_now_add=True)),
+                ('object_id', models.PositiveBigIntegerField()),
+                ('field', models.CharField(max_length=200)),
+                ('type', models.CharField(max_length=30)),
+                ('value', models.TextField(db_index=True)),
+                ('weight', models.PositiveSmallIntegerField(default=1000)),
+                ('object_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='contenttypes.contenttype')),
+            ],
+            options={
+                'ordering': ('weight', 'object_type', 'object_id'),
+            },
+        ),
+    ]

netbox/extras/models/__init__.py

@@ -2,9 +2,11 @@ from .change_logging import ObjectChange
 from .configcontexts import ConfigContext, ConfigContextModel
 from .customfields import CustomField
 from .models import *
+from .search import *
 from .tags import Tag, TaggedItem
 
 __all__ = (
+    'CachedValue',
     'ConfigContext',
     'ConfigContextModel',
     'ConfigRevision',

netbox/extras/models/customfields.py

@@ -16,6 +16,7 @@ from extras.choices import *
 from extras.utils import FeatureQuery
 from netbox.models import ChangeLoggedModel
 from netbox.models.features import CloningMixin, ExportTemplatesMixin, WebhooksMixin
+from netbox.search import FieldTypes
 from utilities import filters
 from utilities.forms import (
     CSVChoiceField, CSVMultipleChoiceField, DatePicker, DynamicModelChoiceField, DynamicModelMultipleChoiceField,
@@ -30,6 +31,15 @@ __all__ = (
     'CustomFieldManager',
 )
 
+SEARCH_TYPES = {
+    CustomFieldTypeChoices.TYPE_TEXT: FieldTypes.STRING,
+    CustomFieldTypeChoices.TYPE_LONGTEXT: FieldTypes.STRING,
+    CustomFieldTypeChoices.TYPE_INTEGER: FieldTypes.INTEGER,
+    CustomFieldTypeChoices.TYPE_DECIMAL: FieldTypes.FLOAT,
+    CustomFieldTypeChoices.TYPE_DATE: FieldTypes.STRING,
+    CustomFieldTypeChoices.TYPE_URL: FieldTypes.STRING,
+}
+
 
 class CustomFieldManager(models.Manager.from_queryset(RestrictedQuerySet)):
     use_in_migrations = True
@@ -94,6 +104,11 @@ class CustomField(CloningMixin, ExportTemplatesMixin, WebhooksMixin, ChangeLogge
         help_text='If true, this field is required when creating new objects '
                   'or editing an existing object.'
     )
+    search_weight = models.PositiveSmallIntegerField(
+        default=1000,
+        help_text='Weighting for search. Lower values are considered more important. '
+                  'Fields with a search weight of zero will be ignored.'
+    )
     filter_logic = models.CharField(
         max_length=50,
         choices=CustomFieldFilterLogicChoices,
@@ -109,6 +124,7 @@ class CustomField(CloningMixin, ExportTemplatesMixin, WebhooksMixin, ChangeLogge
     )
     weight = models.PositiveSmallIntegerField(
         default=100,
+        verbose_name='Display weight',
         help_text='Fields with higher weights appear lower in a form.'
     )
     validation_minimum = models.IntegerField(
@@ -148,8 +164,9 @@ class CustomField(CloningMixin, ExportTemplatesMixin, WebhooksMixin, ChangeLogge
     objects = CustomFieldManager()
 
     clone_fields = (
-        'content_types', 'type', 'object_type', 'group_name', 'description', 'required', 'filter_logic', 'default',
-        'weight', 'validation_minimum', 'validation_maximum', 'validation_regex', 'choices', 'ui_visibility',
+        'content_types', 'type', 'object_type', 'group_name', 'description', 'required', 'search_weight',
+        'filter_logic', 'default', 'weight', 'validation_minimum', 'validation_maximum', 'validation_regex', 'choices',
+        'ui_visibility',
     )
 
     class Meta:
@@ -167,6 +184,10 @@ class CustomField(CloningMixin, ExportTemplatesMixin, WebhooksMixin, ChangeLogge
         # Cache instance's original name so we can check later whether it has changed
         self._name = self.name
 
+    @property
+    def search_type(self):
+        return SEARCH_TYPES.get(self.type)
+
     def populate_initial_data(self, content_types):
         """
         Populate initial custom field data upon either a) the creation of a new CustomField, or

netbox/extras/models/search.py

@@ -0,0 +1,50 @@
+import uuid
+
+from django.contrib.contenttypes.models import ContentType
+from django.db import models
+
+from utilities.fields import RestrictedGenericForeignKey
+
+__all__ = (
+    'CachedValue',
+)
+
+
+class CachedValue(models.Model):
+    id = models.UUIDField(
+        primary_key=True,
+        default=uuid.uuid4,
+        editable=False
+    )
+    timestamp = models.DateTimeField(
+        auto_now_add=True,
+        editable=False
+    )
+    object_type = models.ForeignKey(
+        to=ContentType,
+        on_delete=models.CASCADE,
+        related_name='+'
+    )
+    object_id = models.PositiveBigIntegerField()
+    object = RestrictedGenericForeignKey(
+        ct_field='object_type',
+        fk_field='object_id'
+    )
+    field = models.CharField(
+        max_length=200
+    )
+    type = models.CharField(
+        max_length=30
+    )
+    value = models.TextField(
+        db_index=True
+    )
+    weight = models.PositiveSmallIntegerField(
+        default=1000
+    )
+
+    class Meta:
+        ordering = ('weight', 'object_type', 'object_id')
+
+    def __str__(self):
+        return f'{self.object_type} {self.object_id}: {self.field}={self.value}'

netbox/extras/plugins/__init__.py

@@ -75,7 +75,7 @@ class PluginConfig(AppConfig):
         try:
             search_indexes = import_string(f"{self.__module__}.{self.search_indexes}")
             for idx in search_indexes:
-                register_search()(idx)
+                register_search(idx)
         except ImportError:
             pass
 

netbox/extras/registry.py

@@ -29,5 +29,5 @@ registry['model_features'] = {
     feature: collections.defaultdict(set) for feature in EXTRAS_FEATURES
 }
 registry['denormalized_fields'] = collections.defaultdict(list)
-registry['search'] = collections.defaultdict(dict)
+registry['search'] = dict()
 registry['views'] = collections.defaultdict(dict)

netbox/extras/search.py

@@ -1,14 +1,11 @@
-import extras.filtersets
-import extras.tables
-from extras.models import JournalEntry
 from netbox.search import SearchIndex, register_search
+from . import models
 
 
-@register_search()
+@register_search
 class JournalEntryIndex(SearchIndex):
-    model = JournalEntry
-    queryset = JournalEntry.objects.prefetch_related('assigned_object', 'created_by')
-    filterset = extras.filtersets.JournalEntryFilterSet
-    table = extras.tables.JournalEntryTable
-    url = 'extras:journalentry_list'
+    model = models.JournalEntry
+    fields = (
+        ('comments', 5000),
+    )
     category = 'Journal'

netbox/extras/tables/tables.py

@@ -34,8 +34,8 @@ class CustomFieldTable(NetBoxTable):
     class Meta(NetBoxTable.Meta):
         model = CustomField
         fields = (
-            'pk', 'id', 'name', 'content_types', 'label', 'type', 'group_name', 'required', 'weight', 'default',
-            'description', 'filter_logic', 'ui_visibility', 'choices', 'created', 'last_updated',
+            'pk', 'id', 'name', 'content_types', 'label', 'type', 'group_name', 'required', 'default', 'description',
+            'search_weight', 'filter_logic', 'ui_visibility', 'weight', 'choices', 'created', 'last_updated',
         )
         default_columns = ('pk', 'name', 'content_types', 'label', 'group_name', 'type', 'required', 'description')
 

netbox/extras/tests/dummy_plugin/search.py

@@ -4,8 +4,9 @@ from .models import DummyModel
 
 class DummyModelIndex(SearchIndex):
     model = DummyModel
-    queryset = DummyModel.objects.all()
-    url = 'plugins:dummy_plugin:dummy_models'
+    fields = (
+        ('name', 100),
+    )
 
 
 indexes = (

netbox/extras/tests/test_customfields.py

@@ -292,6 +292,7 @@ class CustomFieldTest(TestCase):
         cf = CustomField.objects.create(
             name='object_field',
             type=CustomFieldTypeChoices.TYPE_OBJECT,
+            object_type=ContentType.objects.get_for_model(VLAN),
             required=False
         )
         cf.content_types.set([self.object_type])
@@ -323,6 +324,7 @@ class CustomFieldTest(TestCase):
         cf = CustomField.objects.create(
             name='object_field',
             type=CustomFieldTypeChoices.TYPE_MULTIOBJECT,
+            object_type=ContentType.objects.get_for_model(VLAN),
             required=False
         )
         cf.content_types.set([self.object_type])

netbox/extras/tests/test_views.py

@@ -32,6 +32,7 @@ class CustomFieldTestCase(ViewTestCases.PrimaryObjectViewTestCase):
             'label': 'Field X',
             'type': 'text',
             'content_types': [site_ct.pk],
+            'search_weight': 2000,
             'filter_logic': CustomFieldFilterLogicChoices.FILTER_EXACT,
             'default': None,
             'weight': 200,
@@ -40,11 +41,11 @@ class CustomFieldTestCase(ViewTestCases.PrimaryObjectViewTestCase):
         }
 
         cls.csv_data = (
-            'name,label,type,content_types,object_type,weight,filter_logic,choices,validation_minimum,validation_maximum,validation_regex,ui_visibility',
-            'field4,Field 4,text,dcim.site,,100,exact,,,,[a-z]{3},read-write',
-            'field5,Field 5,integer,dcim.site,,100,exact,,1,100,,read-write',
-            'field6,Field 6,select,dcim.site,,100,exact,"A,B,C",,,,read-write',
-            'field7,Field 7,object,dcim.site,dcim.region,100,exact,,,,,read-write',
+            'name,label,type,content_types,object_type,weight,search_weight,filter_logic,choices,validation_minimum,validation_maximum,validation_regex,ui_visibility',
+            'field4,Field 4,text,dcim.site,,100,1000,exact,,,,[a-z]{3},read-write',
+            'field5,Field 5,integer,dcim.site,,100,2000,exact,,1,100,,read-write',
+            'field6,Field 6,select,dcim.site,,100,3000,exact,"A,B,C",,,,read-write',
+            'field7,Field 7,object,dcim.site,dcim.region,100,4000,exact,,,,,read-write',
         )
 
         cls.bulk_edit_data = {