fix slow query (#477)

fqrious · web-flow · commit c26f3d4f364c · 2026-04-03T15:58:27.000+01:00
* use managed `is_dupe` field to filter instead of  window_agg for deduplication

* fix failing tests

* add sort param on every index

* add more indexes
diff --git a/obstracts/server/migrations/0021_objectvalue_is_dupe.py b/obstracts/server/migrations/0021_objectvalue_is_dupe.py
@@ -0,0 +1,23 @@
+# Generated by Django 5.2.11 on 2026-04-02 14:16
+
+from django.db import migrations, models
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('obstracts', '0020_objectvalue_obstracts_s_stix_id_modified_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='objectvalue',
+            name='is_dupe',
+            field=models.BooleanField(db_index=True, default=False),
+        ),
+
+        migrations.AddField(
+            model_name='objectvalue',
+            name='value',
+            field=models.CharField(blank=True, max_length=64, null=True),
+        ),
+    ]
diff --git a/obstracts/server/migrations/0022_remove_objectvalue_obstracts_s_stix_id_type_idx_and_more.py b/obstracts/server/migrations/0022_remove_objectvalue_obstracts_s_stix_id_type_idx_and_more.py
@@ -0,0 +1,105 @@
+# Generated by Django 5.2.11 on 2026-04-03 13:46
+
+import django.contrib.postgres.fields
+import django.db.models.fields.json
+import django.db.models.functions.text
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('obstracts', '0021_objectvalue_is_dupe'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='objectvalue',
+            name='value',
+        ),
+        migrations.RemoveIndex(
+            model_name='objectvalue',
+            name='obstracts_s_stix_id_type_idx',
+        ),
+        migrations.RemoveIndex(
+            model_name='objectvalue',
+            name='obstracts_s_stix_id_modified',
+        ),
+        migrations.RemoveIndex(
+            model_name='objectvalue',
+            name='obstracts_ov_type_stix_idx',
+        ),
+        migrations.RemoveIndex(
+            model_name='objectvalue',
+            name='obstracts_ov_kbase_stix_idx',
+        ),
+        migrations.RemoveIndex(
+            model_name='objectvalue',
+            name='obstracts_ov_first_value_idx',
+        ),
+        migrations.AddField(
+            model_name='objectvalue',
+            name='values_concat',
+            field=models.GeneratedField(db_persist=True, expression=models.Func(models.F('values'), function='jsonb_values_concat'), null=True, output_field=models.TextField()),
+        ),
+        migrations.AddField(
+            model_name='objectvalue',
+            name='values_list',
+            field=models.GeneratedField(db_persist=True, expression=models.Func(models.F('values'), function='jsonb_values_list'), null=True, output_field=django.contrib.postgres.fields.ArrayField(base_field=models.TextField(), size=None)),
+        ),
+        migrations.AlterField(
+            model_name='objectvalue',
+            name='is_dupe',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AlterField(
+            model_name='objectvalue',
+            name='knowledgebase',
+            field=models.CharField(blank=True, max_length=64, null=True),
+        ),
+        migrations.AlterField(
+            model_name='objectvalue',
+            name='type',
+            field=models.CharField(max_length=256),
+        ),
+        migrations.AddIndex(
+            model_name='objectvalue',
+            index=models.Index(condition=models.Q(('is_dupe', False)), fields=['type', 'stix_id'], name='obstracts_ov_type_stix_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='objectvalue',
+            index=models.Index(condition=models.Q(('is_dupe', False)), fields=['created', 'knowledgebase'], name='obstracts_ov_kbase_c_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='objectvalue',
+            index=models.Index(condition=models.Q(('is_dupe', False)), fields=['modified', 'knowledgebase'], name='obstracts_ov_kbase_m_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='objectvalue',
+            index=models.Index(condition=models.Q(('is_dupe', False)), fields=['created', 'type'], name='obstracts_ov_created_type_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='objectvalue',
+            index=models.Index(condition=models.Q(('is_dupe', False)), fields=['modified', 'type'], name='obstracts_ov_modified_type_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='objectvalue',
+            index=models.Index(django.db.models.fields.json.KeyTextTransform('kb_type', 'values'), models.F('type'), condition=models.Q(('is_dupe', False)), name='obstracts_ov_kb_type_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='objectvalue',
+            index=models.Index(models.F('created'), django.db.models.functions.text.Upper(django.db.models.fields.json.KeyTextTransform('kb_id', 'values')), models.F('type'), condition=models.Q(('is_dupe', False)), name='obstracts_ov_kb_id_cidx'),
+        ),
+        migrations.AddIndex(
+            model_name='objectvalue',
+            index=models.Index(models.F('modified'), django.db.models.functions.text.Upper(django.db.models.fields.json.KeyTextTransform('kb_id', 'values')), models.F('type'), condition=models.Q(('is_dupe', False)), name='obstracts_ov_kb_id_midx'),
+        ),
+        migrations.AddIndex(
+            model_name='objectvalue',
+            index=models.Index(models.F('values_concat'), models.F('type'), condition=models.Q(('is_dupe', False)), name='obstracts_ov_values_concat_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='objectvalue',
+            index=models.Index(models.F('values_concat'), models.F('knowledgebase'), condition=models.Q(('is_dupe', False)), name='obstracts_ov_values_c_kbidx'),
+        ),
+    ]
diff --git a/obstracts/server/migrations/0023_add_index_on_values_listpy b/obstracts/server/migrations/0023_add_index_on_values_listpy
@@ -0,0 +1,31 @@
+# Generated by Django 5.2.11 on 2026-04-02 14:16
+
+from django.db import migrations, models
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('obstracts', '0022_remove_objectvalue_obstracts_s_stix_id_type_idx_and_more'),
+    ]
+
+    operations = [
+        migrations.RunSQL(
+            """
+            DROP INDEX IF EXISTS ctx_values_list_idx;
+            CREATE INDEX ctx_values_list_idx
+            ON obstracts_objectvalue
+            USING gin (values_list array_ops, type gin_trgm_ops) WHERE NOT is_dupe;
+            """,
+            reverse_sql=migrations.RunSQL.noop,
+        ),
+
+        migrations.RunSQL(
+            sql="""
+            DROP INDEX IF EXISTS ctx_values_concat_idx;
+            CREATE INDEX ctx_values_concat_idx
+            ON obstracts_objectvalue
+            USING gin (values_concat gin_trgm_ops, type gin_trgm_ops) WHERE NOT is_dupe;
+            """,
+            reverse_sql=migrations.RunSQL.noop,
+        ),
+    ]
diff --git a/obstracts/server/models.py b/obstracts/server/models.py
@@ -6,6 +6,8 @@
 import typing
 from django.conf import settings
 from django.db import models
+from django.db.models.fields.json import KeyTextTransform
+from django.db.models.functions import Upper
 from django.utils.text import slugify
 from pgvector.django import CosineDistance
 import txt2stix, txt2stix.extractions
@@ -29,6 +31,7 @@
 from obstracts.classifier.models import Cluster, DocumentEmbedding
 from obstracts.classifier.tasks import create_embedding_text, compute_embedding_for_document
 from obstracts.server.values.filters import DictFirstValue
+from django.contrib.postgres import indexes as pg_indexes
 
 # Create your models here.
 if typing.TYPE_CHECKING:
@@ -380,27 +383,43 @@ class ObjectValue(models.Model):
     Stores extracted values from STIX objects for efficient querying and filtering.
     """
     stix_id = models.CharField(max_length=256, db_index=True)
-    type = models.CharField(max_length=256, db_index=True)
-    knowledgebase = models.CharField(max_length=64, null=True, blank=True, db_index=True)
+    type = models.CharField(max_length=256)
+    knowledgebase = models.CharField(max_length=64, null=True, blank=True)
     values = models.JSONField()
     file = models.ForeignKey(File, on_delete=models.CASCADE, related_name='object_values')
     created = models.DateTimeField(default=None, null=True)
     modified = models.DateTimeField(default=None, null=True)
+    is_dupe = models.BooleanField(default=False)
+    values_concat = models.GeneratedField(
+        expression=models.Func(models.F("values"), function="jsonb_values_concat"),
+        output_field=models.TextField(),
+        db_persist=True,
+        null=True, blank=True,
+    )
+    values_list = models.GeneratedField(
+        expression=models.Func(models.F("values"), function="jsonb_values_list"),
+        output_field=ArrayField(base_field=models.TextField()),
+        db_persist=True, null=True, blank=True,
+    )
 
     class Meta:
         indexes = [
-            models.Index(fields=['stix_id', 'type'], name='obstracts_s_stix_id_type_idx'),
-            models.Index(fields=['stix_id', 'modified'], name='obstracts_s_stix_id_modified'),
-            models.Index(fields=['type', 'stix_id'], name='obstracts_ov_type_stix_idx'),
-            models.Index(fields=['knowledgebase', 'stix_id'], name='obstracts_ov_kbase_stix_idx'),
-            models.Index(DictFirstValue('values'), name='obstracts_ov_first_value_idx'),
+            models.Index(fields=['type', 'stix_id'], name='obstracts_ov_type_stix_idx', condition=models.Q(is_dupe=False)),
+            models.Index(fields=['created', 'knowledgebase'], name='obstracts_ov_kbase_c_idx', condition=models.Q(is_dupe=False)),
+            models.Index(fields=['modified', 'knowledgebase'], name='obstracts_ov_kbase_m_idx', condition=models.Q(is_dupe=False)),
+            models.Index(fields=['created', 'type'], name='obstracts_ov_created_type_idx', condition=models.Q(is_dupe=False)),
+            models.Index(fields=['modified', 'type'], name='obstracts_ov_modified_type_idx', condition=models.Q(is_dupe=False)),
+            models.Index(KeyTextTransform('kb_type', 'values'), 'type', name='obstracts_ov_kb_type_idx', condition=models.Q(is_dupe=False)),
+            models.Index('created', Upper(KeyTextTransform('kb_id', 'values')), 'type', name='obstracts_ov_kb_id_cidx', condition=models.Q(is_dupe=False)),
+            models.Index('modified', Upper(KeyTextTransform('kb_id', 'values')), 'type', name='obstracts_ov_kb_id_midx', condition=models.Q(is_dupe=False)),
+            models.Index('values_concat', 'type', name='obstracts_ov_values_concat_idx', condition=models.Q(is_dupe=False)),
+            models.Index('values_concat', 'knowledgebase', name='obstracts_ov_values_c_kbidx', condition=models.Q(is_dupe=False)),
         ]
         unique_together = [['stix_id', 'file']]
 
     def __str__(self):
         return f'ObjectValue(stix_id={self.stix_id}, knowledgebase={self.knowledgebase})'
 
-
 class Job(models.Model):
     id = models.UUIDField(primary_key=True, editable=False)
     history4feed_job = models.OneToOneField(
diff --git a/obstracts/server/values/values.py b/obstracts/server/values/values.py
@@ -289,19 +289,26 @@ def process_uploaded_objects_hook(instance, collection_name, objects, **kwargs):
         object_values_to_create.append(
             ObjectValue(
                 file_id=post_uuid,
-                **metadata
+                **metadata,
+                is_dupe=False,  # will be updated later in a deduplication step
             )
         )
 
     # Bulk create with ignore_conflicts to handle duplicates
     if object_values_to_create:
-        created_count = len(
-            ObjectValue.objects.bulk_create(
-                object_values_to_create, ignore_conflicts=True
-            )
+        created = ObjectValue.objects.bulk_create(
+            object_values_to_create, ignore_conflicts=True
+        )
+        new_dupes = ObjectValue.objects.filter(
+            stix_id__in=[obj.stix_id for obj in created],
+            is_dupe=False,
+        ).exclude(
+            file_id__in=[obj.file_id for obj in created],
         )
+        new_dupes.update(is_dupe=True)
         logging.info(
-            f"Created {created_count} ObjectValue records for {len(object_values_to_create)} objects"
+            f"Created {len(created)} ObjectValue records for {len(object_values_to_create)} objects"
         )
+        logging.info(f"Marked {new_dupes.count()} ObjectValue records as duplicates")
     else:
         logging.info("No ObjectValue records to create")
diff --git a/obstracts/server/values/views.py b/obstracts/server/values/views.py
@@ -83,9 +83,9 @@ def filter_value(self, queryset, name, value):
         value_exact = self.data.get("value_exact", "false").lower() == "true"
 
         if value_exact:
-            return queryset.filter(values__jsonb_vexact=value)
+            return queryset.filter(values_list__contains=[value.lower()])
         else:
-            return queryset.filter(values__jsonb_vcontains=value)
+            return queryset.filter(values_concat__icontains=value.lower())
 
     def filter_noop(self, queryset, name, value):
         """
@@ -109,9 +109,12 @@ class BaseObjectValueView(mixins.ListModelMixin, viewsets.GenericViewSet):
     queryset = ObjectValue.objects.all()
     serializer_class = ObjectValueSerializer
     pagination_class = Pagination("values")
-    filter_backends = [DjangoFilterBackend, Ordering]
+    filter_backends = [
+        DjangoFilterBackend,
+        Ordering
+    ]
     filterset_class = ObjectValueFilterSet
-    ordering_fields = ["stix_id", "type", "knowledgebase", "value"]
+    ordering_fields = ["value"]
     ordering = "stix_id_descending"
     openapi_tags = ["Object Values"]
 
@@ -126,21 +129,12 @@ def get_queryset(self):
             queryset = queryset.filter(type__in=self.allowed_types)
 
 
-        from django.db.models import F, Window
-        from django.db.models.functions import RowNumber
-        # Aggregate all post_ids for each unique stix_id
-        queryset = queryset.annotate(
-            rn=Window(
-                expression=RowNumber(),
-                partition_by=[F("stix_id")],
-                order_by=F("stix_id").desc(),
-            )
-        ).filter(
-            rn=1
-        ).annotate(
-            value=DictFirstValue(F("values")),
-        )
 
+        from django.db.models import F
+        queryset = queryset.alias(value=F('values_concat'))
+        queryset = queryset.filter(
+            is_dupe=False
+        )
         return queryset
 
 
@@ -181,7 +175,7 @@ class SCOValueView(BaseObjectValueView):
     """View for STIX Cyber Observable Objects (SCOs) only."""
 
     allowed_types = list(sco_value_map.keys())
-    ordering_fields = ["value", "stix_id", "type"]
+    ordering_fields = ["value"]
     ordering = "value_ascending"
 
     class filterset_class(ObjectValueFilterSet):
@@ -231,7 +225,8 @@ class SDOValueView(BaseObjectValueView):
     """View for STIX Domain Objects (SDOs) only."""
 
     allowed_types = list(sdo_value_map.keys())
-    ordering_fields = ["stix_id", "type", "knowledgebase", "value", "created", "modified"]
+    ordering_fields = ["value", "created", "modified"]
+    ordering = "modified_descending"
 
     class filterset_class(ObjectValueFilterSet):
         knowledgebases = ChoiceCSVFilter(
diff --git a/tests/src/views/test_values_view.py b/tests/src/views/test_values_view.py
