Add Support for Searching Processing Timelines

data/timesketch.conf

@@ -401,3 +401,10 @@ EXAMPLES_NL2Q = '/etc/timesketch/nl2q/examples_nl2q'
 
 # LLM event summarization configuration
 PROMPT_LLM_SUMMARIZATION = '/etc/timesketch/llm_summarize/prompt.txt'
+
+#-------------------------------------------------------------------------------
+# Timesketch UI Option
+
+# Get the search processing timelines setting.
+# If set to True, the search processing timelines options will be displayed in the UI.
+SEARCH_PROCESSING_TIMELINES = False

timesketch/api/v1/resources/aggregation.py

@@ -18,6 +18,7 @@
 
 from opensearchpy.exceptions import NotFoundError
 
+from flask import current_app
 from flask import jsonify
 from flask import request
 from flask import abort
@@ -463,10 +464,17 @@ def post(self, sketch_id: int):
                 "Not able to run aggregation on an archived sketch.",
             )
 
+        include_processing_timelines = form.include_processing_timelines.data
+        allowed_statuses = ["ready"]
+        if include_processing_timelines and current_app.config.get(
+            "SEARCH_PROCESSING_TIMELINES", False
+        ):
+            allowed_statuses.append("processing")
+
         sketch_indices = {
             t.searchindex.index_name
             for t in sketch.timelines
-            if t.get_status.status.lower() == "ready"
+            if t.get_status.status.lower() in allowed_statuses
         }
 
         aggregation_dsl = form.aggregation_dsl.data
@@ -490,7 +498,9 @@ def post(self, sketch_id: int):
                 aggregator_parameters = {}
 
             indices = aggregator_parameters.pop("index", sketch_indices)
-            indices, timeline_ids = lib_utils.get_validated_indices(indices, sketch)
+            indices, timeline_ids = lib_utils.get_validated_indices(
+                indices, sketch, form.include_processing_timelines.data
+            )
 
             if not (indices or timeline_ids):
                 abort(HTTP_STATUS_CODE_BAD_REQUEST, "No indices to aggregate on")

timesketch/api/v1/resources/event.py

@@ -27,10 +27,11 @@
 import numpy as np
 import pandas as pd
 
+from flask import current_app
 from flask import jsonify
 from flask import request
 from flask import abort
-from flask_restful import Resource
+from flask_restful import Resource, inputs
 from flask_restful import reqparse
 from flask_login import login_required
 from flask_login import current_user
@@ -249,6 +250,12 @@ def __init__(self):
             "searchindex_id", type=str, required=True, location="args"
         )
         self.parser.add_argument("event_id", type=str, required=True, location="args")
+        self.parser.add_argument(
+            "include_processing_timelines",
+            type=inputs.boolean,
+            required=False,
+            location="args",
+        )
 
     @login_required
     def get(self, sketch_id: int):
@@ -286,10 +293,18 @@ def get(self, sketch_id: int):
             )
 
         event_id = args.get("event_id")
+        include_processing_timelines = bool(
+            args.get("include_processing_timelines", False)
+        )
+        allowed_statuses = ["ready"]
+        if include_processing_timelines and current_app.config.get(
+            "SEARCH_PROCESSING_TIMELINES", False
+        ):
+            allowed_statuses.append("processing")
         indices = [
             t.searchindex.index_name
             for t in sketch.timelines
-            if t.get_status.status.lower() == "ready"
+            if t.get_status.status.lower() in allowed_statuses
         ]
 
         # Check if the requested searchindex is part of the sketch
@@ -827,10 +842,14 @@ def post(self, sketch_id: int):
         if _search_node_id:
             current_search_node = self._get_current_search_node(_search_node_id, sketch)
 
+        allowed_statuses = ["ready"]
+        if current_app.config.get("SEARCH_PROCESSING_TIMELINES", False):
+            allowed_statuses.append("processing")
+
         indices = [
             t.searchindex.index_name
             for t in sketch.timelines
-            if t.get_status.status.lower() == "ready"
+            if t.get_status.status.lower() in allowed_statuses
         ]
         annotation_type = form.annotation_type.data
         events = form.events.raw_data

timesketch/api/v1/resources/explore.py

@@ -22,6 +22,7 @@
 
 from flask import abort
 from flask import jsonify
+from flask import current_app
 from flask import request
 from flask import send_file
 from flask_restful import Resource
@@ -143,6 +144,12 @@ def post(self, sketch_id: int):
         parent = request.json.get("parent", None)
         incognito = request.json.get("incognito", False)
 
+        include_processing_timelines = False
+        if current_app.config.get("SEARCH_PROCESSING_TIMELINES", False):
+            include_processing_timelines = request.json.get(
+                "include_processing_timelines", False
+            )
+
         return_field_string = form.fields.data
         if return_field_string:
             return_fields = [x.strip() for x in return_field_string.split(",")]
@@ -163,7 +170,9 @@ def post(self, sketch_id: int):
 
         # Make sure that the indices in the filter are part of the sketch.
         # This will also remove any deleted timeline from the search result.
-        indices, timeline_ids = get_validated_indices(indices, sketch)
+        indices, timeline_ids = get_validated_indices(
+            indices, sketch, include_processing_timelines
+        )
 
         # Remove indices that don't exist from search.
         indices = utils.validate_indices(indices, self.datastore)

timesketch/api/v1/resources/settings.py

@@ -62,4 +62,11 @@ def get(self):
             result["llm_config_warning"] = warning_message
             logger.warning(warning_message)
 
+        # Get the search processing timelines setting, default is False if not set.
+        # if set to True, the search processing timelines will be displayed in the UI.
+        search_processing_timelines = current_app.config.get(
+            "SEARCH_PROCESSING_TIMELINES", False
+        )
+        result["SEARCH_PROCESSING_TIMELINES"] = search_processing_timelines
+
         return jsonify(result)

timesketch/api/v1/resources/user.py

@@ -16,6 +16,7 @@
 import logging
 
 from flask import abort
+from flask import current_app
 from flask import jsonify
 from flask import request
 from flask_restful import Resource
@@ -170,6 +171,14 @@ def get(self):
         """
         profile = UserProfile.get_or_create(user=current_user)
         settings = json.loads(profile.settings)
+
+        # If the value of SEARCH_PROCESSING_TIMELINES changes to false while the user
+        # had the option enabled, it remains enabled without functioning.
+        # Therefore, if SEARCH_PROCESSING_TIMELINES changes to false, we disable the
+        # showProcessingTimelineEvents option in the user's settings for display
+        # consistency.
+        if not current_app.config.get("SEARCH_PROCESSING_TIMELINES", False):
+            settings["showProcessingTimelineEvents"] = False
         schema = {"objects": [settings], "meta": {}}
         return jsonify(schema)
 

timesketch/api/v1/resources_test.py

@@ -1371,7 +1371,11 @@ def test_system_settings_resource(self):
 
         self.login()
         response = self.client.get(self.resource_url)
-        expected_response = {"DFIQ_ENABLED": False, "LLM_PROVIDER": "test"}
+        expected_response = {
+            "DFIQ_ENABLED": False,
+            "LLM_PROVIDER": "test",
+            "SEARCH_PROCESSING_TIMELINES": False,
+        }
         self.assertEqual(response.json, expected_response)
 
 

timesketch/frontend-ng/src/components/Analyzer/TimelineChip.vue

@@ -38,6 +38,9 @@ limitations under the License.
             </template>
             <span>{{ timeline.name }}</span>
           </v-tooltip>
+          <span v-if="timeline.status[0].status === 'processing'" class="ml-3 mr-3">
+            <v-progress-circular small indeterminate color="grey" :size="17" :width="2"></v-progress-circular>
+          </span>
         </div>
       </v-chip>
     </template>

timesketch/frontend-ng/src/components/Analyzer/TimelineSearch.vue

@@ -63,7 +63,7 @@ export default {
   components:{
     TsAnalyzerTimelineChip,
   },
-  props: ['analyzerTimelineId', 'componentName'],
+  props: ['analyzerTimelineId', 'componentName', 'includeProcessingTimelines'],
   data() {
     return {
       selectedTimelines: [],
@@ -75,8 +75,9 @@ export default {
     },
     allReadyTimelines() {
       // Sort alphabetically based on timeline name.
-      const timelines = this.sketch.timelines.filter(
-        tl => tl.status[0].status === 'ready'
+      const timelines = this.sketch.timelines.filter(tl =>
+        tl.status[0].status === 'ready' ||
+        (this.includeProcessingTimelines && tl.status[0].status === 'processing')
       );
       timelines.sort((a, b) => a.name.localeCompare(b.name))
       return timelines;

timesketch/frontend-ng/src/components/Explore/AggregateDialog.vue

@@ -313,6 +313,9 @@ export default {
     }
   },
   computed: {
+    settings() {
+      return this.$store.state.settings
+    },
     sketch() {
       return this.$store.state.sketch
     },
@@ -556,7 +559,8 @@ export default {
         aggregator_parameters: {
           field: this.eventKey,
           field_query_string: this.eventValue
-        }
+        },
+        include_processing_timelines: !!this.settings.showProcessingTimelineEvents,
       }).then((response) => {
         this.stats = response.data.objects[0].field_summary.buckets[0]
         this.statsReady = true
@@ -570,7 +574,8 @@ export default {
         aggregator_parameters: {
           field: this.eventKey,
           date_interval: this.selectedDistributionInterval
-        }
+        },
+        include_processing_timelines: !!this.settings.showProcessingTimelineEvents,
       }).then((response) => {
         this.eventDistributionData = response.data.objects[0].datefield_summary.buckets[0]
         this.eventDistributionReady = true
@@ -624,7 +629,8 @@ export default {
           supported_intervals: supportedIntervals,
           start_time: startTime.toISOString().slice(0, -1),
           end_time: endTime.toISOString().slice(0, -1),
-        }
+        },
+        include_processing_timelines: !!this.settings.showProcessingTimelineEvents,
       }).then((response) => {
         this.data = response.data.objects[0].date_histogram.buckets[0]
         this.recentHistogramSeries = [{

timesketch/frontend-ng/src/components/Explore/EventActionMenu.vue

@@ -59,13 +59,16 @@ export default {
     sketch() {
       return this.$store.state.sketch
     },
+    settings() {
+      return this.$store.state.settings
+    },
   },
   methods: {
     showContextWindow() {
       EventBus.$emit('showContextWindow', this.event)
     },
     copyEventAsJSON() {
-      ApiClient.getEvent(this.sketch.id, this.event._index, this.event._id)
+      ApiClient.getEvent(this.sketch.id, this.event._index, this.event._id, !!this.settings.showProcessingTimelineEvents)
         .then((response) => {
           let fullEvent = response.data.objects
           let eventJSON = JSON.stringify(fullEvent, null, 3)

timesketch/frontend-ng/src/components/Explore/EventDetail.vue

@@ -246,12 +246,16 @@ export default {
     contextLinkConf() {
       return this.$store.state.contextLinkConf
     },
+    settings() {
+      return this.$store.state.settings
+    },
   },
   methods: {
     getEvent: function () {
       let searchindexId = this.event._index
       let eventId = this.event._id
-      ApiClient.getEvent(this.sketch.id, searchindexId, eventId)
+      let includeProcessingTimelines = !!this.settings.showProcessingTimelineEvents
+      ApiClient.getEvent(this.sketch.id, searchindexId, eventId, includeProcessingTimelines)
         .then((response) => {
           this.fullEvent = response.data.objects
           this.comments = response.data.meta.comments