diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 3600eae0688..bbe35ebc7dc 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -9,6 +9,7 @@ This is a **Java agent** that extends OpenTelemetry Java Agent to provide Azure - **Agent Entry Point**: `Agent.java` - wraps OpenTelemetry Agent with Application Insights-specific initialization - **Agent Bootstrap**: Minimal classes loaded into bootstrap classloader for early initialization - **Agent Tooling**: Main Application Insights logic (configuration, exporters, processors) isolated in agent classloader +- **Profiler Subsystem**: JFR-based profiling with threshold triggers (CPU, memory, request), manual triggers (file touch, JMX MBean), and global cooldown - **Instrumentation Modules**: C Functions, ASP.NET Core interop, etc. - **Classic SDK**: Legacy 2.x SDK maintained for compatibility diff --git a/CHANGELOG.md b/CHANGELOG.md index a7f428cfab9..2efaf228d52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # CHANGELOG +## Unreleased + +### Enhancements + +* Add manual profile triggering via file touch and JMX MBean +* Add global cooldown (`globalCooldownSeconds`) to prevent rapid successive profiles from different + trigger sources +* Add `manualTriggeredSettings` configuration for controlling the JFR profile used during manual + recordings + ## Version 3.7.8 GA (03/24/2026) * Update to OpenTelemetry Java Instrumentation 2.26.1 diff --git a/agent/agent-profiler/agent-alerting-api/src/main/java/com/microsoft/applicationinsights/alerting/config/AlertingProfileFileTriggerConfiguration.java b/agent/agent-profiler/agent-alerting-api/src/main/java/com/microsoft/applicationinsights/alerting/config/AlertingProfileFileTriggerConfiguration.java new file mode 100644 index 00000000000..2d448a75cda --- /dev/null +++ b/agent/agent-profiler/agent-alerting-api/src/main/java/com/microsoft/applicationinsights/alerting/config/AlertingProfileFileTriggerConfiguration.java @@ -0,0 +1,97 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.microsoft.applicationinsights.alerting.config; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import java.io.File; + +/** + * Configuration for the file-based manual profile trigger. + * + *

When enabled, the alerting subsystem periodically checks for the existence of a trigger file. + * If the file exists and was recently modified (within {@link #MANUAL_TRIGGER_FILE_MAX_AGE_MS}), it + * is deleted and a manual profile recording is initiated. + * + *

This provides an operator-friendly mechanism for triggering on-demand profiles without + * requiring JMX access – simply {@code touch} the trigger file from a shell or orchestration tool. + */ +public class AlertingProfileFileTriggerConfiguration { + private static final String DEFAULT_TRIGGER_FILENAME = + "applicationinsights-agent-profile-trigger"; + + /** + * Maximum age (in milliseconds) of the trigger file for it to be considered valid. Files older + * than this threshold are ignored to prevent stale trigger files from initiating unexpected + * recordings (e.g., after a restart). + */ + public static final long MANUAL_TRIGGER_FILE_MAX_AGE_MS = 60_000; // 1 minute + + // Whether the file-based manual trigger is enabled. + private final boolean enabled; + + // Path to the file that triggers a manual profile when created/touched. + // If relative, it is resolved against the agent's temp directory. + private final File filePath; + + /** The default duration (in seconds) used for profiles triggered via this file mechanism. */ + private final int defaultProfileDurationSeconds; + + private AlertingProfileFileTriggerConfiguration( + boolean enabled, File filePath, int defaultProfileDurationSeconds) { + this.enabled = enabled; + this.filePath = filePath; + this.defaultProfileDurationSeconds = defaultProfileDurationSeconds; + } + + /** + * Creates a file trigger configuration, resolving relative paths against the provided temp + * directory. + * + * @param enabled whether file-based triggering is active + * @param filePath path to the trigger file (absolute or relative to {@code tempDir}) + * @param defaultProfileDurationSeconds duration in seconds for the profile if no override is + * configured in the collection plan + * @param tempDir base directory used to resolve relative file paths + * @return a fully resolved configuration instance + */ + @SuppressFBWarnings( + value = "SECPTI", + justification = "File path is set by trusted user configuration (applicationinsights.json)") + public static AlertingProfileFileTriggerConfiguration create( + boolean enabled, String filePath, int defaultProfileDurationSeconds, File tempDir) { + String resolvedFilePath = filePath; + if (resolvedFilePath == null || resolvedFilePath.trim().isEmpty()) { + resolvedFilePath = DEFAULT_TRIGGER_FILENAME; + } + + File manualTriggerFile = new File(resolvedFilePath); + if (!manualTriggerFile.isAbsolute() && tempDir != null) { + manualTriggerFile = new File(tempDir, resolvedFilePath); + } + + return new AlertingProfileFileTriggerConfiguration( + enabled, manualTriggerFile, defaultProfileDurationSeconds); + } + + /** Creates a disabled configuration suitable for tests. */ + public static AlertingProfileFileTriggerConfiguration createDefault() { + return new AlertingProfileFileTriggerConfiguration( + false, new File(DEFAULT_TRIGGER_FILENAME), 120); + } + + /** Returns the default profile duration in seconds for file-triggered recordings. */ + public int getDefaultProfileDurationSeconds() { + return defaultProfileDurationSeconds; + } + + /** Returns the resolved path of the trigger file. */ + public File getFilePath() { + return filePath; + } + + /** Returns whether the file-based manual trigger is enabled. */ + public boolean isEnabled() { + return enabled; + } +} diff --git a/agent/agent-profiler/agent-alerting/src/main/java/com/microsoft/applicationinsights/alerting/AlertingSubsystem.java b/agent/agent-profiler/agent-alerting/src/main/java/com/microsoft/applicationinsights/alerting/AlertingSubsystem.java index f45719c797b..f8508defb72 100644 --- a/agent/agent-profiler/agent-alerting/src/main/java/com/microsoft/applicationinsights/alerting/AlertingSubsystem.java +++ b/agent/agent-profiler/agent-alerting/src/main/java/com/microsoft/applicationinsights/alerting/AlertingSubsystem.java @@ -11,9 +11,11 @@ import com.microsoft.applicationinsights.alerting.config.AlertConfiguration; import com.microsoft.applicationinsights.alerting.config.AlertMetricType; import com.microsoft.applicationinsights.alerting.config.AlertingConfiguration; +import com.microsoft.applicationinsights.alerting.config.AlertingProfileFileTriggerConfiguration; import com.microsoft.applicationinsights.alerting.config.CollectionPlanConfiguration; import com.microsoft.applicationinsights.alerting.config.CollectionPlanConfiguration.EngineMode; import com.microsoft.applicationinsights.alerting.config.DefaultConfiguration; +import java.io.File; import java.time.Instant; import java.util.ArrayList; import java.util.HashSet; @@ -33,7 +35,6 @@ public class AlertingSubsystem { private static final Logger logger = LoggerFactory.getLogger(AlertingSubsystem.class); - // Downstream observer of alerts produced by the alerting system private final Consumer alertHandler; @@ -46,25 +47,40 @@ public class AlertingSubsystem { // Current configuration of the alerting subsystem private AlertingConfiguration alertConfig; - private boolean enableRequestTriggerUpdates; + /** Configuration controlling the file-based manual profile trigger. */ + private final AlertingProfileFileTriggerConfiguration alertingProfileFileTriggerConfiguration; - protected AlertingSubsystem(Consumer alertHandler) { - this(alertHandler, TimeSource.DEFAULT, false); - } + private boolean enableRequestTriggerUpdates; protected AlertingSubsystem( Consumer alertHandler, TimeSource timeSource, - boolean enableRequestTriggerUpdates) { + boolean enableRequestTriggerUpdates, + AlertingProfileFileTriggerConfiguration alertingProfileFileTriggerConfiguration) { this.alertHandler = alertHandler; this.alertPipelines = new AlertPipelines(alertHandler); this.timeSource = timeSource; this.enableRequestTriggerUpdates = enableRequestTriggerUpdates; + this.alertingProfileFileTriggerConfiguration = alertingProfileFileTriggerConfiguration; } + /** + * Creates and initializes an {@link AlertingSubsystem} with an initially-disabled configuration. + * + * @param alertHandler downstream consumer that handles generated alert breaches + * @param timeSource time source used for alert evaluation windows + * @param alertingProfileFileTriggerConfiguration configuration for the file-based manual trigger + * @return a fully initialized alerting subsystem ready to receive configuration updates + */ public static AlertingSubsystem create( - Consumer alertHandler, TimeSource timeSource) { - AlertingSubsystem alertingSubsystem = new AlertingSubsystem(alertHandler, timeSource, true); + Consumer alertHandler, + TimeSource timeSource, + AlertingProfileFileTriggerConfiguration alertingProfileFileTriggerConfiguration) { + + AlertingSubsystem alertingSubsystem = + new AlertingSubsystem( + alertHandler, timeSource, true, alertingProfileFileTriggerConfiguration); + // init with disabled config alertingSubsystem.initialize( AlertingConfiguration.create( @@ -145,14 +161,23 @@ private void updateRequestPipelineConfig( } } - /** Determine if a manual alert has been requested. */ + /** + * Determine if a manual alert has been requested via any supported mechanism. Currently evaluates + * both the server-side collection plan and the local file-based trigger. + */ private void evaluateManualTrigger(AlertingConfiguration alertConfig) { + evaluateCollectionPlanTrigger(alertConfig); + evaluateFileTrigger(alertConfig); + } + + /** Check if the collection plan configuration requests a manual profile. */ + private void evaluateCollectionPlanTrigger(AlertingConfiguration alertConfig) { CollectionPlanConfiguration config = alertConfig.getCollectionPlanConfiguration(); boolean shouldTrigger = config.isSingle() && config.getMode() == EngineMode.immediate - && Instant.now().isBefore(config.getExpiration()) + && timeSource.getNow().isBefore(config.getExpiration()) && !manualTriggersExecuted.contains(config.getSettingsMoniker()); if (shouldTrigger) { @@ -176,6 +201,78 @@ private void evaluateManualTrigger(AlertingConfiguration alertConfig) { } } + /** + * Check if a trigger file is present on the local file system and was recently modified. If so, + * delete the file and trigger a manual profile. The global cooldown in Profiler prevents + * overlapping profiles. + */ + public void evaluateFileTrigger() { + if (!alertingProfileFileTriggerConfiguration.isEnabled()) { + return; + } + + if (alertConfig == null) { + return; + } + + evaluateFileTrigger(alertConfig); + } + + private void evaluateFileTrigger(AlertingConfiguration currentConfig) { + if (!alertingProfileFileTriggerConfiguration.isEnabled()) { + return; + } + + if (currentConfig == null) { + return; + } + + File manualTriggerFile = alertingProfileFileTriggerConfiguration.getFilePath(); + if (manualTriggerFile == null || !manualTriggerFile.exists()) { + return; + } + + long lastModified = manualTriggerFile.lastModified(); + long age = timeSource.getNow().toEpochMilli() - lastModified; + + if (age < 0 || age > AlertingProfileFileTriggerConfiguration.MANUAL_TRIGGER_FILE_MAX_AGE_MS) { + return; + } + + // Delete the trigger file to prevent re-triggering + if (!manualTriggerFile.delete()) { + logger.warn( + "Failed to delete manual profile trigger file: {}", manualTriggerFile.getAbsolutePath()); + return; + } + + logger.info("Manual profile trigger file detected, initiating profile recording"); + + // Use the collection plan's duration if configured, otherwise fall back to the + // file trigger's default duration setting. + CollectionPlanConfiguration collectionPlan = currentConfig.getCollectionPlanConfiguration(); + int durationSeconds = collectionPlan.getImmediateProfilingDurationSeconds(); + if (durationSeconds <= 0) { + durationSeconds = alertingProfileFileTriggerConfiguration.getDefaultProfileDurationSeconds(); + } + + AlertBreach alertBreach = + AlertBreach.builder() + .setType(AlertMetricType.MANUAL) + .setAlertValue(0.0) + .setAlertConfiguration( + AlertConfiguration.builder() + .setType(AlertMetricType.MANUAL) + .setEnabled(true) + .setProfileDurationSeconds(durationSeconds) + .build()) + .setProfileId(UUID.randomUUID().toString()) + .setCpuMetric(0) + .setMemoryUsage(0) + .build(); + alertHandler.accept(alertBreach); + } + public void setPipeline(AlertMetricType type, AlertPipeline alertPipeline) { alertPipelines.setAlertPipeline(type, alertPipeline); } diff --git a/agent/agent-profiler/agent-alerting/src/test/java/com/microsoft/applicationinsights/alerting/AlertingSubsystemFileTriggerTest.java b/agent/agent-profiler/agent-alerting/src/test/java/com/microsoft/applicationinsights/alerting/AlertingSubsystemFileTriggerTest.java new file mode 100644 index 00000000000..0bba791b58d --- /dev/null +++ b/agent/agent-profiler/agent-alerting/src/test/java/com/microsoft/applicationinsights/alerting/AlertingSubsystemFileTriggerTest.java @@ -0,0 +1,237 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.microsoft.applicationinsights.alerting; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.microsoft.applicationinsights.alerting.alert.AlertBreach; +import com.microsoft.applicationinsights.alerting.config.AlertConfiguration; +import com.microsoft.applicationinsights.alerting.config.AlertMetricType; +import com.microsoft.applicationinsights.alerting.config.AlertingConfiguration; +import com.microsoft.applicationinsights.alerting.config.AlertingProfileFileTriggerConfiguration; +import com.microsoft.applicationinsights.alerting.config.CollectionPlanConfiguration; +import com.microsoft.applicationinsights.alerting.config.CollectionPlanConfiguration.EngineMode; +import com.microsoft.applicationinsights.alerting.config.DefaultConfiguration; +import java.io.File; +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class AlertingSubsystemFileTriggerTest { + + @TempDir File tempDir; + + private static TestTimeSource createTimeSourceAtFileModification(File triggerFile) { + TestTimeSource timeSource = new TestTimeSource(); + timeSource.setNow(Instant.ofEpochMilli(triggerFile.lastModified())); + return timeSource; + } + + private static AlertingConfiguration defaultAlertingConfig() { + return AlertingConfiguration.create( + AlertConfiguration.builder() + .setType(AlertMetricType.CPU) + .setEnabled(false) + .setThreshold(80.0f) + .setProfileDurationSeconds(30) + .setCooldownSeconds(14400) + .build(), + AlertConfiguration.builder() + .setType(AlertMetricType.MEMORY) + .setEnabled(false) + .setThreshold(20.0f) + .setProfileDurationSeconds(120) + .setCooldownSeconds(14400) + .build(), + DefaultConfiguration.builder().build(), + CollectionPlanConfiguration.builder() + .setSingle(false) + .setMode(EngineMode.immediate) + .setExpiration(Instant.now()) + .setImmediateProfilingDurationSeconds(0) + .setSettingsMoniker("") + .build(), + new ArrayList<>()); + } + + @Test + void fileTriggerFiresWhenFileExistsAndIsRecent() throws IOException { + File triggerFile = new File(tempDir, "trigger"); + assertThat(triggerFile.createNewFile()).isTrue(); + + AtomicReference breach = new AtomicReference<>(); + Consumer consumer = breach::set; + + AlertingProfileFileTriggerConfiguration config = + AlertingProfileFileTriggerConfiguration.create(true, "trigger", 120, tempDir); + + AlertingSubsystem subsystem = + AlertingSubsystem.create(consumer, createTimeSourceAtFileModification(triggerFile), config); + + subsystem.updateConfiguration(defaultAlertingConfig()); + + assertThat(breach.get()).isNotNull(); + assertThat(breach.get().getType()).isEqualTo(AlertMetricType.MANUAL); + assertThat(breach.get().getAlertConfiguration().getProfileDurationSeconds()).isEqualTo(120); + // trigger file should be deleted + assertThat(triggerFile.exists()).isFalse(); + } + + @Test + void fileTriggerDoesNotFireWhenDisabled() throws IOException { + File triggerFile = new File(tempDir, "trigger"); + assertThat(triggerFile.createNewFile()).isTrue(); + + AtomicReference breach = new AtomicReference<>(); + Consumer consumer = breach::set; + + AlertingProfileFileTriggerConfiguration config = + AlertingProfileFileTriggerConfiguration.create(false, "trigger", 120, tempDir); + + AlertingSubsystem subsystem = AlertingSubsystem.create(consumer, new TestTimeSource(), config); + + subsystem.updateConfiguration(defaultAlertingConfig()); + + assertThat(breach.get()).isNull(); + // file should not be deleted when disabled + assertThat(triggerFile.exists()).isTrue(); + } + + @Test + void fileTriggerDoesNotFireWhenFileDoesNotExist() { + AtomicReference breach = new AtomicReference<>(); + Consumer consumer = breach::set; + + AlertingProfileFileTriggerConfiguration config = + AlertingProfileFileTriggerConfiguration.create(true, "trigger", 120, tempDir); + + AlertingSubsystem subsystem = AlertingSubsystem.create(consumer, new TestTimeSource(), config); + + subsystem.updateConfiguration(defaultAlertingConfig()); + + assertThat(breach.get()).isNull(); + } + + @Test + void fileTriggerDoesNotFireWhenFileIsTooOld() throws IOException { + File triggerFile = new File(tempDir, "trigger"); + assertThat(triggerFile.createNewFile()).isTrue(); + + // Use a time source with a known "current" time, and set the file's lastModified to be + // older than the max age relative to that time. + long currentTimeMillis = System.currentTimeMillis(); + long oldLastModified = + currentTimeMillis + - AlertingProfileFileTriggerConfiguration.MANUAL_TRIGGER_FILE_MAX_AGE_MS + - 10_000; + assertThat(triggerFile.setLastModified(oldLastModified)).isTrue(); + + TestTimeSource timeSource = new TestTimeSource(); + timeSource.setNow(Instant.ofEpochMilli(currentTimeMillis)); + + AtomicReference breach = new AtomicReference<>(); + Consumer consumer = breach::set; + + AlertingProfileFileTriggerConfiguration config = + AlertingProfileFileTriggerConfiguration.create(true, "trigger", 120, tempDir); + + AlertingSubsystem subsystem = AlertingSubsystem.create(consumer, timeSource, config); + + subsystem.updateConfiguration(defaultAlertingConfig()); + + assertThat(breach.get()).isNull(); + // file should not be deleted when too old + assertThat(triggerFile.exists()).isTrue(); + } + + @Test + void fileTriggerDeletesFileOnSuccessfulTrigger() throws IOException { + File triggerFile = new File(tempDir, "trigger"); + assertThat(triggerFile.createNewFile()).isTrue(); + + AtomicReference breach = new AtomicReference<>(); + Consumer consumer = breach::set; + + AlertingProfileFileTriggerConfiguration config = + AlertingProfileFileTriggerConfiguration.create(true, "trigger", 120, tempDir); + + AlertingSubsystem subsystem = + AlertingSubsystem.create(consumer, createTimeSourceAtFileModification(triggerFile), config); + + subsystem.updateConfiguration(defaultAlertingConfig()); + + assertThat(breach.get()).isNotNull(); + assertThat(triggerFile.exists()).isFalse(); + } + + @Test + void fileTriggerUsesDefaultDurationWhenCollectionPlanHasZero() throws IOException { + File triggerFile = new File(tempDir, "trigger"); + assertThat(triggerFile.createNewFile()).isTrue(); + + AtomicReference breach = new AtomicReference<>(); + Consumer consumer = breach::set; + + int expectedDefaultDuration = 90; + AlertingProfileFileTriggerConfiguration config = + AlertingProfileFileTriggerConfiguration.create( + true, "trigger", expectedDefaultDuration, tempDir); + + AlertingSubsystem subsystem = + AlertingSubsystem.create(consumer, createTimeSourceAtFileModification(triggerFile), config); + + // The default alerting config has immediateProfilingDurationSeconds=0 + subsystem.updateConfiguration(defaultAlertingConfig()); + + assertThat(breach.get()).isNotNull(); + assertThat(breach.get().getAlertConfiguration().getProfileDurationSeconds()) + .isEqualTo(expectedDefaultDuration); + } + + @Test + void fileTriggerDoesNotFireWhenFileTimestampIsInFuture() throws IOException { + File triggerFile = new File(tempDir, "trigger"); + assertThat(triggerFile.createNewFile()).isTrue(); + + TestTimeSource timeSource = new TestTimeSource(); + long currentMillis = System.currentTimeMillis(); + timeSource.setNow(Instant.ofEpochMilli(currentMillis)); + assertThat(triggerFile.setLastModified(currentMillis + 1_000)).isTrue(); + + AtomicReference breach = new AtomicReference<>(); + Consumer consumer = breach::set; + + AlertingProfileFileTriggerConfiguration config = + AlertingProfileFileTriggerConfiguration.create(true, "trigger", 120, tempDir); + + AlertingSubsystem subsystem = AlertingSubsystem.create(consumer, timeSource, config); + + subsystem.updateConfiguration(defaultAlertingConfig()); + + assertThat(breach.get()).isNull(); + assertThat(triggerFile.exists()).isTrue(); + } + + @Test + void fileTriggerConfigurationFallsBackToDefaultPathWhenConfiguredPathIsNull() { + AlertingProfileFileTriggerConfiguration config = + AlertingProfileFileTriggerConfiguration.create(true, null, 120, tempDir); + + assertThat(config.getFilePath().getAbsolutePath()) + .isEqualTo( + new File(tempDir, "applicationinsights-agent-profile-trigger").getAbsolutePath()); + } + + @Test + void fileTriggerConfigurationAllowsNullTempDirForRelativePath() { + AlertingProfileFileTriggerConfiguration config = + AlertingProfileFileTriggerConfiguration.create(true, "trigger", 120, null); + + assertThat(config.getFilePath().getPath()).isEqualTo("trigger"); + } +} diff --git a/agent/agent-profiler/agent-alerting/src/test/java/com/microsoft/applicationinsights/alerting/AlertingSubsystemTest.java b/agent/agent-profiler/agent-alerting/src/test/java/com/microsoft/applicationinsights/alerting/AlertingSubsystemTest.java index 99881d032e2..8f53a417731 100644 --- a/agent/agent-profiler/agent-alerting/src/test/java/com/microsoft/applicationinsights/alerting/AlertingSubsystemTest.java +++ b/agent/agent-profiler/agent-alerting/src/test/java/com/microsoft/applicationinsights/alerting/AlertingSubsystemTest.java @@ -9,6 +9,7 @@ import com.microsoft.applicationinsights.alerting.config.AlertConfiguration; import com.microsoft.applicationinsights.alerting.config.AlertMetricType; import com.microsoft.applicationinsights.alerting.config.AlertingConfiguration; +import com.microsoft.applicationinsights.alerting.config.AlertingProfileFileTriggerConfiguration; import com.microsoft.applicationinsights.alerting.config.CollectionPlanConfiguration; import com.microsoft.applicationinsights.alerting.config.CollectionPlanConfiguration.EngineMode; import com.microsoft.applicationinsights.alerting.config.DefaultConfiguration; @@ -23,7 +24,9 @@ class AlertingSubsystemTest { private static AlertingSubsystem getAlertMonitor( Consumer consumer, TestTimeSource timeSource) { - AlertingSubsystem monitor = AlertingSubsystem.create(consumer, timeSource); + AlertingSubsystem monitor = + AlertingSubsystem.create( + consumer, timeSource, AlertingProfileFileTriggerConfiguration.createDefault()); monitor.updateConfiguration( AlertingConfiguration.create( @@ -82,7 +85,9 @@ void manualAlertWorks() { Consumer consumer = called::set; TestTimeSource timeSource = new TestTimeSource(); - AlertingSubsystem service = AlertingSubsystem.create(consumer, timeSource); + AlertingSubsystem service = + AlertingSubsystem.create( + consumer, timeSource, AlertingProfileFileTriggerConfiguration.createDefault()); service.updateConfiguration( AlertingConfiguration.create( @@ -122,8 +127,11 @@ void manualAlertDoesNotTriggerAfterExpired() { AtomicReference called = new AtomicReference<>(); Consumer consumer = called::set; TestTimeSource timeSource = new TestTimeSource(); + timeSource.setNow(Instant.now()); - AlertingSubsystem service = AlertingSubsystem.create(consumer, timeSource); + AlertingSubsystem service = + AlertingSubsystem.create( + consumer, timeSource, AlertingProfileFileTriggerConfiguration.createDefault()); service.updateConfiguration( AlertingConfiguration.create( diff --git a/agent/agent-profiler/agent-alerting/src/test/java/com/microsoft/applicationinsights/alerting/TestTimeSource.java b/agent/agent-profiler/agent-alerting/src/test/java/com/microsoft/applicationinsights/alerting/TestTimeSource.java index 90d81746c91..a2d41814605 100644 --- a/agent/agent-profiler/agent-alerting/src/test/java/com/microsoft/applicationinsights/alerting/TestTimeSource.java +++ b/agent/agent-profiler/agent-alerting/src/test/java/com/microsoft/applicationinsights/alerting/TestTimeSource.java @@ -15,6 +15,10 @@ public Instant getNow() { return now; } + void setNow(Instant now) { + this.now = now; + } + void increment(int milliseconds) { this.now = now.plusMillis(milliseconds); } diff --git a/agent/agent-profiler/request-triggers.md b/agent/agent-profiler/request-triggers.md index 6a2a0876aee..538b84912aa 100644 --- a/agent/agent-profiler/request-triggers.md +++ b/agent/agent-profiler/request-triggers.md @@ -140,6 +140,11 @@ Currently, we support: - `type` - Currently supports `fixed-duration-cooldown` - `value` - Time in seconds during which a profile will not be triggered +> **Note:** In addition to per-trigger throttling, a global cooldown (`globalCooldownSeconds`, +> default 120s) is applied after any profile recording completes — regardless of trigger source. +> During global cooldown, all triggers (CPU, memory, request, manual, periodic) are suppressed. +> See [Profiler Configuration](../../docs/README.md#configuration-file) for details. + ```json { "throttling": { diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/Configuration.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/Configuration.java index 26005a97585..57861e0a0d0 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/Configuration.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/Configuration.java @@ -1531,6 +1531,38 @@ public static class ProfilerConfiguration { public boolean enableRequestTriggering = false; public List requestTriggerEndpoints = new ArrayList<>(); @Nullable public String cgroupPath = null; + + /** Configuration for the file-based manual profile trigger mechanism. */ + public ManualProfileTriggerConfiguration manualTrigger = + new ManualProfileTriggerConfiguration(); + + // Global cooldown in seconds applied after any profile recording completes, regardless of + // trigger source. During cooldown, all trigger sources (CPU, memory, request, manual, periodic) + // are suppressed. Set to 0 to disable (individual trigger cooldowns still apply). + public int globalCooldownSeconds = 120; + + // Whether to register a JMX MBean that allows triggering profiles via JMX tools. + public boolean enableProfilerControlMBean = false; + } + + /** + * Configuration for the file-based manual profile trigger. + * + *

When enabled, the agent watches for the creation of a trigger file. Touching or creating the + * file initiates an on-demand profile recording. The file is deleted after the trigger is + * processed to prevent repeated recordings. + */ + public static class ManualProfileTriggerConfiguration { + // Whether the file-based manual trigger is enabled. + public boolean enabled = false; + + // Path to the file that triggers a manual profile when created/touched. + // If relative, it is resolved against the agent's temp directory. + public String filePath = "applicationinsights-agent-profile-trigger"; + + // Default duration (in seconds) for profiles initiated by the file trigger when no override + // is specified in the collection plan configuration. + public int defaultProfileDurationSeconds = 120; } public static class GcEventConfiguration { diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/PerformanceMonitoringService.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/PerformanceMonitoringService.java index 8a4860ab594..d8af15d22db 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/PerformanceMonitoringService.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/PerformanceMonitoringService.java @@ -15,6 +15,7 @@ import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryObservers; import com.microsoft.applicationinsights.alerting.AlertingSubsystem; import com.microsoft.applicationinsights.alerting.config.AlertingConfiguration; +import com.microsoft.applicationinsights.alerting.config.AlertingProfileFileTriggerConfiguration; import com.microsoft.applicationinsights.diagnostics.DiagnosticEngine; import com.microsoft.applicationinsights.diagnostics.DiagnosticEngineFactory; import com.microsoft.applicationinsights.diagnostics.appinsights.CodeOptimizerApplicationInsightFactoryJfr; @@ -102,6 +103,14 @@ synchronized void enableProfiler( profiler = new Profiler(configuration, tempDir); + // Build file-trigger configuration, resolving relative paths against the agent's temp directory + AlertingProfileFileTriggerConfiguration alertingProfileFileTriggerConfiguration = + AlertingProfileFileTriggerConfiguration.create( + configuration.manualTrigger.enabled, + configuration.manualTrigger.filePath, + configuration.manualTrigger.defaultProfileDurationSeconds, + tempDir); + alerting = AlertingSubsystemInit.create( configuration, @@ -110,7 +119,8 @@ synchronized void enableProfiler( profiler, telemetryClient, diagnosticEngine, - alertServiceExecutorService); + alertServiceExecutorService, + alertingProfileFileTriggerConfiguration); uploadService = new UploadService( @@ -191,4 +201,10 @@ public void updateConfiguration(AlertingConfiguration alertingConfig) { alerting.updateConfiguration(alertingConfig); } } + + public void evaluateFileTrigger() { + if (alerting != null) { + alerting.evaluateFileTrigger(); + } + } } diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/Profiler.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/Profiler.java index 5e877d9f1fb..6ad09d39bbb 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/Profiler.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/Profiler.java @@ -8,6 +8,7 @@ import com.microsoft.applicationinsights.agent.internal.profiler.upload.UploadListener; import com.microsoft.applicationinsights.agent.internal.profiler.upload.UploadService; import com.microsoft.applicationinsights.alerting.alert.AlertBreach; +import com.microsoft.applicationinsights.alerting.analysis.TimeSource; import com.microsoft.applicationinsights.alerting.config.AlertConfiguration; import com.microsoft.applicationinsights.alerting.config.AlertMetricType; import io.opentelemetry.contrib.jfr.connection.FlightRecorderConnection; @@ -39,6 +40,8 @@ *

*/ public class Profiler { @@ -60,6 +63,15 @@ public class Profiler { @Nullable private Recording activeRecording = null; @Nullable private File activeRecordingFile = null; + // Global cooldown: earliest time at which a new recording is allowed, regardless of trigger type. + // This prevents rapid successive profiles from different trigger sources (e.g., file trigger + // immediately followed by a CPU threshold breach). Reset after each recording completes. + private volatile Instant globalCooldownUntil = Instant.MIN; + + // Duration (in seconds) of the global cooldown period. A value of 0 disables the global cooldown + // (individual per-trigger cooldowns still apply). + private final int globalCooldownSeconds; + private final RecordingConfiguration memoryRecordingConfiguration; private final RecordingConfiguration cpuRecordingConfiguration; private final RecordingConfiguration spanRecordingConfiguration; @@ -67,7 +79,15 @@ public class Profiler { private final File temporaryDirectory; + private final TimeSource timeSource; + public Profiler(Configuration.ProfilerConfiguration config, File tempDir) { + this(config, tempDir, TimeSource.DEFAULT); + } + + public Profiler(Configuration.ProfilerConfiguration config, File tempDir, TimeSource timeSource) { + + this.timeSource = timeSource; periodicConfig = AlertConfiguration.builder() @@ -78,6 +98,8 @@ public Profiler(Configuration.ProfilerConfiguration config, File tempDir) { .setCooldownSeconds(config.periodicRecordingIntervalSeconds) .build(); + globalCooldownSeconds = config.globalCooldownSeconds; + memoryRecordingConfiguration = AlternativeJfrConfigurations.getMemoryProfileConfig(config); cpuRecordingConfiguration = AlternativeJfrConfigurations.getCpuProfileConfig(config); spanRecordingConfiguration = AlternativeJfrConfigurations.getSpanProfileConfig(config); @@ -109,6 +131,17 @@ public void initialize( } } + // visible for testing + void initialize( + UploadService uploadService, + ScheduledExecutorService scheduledExecutorService, + FlightRecorderConnection flightRecorderConnection) { + this.uploadService = uploadService; + this.scheduledExecutorService = scheduledExecutorService; + this.recordingOptionsBuilder = new RecordingOptions.Builder(); + this.flightRecorderConnection = flightRecorderConnection; + } + /** Apply new configuration settings obtained from Service Profiler. */ public void updateConfiguration(ProfilerConfiguration newConfig) { logger.debug("Received config {}", newConfig.getLastModified()); @@ -118,7 +151,7 @@ public void updateConfiguration(ProfilerConfiguration newConfig) { // visible for tests void profileAndUpload(AlertBreach alertBreach, Duration duration, UploadListener uploadListener) { - Instant recordingStart = Instant.now(); + Instant recordingStart = timeSource.getNow(); executeProfile( alertBreach.getType(), duration, @@ -133,6 +166,15 @@ private Recording startRecording(AlertMetricType alertType, Duration duration) { return null; } + // Enforce global cooldown across all trigger sources + if (globalCooldownSeconds > 0 && timeSource.getNow().isBefore(globalCooldownUntil)) { + logger.info( + "Alert received (type={}), but global cooldown is active until {}. Ignoring request.", + alertType, + globalCooldownUntil); + return null; + } + RecordingConfiguration recordingConfiguration; switch (alertType) { case REQUEST: @@ -278,10 +320,17 @@ private static void writeFileFromStream(Recording recording, File recordingFile) } } - private void clearActiveRecording() { + // visible for testing + void clearActiveRecording() { synchronized (activeRecordingLock) { activeRecording = null; + // Start global cooldown now that the recording is complete + if (globalCooldownSeconds > 0) { + globalCooldownUntil = timeSource.getNow().plusSeconds(globalCooldownSeconds); + logger.debug("Global profile cooldown active until {}", globalCooldownUntil); + } + // delete uploaded profile if (activeRecordingFile != null && activeRecordingFile.exists()) { if (!activeRecordingFile.delete()) { @@ -292,6 +341,18 @@ private void clearActiveRecording() { } } + // visible for testing + Instant getGlobalCooldownUntil() { + return globalCooldownUntil; + } + + // visible for testing + boolean isRecordingActive() { + synchronized (activeRecordingLock) { + return activeRecording != null; + } + } + /** Dump JFR profile to file. */ // visible for testing protected File createJfrFile(Duration duration) throws IOException { @@ -302,7 +363,7 @@ protected File createJfrFile(Duration duration) throws IOException { } } - Instant recordingStart = Instant.now(); + Instant recordingStart = timeSource.getNow(); Instant recordingEnd = recordingStart.plus(duration); return new File( diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerControl.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerControl.java new file mode 100644 index 00000000000..f6d5c6df10e --- /dev/null +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerControl.java @@ -0,0 +1,107 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.microsoft.applicationinsights.agent.internal.profiler; + +import com.microsoft.applicationinsights.alerting.alert.AlertBreach; +import com.microsoft.applicationinsights.alerting.config.AlertConfiguration; +import com.microsoft.applicationinsights.alerting.config.AlertMetricType; +import java.lang.management.ManagementFactory; +import java.util.UUID; +import java.util.function.Consumer; +import javax.management.InstanceAlreadyExistsException; +import javax.management.MBeanServer; +import javax.management.ObjectName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * JMX MBean that exposes profile triggering via JMX tools. + * + *

Usage via jmxterm (any JDK): + * + * 

+ *   echo "run -b com.microsoft:type=AI-alert,name=ProfilerControl triggerProfile" | \
+ *     java -jar jmxterm.jar -l <pid>
+ *
+ * + *

Or connect with JConsole and invoke {@code triggerProfile()} on the {@code + * com.microsoft:type=AI-alert,name=ProfilerControl} MBean. + */ +public class ProfilerControl implements ProfilerControlMBean { + + private static final Logger logger = LoggerFactory.getLogger(ProfilerControl.class); + + private static final String OBJECT_NAME = "com.microsoft:type=AI-alert,name=ProfilerControl"; + private static final int DEFAULT_DURATION_SECONDS = 120; + + private final Consumer alertHandler; + + /** + * Creates a new ProfilerControl instance. + * + * @param alertHandler consumer that processes the generated {@link AlertBreach}, typically wired + * to the profiler's recording logic + */ + ProfilerControl(Consumer alertHandler) { + this.alertHandler = alertHandler; + } + + @Override + public String triggerProfile() { + return triggerProfile(DEFAULT_DURATION_SECONDS); + } + + @Override + public String triggerProfile(int durationSeconds) { + if (durationSeconds <= 0) { + return "Error: duration must be positive, got " + durationSeconds; + } + + logger.info("Manual profile triggered via JMX, duration={}s", durationSeconds); + + AlertBreach alertBreach = + AlertBreach.builder() + .setType(AlertMetricType.MANUAL) + .setAlertValue(0.0) + .setAlertConfiguration( + AlertConfiguration.builder() + .setType(AlertMetricType.MANUAL) + .setEnabled(true) + .setProfileDurationSeconds(durationSeconds) + .build()) + .setProfileId(UUID.randomUUID().toString()) + .setCpuMetric(0) + .setMemoryUsage(0) + .build(); + + alertHandler.accept(alertBreach); + return "Profile trigger requested (duration=" + + durationSeconds + + "s, id=" + + alertBreach.getProfileId() + + ")"; + } + + /** + * Registers this MBean with the platform MBean server. Call during profiler initialization. + * + * @param alertHandler the alert handler (typically wired to Profiler.accept) + */ + public static void register(Consumer alertHandler) { + try { + MBeanServer beanServer = ManagementFactory.getPlatformMBeanServer(); + ObjectName objectName = new ObjectName(OBJECT_NAME); + ProfilerControl bean = new ProfilerControl(alertHandler); + beanServer.registerMBean(bean, objectName); + logger.info( + "Registered profiler control MBean: {}. " + + "Trigger profiles via JMX tools (e.g. jmxterm or JConsole).", + OBJECT_NAME); + } catch (InstanceAlreadyExistsException e) { + logger.debug("Profiler control MBean already registered"); + } catch (Exception e) { + logger.warn("Failed to register profiler control MBean", e); + } + } +} diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerControlMBean.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerControlMBean.java new file mode 100644 index 00000000000..6fd76fe05c5 --- /dev/null +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerControlMBean.java @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.microsoft.applicationinsights.agent.internal.profiler; + +/** + * MBean interface for triggering Application Insights profiles via JMX. + * + *

Can be invoked via jmxterm, JConsole, or any JMX-compatible tool. The MBean is registered + * under {@code com.microsoft:type=AI-alert,name=ProfilerControl} when {@code + * enableProfilerControlMBean} is set to {@code true} in the profiler configuration. + * + * @see ProfilerControl + */ +public interface ProfilerControlMBean { + + /** + * Trigger a manual profile with the default duration (120 seconds). + * + * @return a status message indicating whether the profile was requested + */ + String triggerProfile(); + + /** + * Trigger a manual profile with the specified duration. + * + * @param durationSeconds the desired recording duration in seconds; must be positive + * @return a status message indicating whether the profile was requested + */ + String triggerProfile(int durationSeconds); +} diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilingInitializer.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilingInitializer.java index b6fac1be160..4420a6c4a2c 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilingInitializer.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilingInitializer.java @@ -157,12 +157,23 @@ private void startPollingForConfigUpdates() { private void pullProfilerSettings(ConfigService configService) { try { - configService.pullSettings().subscribe(this::applyConfiguration, this::logProfilerPullError); + configService + .pullSettings() + .doFinally(result -> evaluateFileTrigger()) + .subscribe(this::applyConfiguration, this::logProfilerPullError); } catch (Throwable t) { logProfilerPullError(t); } } + private synchronized void evaluateFileTrigger() { + if (currentlyEnabled.get()) { + if (performanceMonitoringService != null) { + performanceMonitoringService.evaluateFileTrigger(); + } + } + } + private void logProfilerPullError(Throwable e) { if (currentlyEnabled.get()) { logger.error("Error pulling service profiler settings", e); @@ -175,8 +186,10 @@ synchronized void applyConfiguration(ProfilerConfiguration config) { if (currentlyEnabled.get() || (config.isEnabled() && config.hasBeenConfigured())) { AlertingConfiguration alertingConfig = AlertConfigParser.toAlertingConfig(config); + boolean manualProfilingConfigured = + configuration.manualTrigger.enabled || configuration.enableProfilerControlMBean; - if (alertingConfig.hasAnEnabledTrigger()) { + if (alertingConfig.hasAnEnabledTrigger() || manualProfilingConfigured) { if (!currentlyEnabled.getAndSet(true)) { enableProfiler(); } diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/AlertingSubsystemInit.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/AlertingSubsystemInit.java index 51f696750f9..b7c5b3a5b85 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/AlertingSubsystemInit.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/AlertingSubsystemInit.java @@ -14,6 +14,7 @@ import com.microsoft.applicationinsights.agent.internal.configuration.Configuration; import com.microsoft.applicationinsights.agent.internal.configuration.GcReportingLevel; import com.microsoft.applicationinsights.agent.internal.profiler.Profiler; +import com.microsoft.applicationinsights.agent.internal.profiler.ProfilerControl; import com.microsoft.applicationinsights.agent.internal.profiler.upload.ServiceProfilerIndex; import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryClient; import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryObservers; @@ -23,6 +24,7 @@ import com.microsoft.applicationinsights.alerting.analysis.pipelines.AlertPipeline; import com.microsoft.applicationinsights.alerting.analysis.pipelines.AlertPipelineMultiplexer; import com.microsoft.applicationinsights.alerting.config.AlertMetricType; +import com.microsoft.applicationinsights.alerting.config.AlertingProfileFileTriggerConfiguration; import com.microsoft.applicationinsights.diagnostics.DiagnosticEngine; import java.util.List; import java.util.Map; @@ -47,14 +49,17 @@ public static AlertingSubsystem create( Profiler profiler, TelemetryClient telemetryClient, DiagnosticEngine diagnosticEngine, - ExecutorService executorService) { + ExecutorService executorService, + AlertingProfileFileTriggerConfiguration alertingProfileFileTriggerConfiguration) { // TODO (trask) delay creation of AlertingSubsystem until after Profiler is created and // initialized? Consumer alertAction = alert -> alertAction(alert, profiler, diagnosticEngine, telemetryClient); - alertingSubsystem = AlertingSubsystem.create(alertAction, TimeSource.DEFAULT); + alertingSubsystem = + AlertingSubsystem.create( + alertAction, TimeSource.DEFAULT, alertingProfileFileTriggerConfiguration); if (configuration.enableRequestTriggering) { if (!configuration.requestTriggerEndpoints.isEmpty()) { @@ -80,6 +85,11 @@ public static AlertingSubsystem create( executorService, fromGcEventMonitorConfiguration(reportingLevel)); + // Register JMX MBean for triggering profiles via JMX tools (e.g. jmxterm, JConsole) + if (configuration.enableProfilerControlMBean) { + ProfilerControl.register(alertAction); + } + return alertingSubsystem; } diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerControlTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerControlTest.java new file mode 100644 index 00000000000..05092ebae19 --- /dev/null +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerControlTest.java @@ -0,0 +1,97 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.microsoft.applicationinsights.agent.internal.profiler; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.microsoft.applicationinsights.alerting.alert.AlertBreach; +import com.microsoft.applicationinsights.alerting.config.AlertMetricType; +import java.lang.management.ManagementFactory; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import javax.management.MBeanServer; +import javax.management.ObjectName; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +class ProfilerControlTest { + + private static final String OBJECT_NAME = "com.microsoft:type=AI-alert,name=ProfilerControl"; + + @AfterEach + void cleanup() throws Exception { + // Unregister MBean if it was registered during the test + MBeanServer beanServer = ManagementFactory.getPlatformMBeanServer(); + ObjectName objectName = new ObjectName(OBJECT_NAME); + if (beanServer.isRegistered(objectName)) { + beanServer.unregisterMBean(objectName); + } + } + + @Test + void triggerProfileWithDefaultDuration() { + AtomicReference received = new AtomicReference<>(); + Consumer handler = received::set; + + ProfilerControl control = new ProfilerControl(handler); + String result = control.triggerProfile(); + + assertThat(result).startsWith("Profile trigger requested"); + + AlertBreach breach = received.get(); + assertThat(breach).isNotNull(); + assertThat(breach.getType()).isEqualTo(AlertMetricType.MANUAL); + assertThat(breach.getAlertConfiguration().getProfileDurationSeconds()).isEqualTo(120); + assertThat(breach.getProfileId()).isNotNull(); + } + + @Test + void registerCreatesAccessibleMBean() throws Exception { + AtomicReference received = new AtomicReference<>(); + Consumer handler = received::set; + + ProfilerControl.register(handler); + + MBeanServer beanServer = ManagementFactory.getPlatformMBeanServer(); + ObjectName objectName = new ObjectName(OBJECT_NAME); + + assertThat(beanServer.isRegistered(objectName)).isTrue(); + + // Invoke triggerProfile via JMX + Object result = beanServer.invoke(objectName, "triggerProfile", null, null); + + assertThat(result).isInstanceOf(String.class); + assertThat((String) result).contains("duration=120s"); + assertThat(received.get()).isNotNull(); + assertThat(received.get().getType()).isEqualTo(AlertMetricType.MANUAL); + } + + @Test + void registerWithCustomDurationViaMBean() throws Exception { + AtomicReference received = new AtomicReference<>(); + Consumer handler = received::set; + + ProfilerControl.register(handler); + + MBeanServer beanServer = ManagementFactory.getPlatformMBeanServer(); + ObjectName objectName = new ObjectName(OBJECT_NAME); + + Object result = + beanServer.invoke(objectName, "triggerProfile", new Object[] {45}, new String[] {"int"}); + + assertThat(result).isInstanceOf(String.class); + assertThat((String) result).contains("duration=45s"); + assertThat(received.get()).isNotNull(); + assertThat(received.get().getAlertConfiguration().getProfileDurationSeconds()).isEqualTo(45); + } + + @Test + void registerDoesNotThrowOnDoubleRegistration() { + Consumer handler = breach -> {}; + + ProfilerControl.register(handler); + // Should not throw + ProfilerControl.register(handler); + } +} diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerGlobalCooldownTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerGlobalCooldownTest.java new file mode 100644 index 00000000000..e7ab6115364 --- /dev/null +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerGlobalCooldownTest.java @@ -0,0 +1,127 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.microsoft.applicationinsights.agent.internal.profiler; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.microsoft.applicationinsights.agent.internal.configuration.Configuration; +import com.microsoft.applicationinsights.agent.internal.profiler.testutil.TestTimeSource; +import com.microsoft.applicationinsights.agent.internal.profiler.upload.UploadListener; +import com.microsoft.applicationinsights.agent.internal.profiler.upload.UploadService; +import com.microsoft.applicationinsights.alerting.alert.AlertBreach; +import com.microsoft.applicationinsights.alerting.config.AlertConfiguration; +import com.microsoft.applicationinsights.alerting.config.AlertMetricType; +import io.opentelemetry.contrib.jfr.connection.FlightRecorderConnection; +import io.opentelemetry.contrib.jfr.connection.Recording; +import io.opentelemetry.contrib.jfr.connection.RecordingConfiguration; +import io.opentelemetry.contrib.jfr.connection.RecordingOptions; +import java.io.File; +import java.time.Duration; +import java.time.Instant; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class ProfilerGlobalCooldownTest { + @TempDir File tempDir; + + private final TestTimeSource timeSource = new TestTimeSource(); + private ScheduledExecutorService executor; + + @AfterEach + void tearDown() { + if (executor != null) { + executor.shutdownNow(); + } + } + + private static AlertBreach manualBreach(int durationSeconds) { + return AlertBreach.builder() + .setType(AlertMetricType.MANUAL) + .setAlertValue(0.0) + .setAlertConfiguration( + AlertConfiguration.builder() + .setType(AlertMetricType.MANUAL) + .setEnabled(true) + .setProfileDurationSeconds(durationSeconds) + .build()) + .setProfileId(UUID.randomUUID().toString()) + .setCpuMetric(0) + .setMemoryUsage(0) + .build(); + } + + private Profiler createProfiler(int globalCooldownSeconds) { + Configuration.ProfilerConfiguration config = new Configuration.ProfilerConfiguration(); + config.globalCooldownSeconds = globalCooldownSeconds; + Profiler profiler = + new Profiler(config, tempDir, timeSource) { + @Override + protected Recording createRecording(RecordingOptions opts, RecordingConfiguration cfg) { + return mock(Recording.class); + } + }; + + FlightRecorderConnection frc = mock(FlightRecorderConnection.class); + when(frc.newRecording(any(), any())).thenReturn(mock(Recording.class)); + + executor = Executors.newScheduledThreadPool(1); + profiler.initialize(mock(UploadService.class), executor, frc); + return profiler; + } + + @Test + void globalCooldownIsSetAfterRecordingCompletes() { + Instant baseTime = Instant.parse("2025-01-01T00:00:00Z"); + timeSource.setNow(baseTime); + + Profiler profiler = createProfiler(120); + UploadListener noOpListener = index -> {}; + profiler.profileAndUpload(manualBreach(1), Duration.ofSeconds(1), noOpListener); + // Before clearing, cooldown should still be at MIN + assertThat(profiler.getGlobalCooldownUntil()).isEqualTo(Instant.MIN); + profiler.clearActiveRecording(); + // After clearing, cooldown should be exactly baseTime + 120s + assertThat(profiler.getGlobalCooldownUntil()).isEqualTo(baseTime.plusSeconds(120)); + } + + @Test + void globalCooldownNotSetWhenDisabled() { + timeSource.setNow(Instant.parse("2025-01-01T00:00:00Z")); + + Profiler profiler = createProfiler(0); + UploadListener noOpListener = index -> {}; + profiler.profileAndUpload(manualBreach(1), Duration.ofSeconds(1), noOpListener); + profiler.clearActiveRecording(); + assertThat(profiler.getGlobalCooldownUntil()).isEqualTo(Instant.MIN); + } + + @Test + void secondProfileRejectedDuringCooldown() { + Instant baseTime = Instant.parse("2025-01-01T00:00:00Z"); + timeSource.setNow(baseTime); + + Profiler profiler = createProfiler(600); + UploadListener noOpListener = index -> {}; + // First profile starts and completes + profiler.profileAndUpload(manualBreach(1), Duration.ofSeconds(1), noOpListener); + profiler.clearActiveRecording(); + // Cooldown should now be active (baseTime + 600s) + assertThat(profiler.getGlobalCooldownUntil()).isEqualTo(baseTime.plusSeconds(600)); + + // Advance time but stay within cooldown window + timeSource.setNow(baseTime.plusSeconds(300)); + + // Second profile should be silently rejected (startRecording returns null due to cooldown) + profiler.profileAndUpload(manualBreach(1), Duration.ofSeconds(1), noOpListener); + // activeRecording should still be null (second profile was rejected) + assertThat(profiler.isRecordingActive()).isFalse(); + } +} diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilingInitializerTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilingInitializerTest.java index 151c578cd89..f1681842127 100644 --- a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilingInitializerTest.java +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilingInitializerTest.java @@ -30,6 +30,7 @@ public class ProfilingInitializerTest { private static class ProfilingInitializerTestCaseBuilder { final String name; final List configurations = new ArrayList<>(); + Configuration.ProfilerConfiguration localConfiguration = defaultLocalConfiguration(); private ProfilingInitializerTestCaseBuilder(String name) { this.name = name; @@ -40,22 +41,31 @@ ProfilingInitializerTestCaseBuilder then(ProfilerConfiguration configuration) { return this; } + ProfilingInitializerTestCaseBuilder withLocalConfiguration( + Configuration.ProfilerConfiguration localConfiguration) { + this.localConfiguration = localConfiguration; + return this; + } + ProfilingInitializerTestCase assertThat(Consumer assertion) { - return new ProfilingInitializerTestCase(name, configurations, assertion); + return new ProfilingInitializerTestCase(name, configurations, localConfiguration, assertion); } } private static class ProfilingInitializerTestCase { final String name; final List configurations; + final Configuration.ProfilerConfiguration localConfiguration; final Consumer assertion; private ProfilingInitializerTestCase( String name, List configurations, + Configuration.ProfilerConfiguration localConfiguration, Consumer assertion) { this.name = name; this.configurations = configurations; + this.localConfiguration = localConfiguration; this.assertion = assertion; } } @@ -143,6 +153,20 @@ private ProfilingInitializerTestCase( .then(userConfiguredTriggersState(true)) .then(userConfiguredTriggersState(true)) .assertThat(ENABLED)); + + tests.add( + new ProfilingInitializerTestCaseBuilder( + "Manual file trigger enabled with no automatic triggers enables profiler") + .withLocalConfiguration(localConfiguration(true, false)) + .then(userConfiguredTriggersState(false)) + .assertThat(ENABLED)); + + tests.add( + new ProfilingInitializerTestCaseBuilder( + "Profiler control MBean enabled with no automatic triggers enables profiler") + .withLocalConfiguration(localConfiguration(false, true)) + .then(userConfiguredTriggersState(false)) + .assertThat(ENABLED)); } @TestFactory @@ -153,7 +177,8 @@ public Collection runTests() { return DynamicTest.dynamicTest( testCase.name, () -> { - ProfilingInitializer profiler = createProfilingInitializer(); + ProfilingInitializer profiler = + createProfilingInitializer(testCase.localConfiguration); testCase.configurations.forEach(profiler::applyConfiguration); @@ -213,7 +238,8 @@ private static ProfilerConfiguration profileNowState( @SuppressWarnings( "DirectInvocationOnMock") // direct mock invocation is intentional for test setup - private static ProfilingInitializer createProfilingInitializer() { + private static ProfilingInitializer createProfilingInitializer( + Configuration.ProfilerConfiguration localConfiguration) { TelemetryClient client = Mockito.mock(TelemetryClient.class); MessageTelemetryBuilder messageTelemetryBuilder = MessageTelemetryBuilder.create(); Mockito.when(client.newMessageTelemetryBuilder()).thenReturn(messageTelemetryBuilder); @@ -225,7 +251,7 @@ private static ProfilingInitializer createProfilingInitializer() { ProfilingInitializer profiler = ProfilingInitializer.initialize( new File("/tmp/"), - new Configuration.ProfilerConfiguration(), + localConfiguration, GcReportingLevel.NONE, "test-role-name", "test-role-instance", @@ -240,4 +266,16 @@ static long toBinaryDate(Instant expiration) { OffsetDateTime offset = OffsetDateTime.of(1, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC); return Duration.between(offset, expiration.atZone(ZoneOffset.UTC)).getSeconds() * 10000000L; } + + private static Configuration.ProfilerConfiguration defaultLocalConfiguration() { + return localConfiguration(false, false); + } + + private static Configuration.ProfilerConfiguration localConfiguration( + boolean manualTriggerEnabled, boolean profilerControlMBeanEnabled) { + Configuration.ProfilerConfiguration configuration = new Configuration.ProfilerConfiguration(); + configuration.manualTrigger.enabled = manualTriggerEnabled; + configuration.enableProfilerControlMBean = profilerControlMBeanEnabled; + return configuration; + } } diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/AlertTriggerSpanProcessorTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/AlertTriggerSpanProcessorTest.java index 62747ce97a1..06004846bd5 100644 --- a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/AlertTriggerSpanProcessorTest.java +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/AlertTriggerSpanProcessorTest.java @@ -10,6 +10,7 @@ import com.microsoft.applicationinsights.alerting.analysis.TimeSource; import com.microsoft.applicationinsights.alerting.analysis.pipelines.AlertPipelineMultiplexer; import com.microsoft.applicationinsights.alerting.config.AlertMetricType; +import com.microsoft.applicationinsights.alerting.config.AlertingProfileFileTriggerConfiguration; import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.api.trace.StatusCode; @@ -121,7 +122,11 @@ private static void run(Handle handle) throws InterruptedException { triggerConfig.filter.value = "foo.*"; triggerConfig.threshold.value = 0.75f; - AlertingSubsystem alertingSubsystem = AlertingSubsystem.create(alertAction, TimeSource.DEFAULT); + AlertingSubsystem alertingSubsystem = + AlertingSubsystem.create( + alertAction, + TimeSource.DEFAULT, + AlertingProfileFileTriggerConfiguration.createDefault()); TestTimeSource timeSource = new TestTimeSource(); timeSource.setNow(Instant.EPOCH); diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/GcEventInitTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/GcEventInitTest.java index 667d1ce7418..5b1257d5f16 100644 --- a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/GcEventInitTest.java +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/triggers/GcEventInitTest.java @@ -12,6 +12,7 @@ import com.microsoft.applicationinsights.alerting.alert.AlertBreach; import com.microsoft.applicationinsights.alerting.analysis.TimeSource; import com.microsoft.applicationinsights.alerting.config.AlertingConfiguration; +import com.microsoft.applicationinsights.alerting.config.AlertingProfileFileTriggerConfiguration; import com.microsoft.gcmonitor.GcCollectionEvent; import com.microsoft.gcmonitor.GcEventConsumer; import com.microsoft.gcmonitor.GcMonitorFactory; @@ -73,7 +74,10 @@ public MemoryManagement monitor( private static AlertingSubsystem getAlertingSubsystem( CompletableFuture alertFuture, TimeSource timeSource) { AlertingSubsystem alertingSubsystem = - AlertingSubsystem.create(alertFuture::complete, timeSource); + AlertingSubsystem.create( + alertFuture::complete, + timeSource, + AlertingProfileFileTriggerConfiguration.createDefault()); AlertingConfiguration config = AlertConfigParser.parse( diff --git a/docs/README.md b/docs/README.md index 8fb8eddc3c2..0574b30ca65 100644 --- a/docs/README.md +++ b/docs/README.md @@ -92,7 +92,15 @@ Additionally, a number of parameters can be configured using environment variabl "profiler": { "enabled": true, "cpuTriggeredSettings": "profile-without-env-data", - "memoryTriggeredSettings": "profile-without-env-data" + "memoryTriggeredSettings": "profile-without-env-data", + "manualTriggeredSettings": "profile-without-env-data", + "globalCooldownSeconds": 120, + "enableProfilerControlMBean": false, + "manualTrigger": { + "enabled": false, + "filePath": "applicationinsights-agent-profile-trigger", + "defaultProfileDurationSeconds": 120 + } } } } @@ -114,3 +122,65 @@ This can be one of: [Warning](#Warning) section for details. - `profile`. Uses the `profile.jfc` jfc configuration that ships with JFR. - A path to a custom jfc configuration file on the file system, i.e `/tmp/myconfig.jfc`. + +`manualTriggeredSettings` - This configuration will be used for manually triggered profiles (via +file trigger or JMX MBean). Accepts the same values as `cpuTriggeredSettings`. + +`globalCooldownSeconds` - (default: 120) Cooldown period in seconds applied after any profile +recording completes, regardless of trigger source. During cooldown, all trigger sources (CPU, +memory, request, manual, periodic) are suppressed. Set to `0` to disable (individual trigger +cooldowns still apply). + +`enableProfilerControlMBean` - (default: false) Whether to register a JMX MBean +(`com.microsoft:type=AI-alert,name=ProfilerControl`) that allows triggering profiles via +JMX tools. See [Manual Profile Triggering](#manual-profile-triggering) for usage. + +`manualTrigger` - Configuration for the file-based manual profile trigger: + +- `enabled` - (default: false) Whether the file-based manual trigger is enabled. +- `filePath` - (default: `applicationinsights-agent-profile-trigger`) Path to the trigger file. + If relative, it is resolved against the agent's temp directory. Creating or touching this file + triggers a profile recording. +- `defaultProfileDurationSeconds` - (default: 120) Duration in seconds for profiles initiated by + the file trigger when no override is specified in the collection plan. + +## Manual Profile Triggering + +In addition to automatic threshold-based triggers, profiles can be initiated manually using either +the file-based trigger or the JMX MBean. + +### File-based trigger + +When `manualTrigger.enabled` is `true`, you can trigger a profile by creating or touching the +trigger file: + +```bash +touch /tmp/applicationinsights-agent-profile-trigger +``` + +The file must have been modified within the last 60 seconds to be considered valid (stale files +are ignored to prevent unintended recordings after restarts). After the trigger is detected, the +file is automatically deleted. + +> **Note:** The file trigger is evaluated on the profiler's configuration polling cycle +> (default every 60 seconds), so there may be up to a one-minute delay between touching the file +> and the profile recording starting. + +### JMX MBean trigger + +When `enableProfilerControlMBean` is `true`, the agent registers a JMX MBean that can be invoked +to trigger profiles: + +**Via jmxterm:** +```bash +echo "run -b com.microsoft:type=AI-alert,name=ProfilerControl triggerProfile" | \ + java -jar jmxterm.jar -l +``` + +**Via JConsole:** + +Connect to the target JVM process, navigate to the MBeans tab, expand +`com.microsoft` → `AI-alert` → `ProfilerControl`, and invoke the `triggerProfile` operation. + +Both manual triggering mechanisms respect the `globalCooldownSeconds` setting — if a profile was +recently recorded, manual triggers will be suppressed until the cooldown expires.