SOLR-18165 Use dot separated metric names (#4223)

Co-authored-by: Matthew Biscocho <[email protected]>
Co-authored-by: Chan Chan <[email protected]>

Author: 3 people

changelog/unreleased/SOLR-18165-dot-separated-metric-names.yml

@@ -0,0 +1,9 @@
+title: Use dot-separated OTel metric names for OTLP export. This is a back-compat break from version 10.0
+type: changed
+authors:
+  - name: Jan Høydahl
+    url: https://home.apache.org/phonebook.html?uid=janhoy
+  - name: Matthew Biscocho
+links:
+  - name: SOLR-18165
+    url: https://issues.apache.org/jira/browse/SOLR-18165

solr/core/src/java/org/apache/solr/core/CoreContainer.java

@@ -814,7 +814,7 @@ private void loadInternal() {
           caffeineCache.initializeMetrics(
               solrMetricsContext,
               Attributes.builder().put(NAME_ATTR, cacheName).build(),
-              "solr_node_cache");
+              "solr.node.cache");
         }
         m.put(cacheName, c);
       }
@@ -957,7 +957,7 @@ private void loadInternal() {
             ExecutorUtil.newMDCAwareFixedThreadPool(
                 cfg.getCoreLoadThreadCount(isZooKeeperAware()),
                 new SolrNamedThreadFactory("coreLoadExecutor")),
-            "solr_node_executor",
+            "solr.node.executor",
             "coreLoadExecutor",
             SolrInfoBean.Category.CONTAINER);
 

solr/core/src/java/org/apache/solr/core/SolrCore.java

@@ -1323,47 +1323,47 @@ public void initializeMetrics(SolrMetricsContext parentContext, Attributes attri
     newSearcherCounter =
         new AttributedLongCounter(
             parentContext.longCounter(
-                "solr_core_searcher_new", "Total number of new searchers opened"),
+                "solr.core.searcher.new", "Total number of new searchers opened"),
             baseSearcherAttributes);
 
     newSearcherMaxReachedCounter =
         new AttributedLongCounter(
             parentContext.longCounter(
-                "solr_core_searcher_warming_max",
+                "solr.core.searcher.warming_max",
                 "Total number of maximum concurrent warming searchers reached"),
             baseSearcherAttributes);
 
     newSearcherOtherErrorsCounter =
         new AttributedLongCounter(
             parentContext.longCounter(
-                "solr_core_searcher_errors", "Total number of searcher errors"),
+                "solr.core.searcher.errors", "Total number of searcher errors"),
             baseSearcherAttributes);
 
     newSearcherTimer =
         new AttributedLongTimer(
             parentContext.longHistogram(
-                "solr_core_indexsearcher_open_time",
+                "solr.core.indexsearcher.open.time",
                 "Time to open new searchers",
                 OtelUnit.MILLISECONDS),
             baseSearcherAttributes);
 
     newSearcherWarmupTimer =
         new AttributedLongTimer(
             parentContext.longHistogram(
-                "solr_core_indexsearcher_open_warmup_time",
+                "solr.core.indexsearcher.open.warmup.time",
                 "Time to warmup new searchers",
                 OtelUnit.MILLISECONDS),
             baseSearcherAttributes);
 
     parentContext.observableLongGauge(
-        "solr_core_ref_count",
+        "solr.core.ref_count",
         "The current number of active references to a Solr core",
         (observableLongMeasurement -> {
           observableLongMeasurement.record(getOpenCount(), baseGaugeCoreAttributes);
         }));
 
     parentContext.observableDoubleGauge(
-        "solr_core_disk_space",
+        "solr.core.disk_space",
         "Solr core disk space metrics",
         (observableDoubleMeasurement -> {
 
@@ -1397,7 +1397,7 @@ public void initializeMetrics(SolrMetricsContext parentContext, Attributes attri
         OtelUnit.MEGABYTES);
 
     parentContext.observableDoubleGauge(
-        "solr_core_index_size",
+        "solr.core.index.size",
         "Index size for a Solr core",
         (observableDoubleMeasurement -> {
           if (!isClosed())
@@ -1407,7 +1407,7 @@ public void initializeMetrics(SolrMetricsContext parentContext, Attributes attri
         OtelUnit.MEGABYTES);
 
     parentContext.observableLongGauge(
-        "solr_core_segments",
+        "solr.core.segments",
         "Number of segments in a Solr core",
         (observableLongMeasurement -> {
           if (isReady())
@@ -1416,7 +1416,7 @@ public void initializeMetrics(SolrMetricsContext parentContext, Attributes attri
 
     if (coreContainer.isZooKeeperAware())
       parentContext.observableLongGauge(
-          "solr_core_is_leader",
+          "solr.core.is_leader",
           "Indicates whether this Solr core is currently the leader",
           (observableLongMeasurement -> {
             observableLongMeasurement.record(

solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java

@@ -879,55 +879,55 @@ public void initializeMetrics(SolrMetricsContext parentContext, Attributes attri
 
     ObservableDoubleMeasurement indexSizeMetric =
         solrMetricsContext.doubleGaugeMeasurement(
-            "solr_core_replication_index_size",
+            "solr.core.replication.index.size",
             "Size of the index in megabytes",
             OtelUnit.MEGABYTES);
 
     ObservableLongMeasurement indexVersionMetric =
         solrMetricsContext.longGaugeMeasurement(
-            "solr_core_replication_index_version", "Current index version");
+            "solr.core.replication.index.version", "Current index version");
 
     ObservableLongMeasurement indexGenerationMetric =
         solrMetricsContext.longGaugeMeasurement(
-            "solr_core_replication_index_generation", "Current index generation");
+            "solr.core.replication.index.generation", "Current index generation");
 
     ObservableLongMeasurement isLeaderMetric =
         solrMetricsContext.longGaugeMeasurement(
-            "solr_core_replication_is_leader", "Whether this node is a leader (1) or not (0)");
+            "solr.core.replication.is_leader", "Whether this node is a leader (1) or not (0)");
 
     ObservableLongMeasurement isFollowerMetric =
         solrMetricsContext.longGaugeMeasurement(
-            "solr_core_replication_is_follower", "Whether this node is a follower (1) or not (0)");
+            "solr.core.replication.is_follower", "Whether this node is a follower (1) or not (0)");
 
     ObservableLongMeasurement replicationEnabledMetric =
         solrMetricsContext.longGaugeMeasurement(
-            "solr_core_replication_is_enabled", "Whether replication is enabled (1) or not (0)");
+            "solr.core.replication.is_enabled", "Whether replication is enabled (1) or not (0)");
 
     ObservableLongMeasurement isPollingDisabledMetric =
         solrMetricsContext.longGaugeMeasurement(
-            "solr_core_replication_is_polling_disabled",
+            "solr.core.replication.is_polling_disabled",
             "Whether polling is disabled (1) or not (0)");
 
     ObservableLongMeasurement isReplicatingMetric =
         solrMetricsContext.longGaugeMeasurement(
-            "solr_core_replication_is_replicating",
+            "solr.core.replication.is_replicating",
             "Whether replication is in progress (1) or not (0)");
 
     ObservableLongMeasurement timeElapsedMetric =
         solrMetricsContext.longGaugeMeasurement(
-            "solr_core_replication_time_elapsed",
+            "solr.core.replication.time_elapsed",
             "Time elapsed during replication in seconds",
             OtelUnit.SECONDS);
 
     ObservableLongMeasurement bytesDownloadedMetric =
         solrMetricsContext.longGaugeMeasurement(
-            "solr_core_replication_downloaded_size",
+            "solr.core.replication.downloaded_size",
             "Total bytes downloaded during replication",
             OtelUnit.BYTES);
 
     ObservableLongMeasurement downloadSpeedMetric =
         solrMetricsContext.longGaugeMeasurement(
-            "solr_core_replication_download_speed", "Download speed in bytes per second");
+            "solr.core.replication.download_speed", "Download speed in bytes per second");
 
     solrMetricsContext.batchCallback(
         () -> {

solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java

@@ -187,27 +187,27 @@ public HandlerMetrics(
 
       requests =
           factory.attributedLongCounter(
-              "solr_core_requests", "HTTP Solr requests", Attributes.empty());
+              "solr.core.requests", "HTTP Solr requests", Attributes.empty());
 
       numServerErrors =
           factory.attributedLongCounter(
-              "solr_core_requests_errors",
+              "solr.core.requests.errors",
               "HTTP Solr request errors",
               Attributes.of(SOURCE_ATTR, "server"));
 
       numClientErrors =
           factory.attributedLongCounter(
-              "solr_core_requests_errors",
+              "solr.core.requests.errors",
               "HTTP Solr request errors",
               Attributes.of(SOURCE_ATTR, "client"));
 
       numTimeouts =
           factory.attributedLongCounter(
-              "solr_core_requests_timeout", "HTTP Solr request timeouts", Attributes.empty());
+              "solr.core.requests.timeout", "HTTP Solr request timeouts", Attributes.empty());
 
       requestTimes =
           factory.attributedLongTimer(
-              "solr_core_requests_times",
+              "solr.core.requests.times",
               "HTTP Solr request times",
               OtelUnit.MILLISECONDS,
               Attributes.empty());

solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java

@@ -132,14 +132,14 @@ public void initializeMetrics(SolrMetricsContext parentContext, Attributes attri
     coreAdminAsyncTracker.standardExecutor =
         solrMetricsContext.instrumentedExecutorService(
             coreAdminAsyncTracker.standardExecutor,
-            "solr_node_executor",
+            "solr.node.executor",
             "asyncCoreAdminExecutor",
             getCategory());
 
     coreAdminAsyncTracker.expensiveExecutor =
         solrMetricsContext.instrumentedExecutorService(
             coreAdminAsyncTracker.expensiveExecutor,
-            "solr_node_expensive_executor",
+            "solr.node.expensive.executor",
             "asyncCoreExpensiveAdminExecutor",
             getCategory());
   }

solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java

@@ -439,6 +439,6 @@ public void initializeMetrics(SolrMetricsContext parentContext, Attributes attri
     httpListenerFactory.initializeMetrics(solrMetricsContext, Attributes.empty());
     commExecutor =
         solrMetricsContext.instrumentedExecutorService(
-            commExecutor, "solr_core_executor", "httpShardExecutor", SolrInfoBean.Category.QUERY);
+            commExecutor, "solr.core.executor", "httpShardExecutor", SolrInfoBean.Category.QUERY);
   }
 }

solr/core/src/java/org/apache/solr/handler/component/SuggestComponent.java

@@ -387,9 +387,8 @@ public void initializeMetrics(SolrMetricsContext parentContext, Attributes attri
     super.initializeMetrics(parentContext, attributes);
     var suggesterAttributes =
         attributes.toBuilder().put(CATEGORY_ATTR, getCategory().toString()).build();
-
     this.solrMetricsContext.observableLongGauge(
-        "solr_core_suggester_total_size",
+        "solr.core.suggester.total.size",
         "Total memory size in bytes of all suggester",
         (observableLongMeasurement) -> {
           observableLongMeasurement.record(ramBytesUsed(), suggesterAttributes);

solr/core/src/java/org/apache/solr/metrics/otel/FilterablePrometheusMetricReader.java

@@ -65,8 +65,9 @@ public MetricSnapshots collect(
       return super.collect();
     }
 
-    // Prometheus appends a suffix to the metrics depending on the metric type. We need to sanitize
-    // the suffix off if they filter by Prometheus name instead of OTEL name.
+    // Users may filter by Prometheus-format names (e.g. "solr_core_requests") or with a
+    // Prometheus type suffix (e.g. "solr_core_requests_total"). Strip any such suffix so we can
+    // compare against the Prometheus base name returned by getMetadata().getPrometheusName().
     Set<String> sanitizedNames =
         includedNames.stream()
             .map(
@@ -84,7 +85,18 @@ public MetricSnapshots collect(
     if (sanitizedNames.isEmpty()) {
       snapshotsToFilter = super.collect();
     } else {
-      snapshotsToFilter = super.collect(sanitizedNames::contains);
+      // We collect all metrics and filter by Prometheus name rather than using
+      // super.collect(Predicate) which matches on OTel internal names. This avoids a mismatch
+      // when OTel names use dot-separators (e.g. "solr.core.requests") but users filter by the
+      // Prometheus underscore-format name they see in the output (e.g. "solr_core_requests").
+      MetricSnapshots all = super.collect();
+      MetricSnapshots.Builder nameFiltered = MetricSnapshots.builder();
+      for (MetricSnapshot snapshot : all) {
+        if (sanitizedNames.contains(snapshot.getMetadata().getPrometheusName())) {
+          nameFiltered.metricSnapshot(snapshot);
+        }
+      }
+      snapshotsToFilter = nameFiltered.build();
     }
 
     // Return named filtered snapshots if not label filters provided

solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedInstrumentFactory.java

@@ -133,7 +133,7 @@ public AttributedLongTimer attributedLongTimer(
 
   /** Replace core metric name prefix to node prefix */
   private String toNodeMetricName(String coreMetricName) {
-    return coreMetricName.replace("solr_core", "solr_node");
+    return coreMetricName.replace("solr.core", "solr.node");
   }
 
   /** Filter out core attributes and keep all others for node-level metrics */