|
| 1 | +// Copyright 2025 Google LLC |
| 2 | +// |
| 3 | +// Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | +// you may not use this file except in compliance with the License. |
| 5 | +// You may obtain a copy of the License at |
| 6 | +// |
| 7 | +// http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | +// |
| 9 | +// Unless required by applicable law or agreed to in writing, software |
| 10 | +// distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | +// See the License for the specific language governing permissions and |
| 13 | +// limitations under the License. |
| 14 | + |
| 15 | +package internal |
| 16 | + |
| 17 | +import ( |
| 18 | + "context" |
| 19 | + "fmt" |
| 20 | + "log" |
| 21 | + "sync" |
| 22 | + "time" |
| 23 | + |
| 24 | + btopt "cloud.google.com/go/bigtable/internal/option" |
| 25 | + "go.opentelemetry.io/otel/attribute" |
| 26 | + "go.opentelemetry.io/otel/metric" |
| 27 | +) |
| 28 | + |
| 29 | +const ( |
| 30 | + // outstandingRPCsMetricName is the name for the outstanding RPCs histogram. |
| 31 | + outstandingRPCsMetricName = "connection_pool/outstanding_rpcs" |
| 32 | + // perConnErrorCountMetricName is the name for the per-connection error count histogram. |
| 33 | + perConnErrorCountMetricName = "per_connection_error_count" |
| 34 | + // clientMeterName is the prefix for metrics name |
| 35 | + clientMeterName = "bigtable.googleapis.com/internal/client/" |
| 36 | +) |
| 37 | + |
| 38 | +// MetricsReporter periodically collects and reports metrics for the connection pool. |
| 39 | +type MetricsReporter struct { |
| 40 | + config btopt.MetricsReporterConfig |
| 41 | + connPoolStatsSupplier connPoolStatsSupplier |
| 42 | + logger *log.Logger |
| 43 | + ticker *time.Ticker |
| 44 | + done chan struct{} |
| 45 | + stopOnce sync.Once |
| 46 | + |
| 47 | + // OpenTelemetry metric instruments |
| 48 | + meterProvider metric.MeterProvider |
| 49 | + outstandingRPCsHistogram metric.Float64Histogram |
| 50 | + perConnectionErrorCountHistogram metric.Float64Histogram |
| 51 | +} |
| 52 | + |
| 53 | +// NewMetricsReporter starts a monitor to export periodic metrics |
| 54 | +func NewMetricsReporter(config btopt.MetricsReporterConfig, connPoolStatsSupplier connPoolStatsSupplier, logger *log.Logger, mp metric.MeterProvider) (*MetricsReporter, error) { |
| 55 | + mr := &MetricsReporter{ |
| 56 | + config: config, |
| 57 | + connPoolStatsSupplier: connPoolStatsSupplier, |
| 58 | + logger: logger, |
| 59 | + done: make(chan struct{}), |
| 60 | + meterProvider: mp, |
| 61 | + } |
| 62 | + |
| 63 | + if !config.Enabled { |
| 64 | + // Stats Disabled, return error during ctor |
| 65 | + return nil, fmt.Errorf("bigtable_connpool: MetricsReporterConfig.Enabled is false") |
| 66 | + } |
| 67 | + |
| 68 | + if mp == nil { |
| 69 | + // State: Stats Enabled but MeterProvider is nil, return error during ctor |
| 70 | + return nil, fmt.Errorf("bigtable_connpool: MetricsReporterConfig.Enabled is true, but MeterProvider is nil") |
| 71 | + } |
| 72 | + |
| 73 | + // create meter |
| 74 | + meter := mp.Meter(clientMeterName) |
| 75 | + var err error |
| 76 | + |
| 77 | + // fail if meter cannot be created. |
| 78 | + mr.outstandingRPCsHistogram, err = meter.Float64Histogram(outstandingRPCsMetricName, metric.WithDescription("Distribution of outstanding RPCs per connection."), metric.WithUnit("1")) |
| 79 | + if err != nil { |
| 80 | + return nil, fmt.Errorf("bigtable_connpool: failed to create %s histogram: %w", outstandingRPCsMetricName, err) |
| 81 | + } |
| 82 | + |
| 83 | + mr.perConnectionErrorCountHistogram, err = meter.Float64Histogram(perConnErrorCountMetricName, metric.WithDescription("Distribution of errors per connection per minute."), metric.WithUnit("1")) |
| 84 | + if err != nil { |
| 85 | + return nil, fmt.Errorf("bigtable_connpool: failed to create %s histogram: %w", perConnErrorCountMetricName, err) |
| 86 | + } |
| 87 | + |
| 88 | + return mr, nil |
| 89 | +} |
| 90 | + |
| 91 | +// Start starts the reporter. |
| 92 | +func (mr *MetricsReporter) Start(ctx context.Context) { |
| 93 | + // config.Enabled should ensure that all relevant thing (meterProvider and meter should exists) |
| 94 | + if !mr.config.Enabled { |
| 95 | + return |
| 96 | + } |
| 97 | + |
| 98 | + mr.ticker = time.NewTicker(mr.config.ReportingInterval) |
| 99 | + go func() { |
| 100 | + defer mr.ticker.Stop() |
| 101 | + for { |
| 102 | + select { |
| 103 | + case <-mr.ticker.C: |
| 104 | + mr.snapshotAndRecordMetrics(ctx) |
| 105 | + case <-mr.done: |
| 106 | + return |
| 107 | + case <-ctx.Done(): |
| 108 | + return |
| 109 | + } |
| 110 | + } |
| 111 | + }() |
| 112 | +} |
| 113 | + |
| 114 | +// Stop stops the reporter gracefully |
| 115 | +func (mr *MetricsReporter) Stop() { |
| 116 | + mr.stopOnce.Do(func() { |
| 117 | + if mr.config.Enabled { |
| 118 | + close(mr.done) |
| 119 | + } |
| 120 | + }) |
| 121 | +} |
| 122 | + |
| 123 | +// snapshotAndRecordMetrics collects and records metrics for the current state of the connection pool. |
| 124 | +func (mr *MetricsReporter) snapshotAndRecordMetrics(ctx context.Context) { |
| 125 | + stats := mr.connPoolStatsSupplier() |
| 126 | + if len(stats) == 0 { |
| 127 | + return |
| 128 | + } |
| 129 | + |
| 130 | + for _, stat := range stats { |
| 131 | + // Record per-connection error count for the interval |
| 132 | + if mr.perConnectionErrorCountHistogram != nil { |
| 133 | + mr.perConnectionErrorCountHistogram.Record(ctx, float64(stat.ErrorCount)) |
| 134 | + } |
| 135 | + transportType := "cloudpath" |
| 136 | + if stat.IsALTSUsed { |
| 137 | + transportType = "directpath" |
| 138 | + } |
| 139 | + |
| 140 | + // Common attributes for this connection |
| 141 | + baseAttrs := []attribute.KeyValue{ |
| 142 | + attribute.String("transport_type", transportType), |
| 143 | + attribute.String("lb_policy", stat.LBPolicy), |
| 144 | + } |
| 145 | + |
| 146 | + if mr.outstandingRPCsHistogram != nil { |
| 147 | + // Record distribution sample for unary load |
| 148 | + unaryAttrs := attribute.NewSet(append(baseAttrs, attribute.Bool("streaming", false))...) |
| 149 | + mr.outstandingRPCsHistogram.Record(ctx, float64(stat.OutstandingUnaryLoad), metric.WithAttributeSet(unaryAttrs)) |
| 150 | + |
| 151 | + // Record distribution sample for streaming load |
| 152 | + streamingAttrs := attribute.NewSet(append(baseAttrs, attribute.Bool("streaming", true))...) |
| 153 | + mr.outstandingRPCsHistogram.Record(ctx, float64(stat.OutstandingStreamingLoad), metric.WithAttributeSet(streamingAttrs)) |
| 154 | + } |
| 155 | + |
| 156 | + } |
| 157 | +} |
0 commit comments