metrics

package

v0.9.2 Latest Latest Go to latest Published: Jun 23, 2026 License: Apache-2.0 Imports: 11 Imported by: 0

Details

Valid go.mod file
Redistributable license
Tagged version
Stable version
Learn more about best practices

Repository

github.com/llm-d/llm-d-router

Links

Open Source Insights

Documentation ¶

Index ¶

Constants
Variables
func AddFlowControlQueueBytes(fairnessID, priority, inferencePool, modelName, targetModelName string, ...)
func DecFlowControlQueueSize(fairnessID, priority, inferencePool, modelName, targetModelName string)
func DecRunningRequests(modelName, targetModelName, fairnessID, priority string)
func ExtProcStreamFinished(code string, seconds float64)
func ExtProcStreamStarted()
func IncFlowControlQueueSize(fairnessID, priority, inferencePool, modelName, targetModelName string)
func IncRunningRequests(modelName, targetModelName, fairnessID, priority string)
func RecordDataLayerExtractError(sourceType, extractorType string)
func RecordDataLayerPollError(sourceType string)
func RecordFlowControlDispatchCycleDuration(duration time.Duration)
func RecordFlowControlPoolSaturation(inferencePool string, saturation float64)
func RecordFlowControlRequestEnqueueDuration(fairnessID string, priority string, outcome string, duration time.Duration)
func RecordFlowControlRequestQueueDuration(...)
func RecordInferenceExtensionInfo(commitSha, buildRef string)
func RecordInferenceModelRewriteDecision(modelRewriteName, modelName, targetModel string)
func RecordInferencePoolAvgKVCache(name string, utilization float64)
func RecordInferencePoolAvgQueueSize(name string, queueSize float64)
func RecordInferencePoolAvgRunningRequests(name string, runningRequests float64)
func RecordInferencePoolReadyPods(name string, runningPods float64)
func RecordInputTokens(modelName, targetModelName, fairnessID, priority string, size int)
func RecordInterTokenLatency(ctx context.Context, modelName, targetModelName, fairnessID, priority string, ...) bool
func RecordNormalizedTimePerOutputToken(ctx context.Context, modelName, targetModelName, fairnessID, priority string, ...) bool
func RecordOutputTokens(modelName, targetModelName, fairnessID, priority string, size int)
func RecordPluginProcessingLatency(extensionPoint, pluginType, pluginName string, duration time.Duration)
func RecordPromptCachedTokens(modelName, targetModelName, fairnessID, priority string, size int)
func RecordRequestCounter(modelName, targetModelName, fairnessID string, priority int)
func RecordRequestErrCounter(modelName, targetModelName, fairnessID, priority string, code string)
func RecordRequestLatencies(ctx context.Context, modelName, targetModelName, fairnessID, priority string, ...) bool
func RecordRequestSizes(modelName, targetModelName, fairnessID, priority string, reqSize int)
func RecordRequestTPOT(ctx context.Context, modelName, targetModelName, fairnessID, priority string, ...) bool
func RecordRequestTTFT(ctx context.Context, modelName, targetModelName, fairnessID, priority string, ...) bool
func RecordResponseSizes(modelName, targetModelName, fairnessID, priority string, size int)
func RecordSchedulerAttempt(err error, targetModelName string, result *fwksched.SchedulingResult)
func RecordSchedulerE2ELatency(duration time.Duration)
func Register(customCollectors ...prometheus.Collector)
func RegisterGRPCStreamMetrics()
func Reset()
func SubFlowControlQueueBytes(fairnessID, priority, inferencePool, modelName, targetModelName string, ...)

Constants ¶

View Source

const (

	// InferenceObjectiveSubsystem is the legacy subsystem for inference objective metrics.
	InferenceObjectiveSubsystem = inferenceObjectiveComponent
	// InferenceExtensionSubsystem is the legacy subsystem for inference extension metrics.
	InferenceExtensionSubsystem = inferenceExtension

	// SchedulerSubsystem is the legacy metric prefix for scheduler.
	SchedulerSubsystem = "llm_d_inference_scheduler"
)

View Source

const (
	SchedulerStatusSuccess = "success"
	SchedulerStatusFailure = "failure"
)

View Source

const (
	// LLMDRouterEndpointPickerSubsystem is the subsystem for llm-d router endpoint picker metrics.
	LLMDRouterEndpointPickerSubsystem = "llm_d_epp"
)

Variables ¶

View Source

var (
	// LlmdDataLayerPollErrorsTotal records data-source poll errors per source type.
	LlmdDataLayerPollErrorsTotal = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Subsystem: LLMDRouterEndpointPickerSubsystem,
			Name:      "datalayer_poll_errors_total",
			Help:      metricsutil.HelpMsgWithStability("Data-source poll errors per source type.", compbasemetrics.ALPHA),
		},
		[]string{"source_type"},
	)

	// LlmdDataLayerExtractErrorsTotal records extract errors per source/extractor type.
	LlmdDataLayerExtractErrorsTotal = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Subsystem: LLMDRouterEndpointPickerSubsystem,
			Name:      "datalayer_extract_errors_total",
			Help:      metricsutil.HelpMsgWithStability("Extract errors per source/extractor type.", compbasemetrics.ALPHA),
		},
		[]string{"source_type", "extractor_type"},
	)
)

--- llm-d Data-layer Metrics ---

View Source

var (
	// DataLayerPollErrorsTotal records data-source poll errors per source type.
	//
	// Deprecated: Use LlmdDataLayerPollErrorsTotal instead.
	DataLayerPollErrorsTotal = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Subsystem: SchedulerSubsystem,
			Name:      "datalayer_poll_errors_total",
			Help:      metricsutil.HelpMsgWithStability("[Deprecated: Use llm_d_epp_datalayer_poll_errors_total] Data-source poll errors per source type.", compbasemetrics.ALPHA),
		},
		[]string{"source_type"},
	)

	// DataLayerExtractErrorsTotal records extract errors per source/extractor type.
	//
	// Deprecated: Use LlmdDataLayerExtractErrorsTotal instead.
	DataLayerExtractErrorsTotal = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Subsystem: SchedulerSubsystem,
			Name:      "datalayer_extract_errors_total",
			Help:      metricsutil.HelpMsgWithStability("[Deprecated: Use llm_d_epp_datalayer_extract_errors_total] Extract errors per source/extractor type.", compbasemetrics.ALPHA),
		},
		[]string{"source_type", "extractor_type"},
	)
)

View Source

var (
	// DescInferencePoolPerEndpointQueueSize is the standardized exported prometheus descriptor.
	DescInferencePoolPerEndpointQueueSize = prometheus.NewDesc(
		"llm_d_epp_per_endpoint_queue_size",
		metricsutil.HelpMsgWithStability("The total number of requests pending in the model server queue for each underlying endpoint.", compbasemetrics.ALPHA),
		[]string{
			"name",
			"model_server_endpoint",
		}, nil,
	)
)

Functions ¶

func AddFlowControlQueueBytes ¶

func AddFlowControlQueueBytes(fairnessID, priority, inferencePool, modelName, targetModelName string, bytes uint64)

AddFlowControlQueueBytes increments the Flow Control queue bytes gauge.

func DecFlowControlQueueSize ¶

func DecFlowControlQueueSize(fairnessID, priority, inferencePool, modelName, targetModelName string)

DecFlowControlQueueSize decrements the Flow Control queue size gauge.

func DecRunningRequests ¶

func DecRunningRequests(modelName, targetModelName, fairnessID, priority string)

DecRunningRequests decreases the current running requests.

func ExtProcStreamFinished ¶

func ExtProcStreamFinished(code string, seconds float64)

ExtProcStreamFinished records an ext_proc stream close with its status and duration.

func ExtProcStreamStarted ¶

func ExtProcStreamStarted()

ExtProcStreamStarted records an ext_proc stream open.

func IncFlowControlQueueSize ¶

func IncFlowControlQueueSize(fairnessID, priority, inferencePool, modelName, targetModelName string)

IncFlowControlQueueSize increments the Flow Control queue size gauge.

func IncRunningRequests ¶

func IncRunningRequests(modelName, targetModelName, fairnessID, priority string)

IncRunningRequests increases the current running requests.

func RecordDataLayerExtractError ¶

func RecordDataLayerExtractError(sourceType, extractorType string)

RecordDataLayerExtractError increments the extract error counter for a source/extractor type.

func RecordDataLayerPollError ¶

func RecordDataLayerPollError(sourceType string)

RecordDataLayerPollError increments the poll error counter for a source type.

func RecordFlowControlDispatchCycleDuration ¶

func RecordFlowControlDispatchCycleDuration(duration time.Duration)

RecordFlowControlDispatchCycleDuration records the duration of a dispatch cycle in the Flow Control layer.

func RecordFlowControlPoolSaturation ¶

func RecordFlowControlPoolSaturation(inferencePool string, saturation float64)

RecordFlowControlPoolSaturation records the current saturation level for an inference pool.

func RecordFlowControlRequestEnqueueDuration ¶

func RecordFlowControlRequestEnqueueDuration(
	fairnessID string, priority string, outcome string,
	duration time.Duration,
)

RecordFlowControlRequestEnqueueDuration records the duration a request was in the enqueuing process in the Flow Control layer.

func RecordFlowControlRequestQueueDuration ¶

func RecordFlowControlRequestQueueDuration(
	fairnessID, priority, outcome,
	inferencePool,
	modelName, targetModelName string,
	duration time.Duration,
)

RecordFlowControlRequestQueueDuration records the duration a request spent in the Flow Control layer.

func RecordInferenceExtensionInfo ¶

func RecordInferenceExtensionInfo(commitSha, buildRef string)

func RecordInferenceModelRewriteDecision ¶

func RecordInferenceModelRewriteDecision(modelRewriteName, modelName, targetModel string)

RecordInferenceModelRewriteDecision records the routing decision for InferenceModelRewrite.

func RecordInferencePoolAvgKVCache ¶

func RecordInferencePoolAvgKVCache(name string, utilization float64)

func RecordInferencePoolAvgQueueSize ¶

func RecordInferencePoolAvgQueueSize(name string, queueSize float64)

func RecordInferencePoolAvgRunningRequests ¶

func RecordInferencePoolAvgRunningRequests(name string, runningRequests float64)

func RecordInferencePoolReadyPods ¶

func RecordInferencePoolReadyPods(name string, runningPods float64)

func RecordInputTokens ¶

func RecordInputTokens(modelName, targetModelName, fairnessID, priority string, size int)

RecordInputTokens records input tokens count.

func RecordInterTokenLatency ¶

func RecordInterTokenLatency(ctx context.Context, modelName, targetModelName, fairnessID, priority string, itlSeconds float64) bool

RecordInterTokenLatency records the time between consecutive response body chunks for streaming requests.

func RecordNormalizedTimePerOutputToken ¶

func RecordNormalizedTimePerOutputToken(ctx context.Context, modelName, targetModelName, fairnessID, priority string, received time.Time, complete time.Time, outputTokenCount int) bool

RecordNormalizedTimePerOutputToken (NTPOT) records the normalized time per output token.

func RecordOutputTokens ¶

func RecordOutputTokens(modelName, targetModelName, fairnessID, priority string, size int)

RecordOutputTokens records output tokens count.

func RecordPluginProcessingLatency ¶

func RecordPluginProcessingLatency(extensionPoint, pluginType, pluginName string, duration time.Duration)

RecordPluginProcessingLatency records the processing latency for a plugin.

func RecordPromptCachedTokens ¶

func RecordPromptCachedTokens(modelName, targetModelName, fairnessID, priority string, size int)

RecordPromptCachedTokens records prompt cached tokens count.

func RecordRequestCounter ¶

func RecordRequestCounter(modelName, targetModelName, fairnessID string, priority int)

RecordRequestCounter records the number of requests.

func RecordRequestErrCounter ¶

func RecordRequestErrCounter(modelName, targetModelName, fairnessID, priority string, code string)

RecordRequestErrCounter records the number of error requests.

func RecordRequestLatencies ¶

func RecordRequestLatencies(ctx context.Context, modelName, targetModelName, fairnessID, priority string, received time.Time, complete time.Time) bool

RecordRequestLatencies records duration of request.

func RecordRequestSizes ¶

func RecordRequestSizes(modelName, targetModelName, fairnessID, priority string, reqSize int)

RecordRequestSizes records the request sizes.

func RecordRequestTPOT ¶

func RecordRequestTPOT(ctx context.Context, modelName, targetModelName, fairnessID, priority string, received time.Time, firstToken time.Time, complete time.Time, outputTokenCount int) bool

RecordRequestTPOT records the average time per output token.

func RecordRequestTTFT ¶

func RecordRequestTTFT(ctx context.Context, modelName, targetModelName, fairnessID, priority string, streaming bool, received time.Time, firstToken time.Time) bool

RecordRequestTTFT records the time to first token.

func RecordResponseSizes ¶

func RecordResponseSizes(modelName, targetModelName, fairnessID, priority string, size int)

RecordResponseSizes records the response sizes.

func RecordSchedulerAttempt ¶

func RecordSchedulerAttempt(err error, targetModelName string, result *fwksched.SchedulingResult)

RecordSchedulerAttempt records a scheduling attempt with status and endpoint information.

func RecordSchedulerE2ELatency ¶

func RecordSchedulerE2ELatency(duration time.Duration)

RecordSchedulerE2ELatency records the end-to-end scheduling latency.

func Register ¶

func Register(customCollectors ...prometheus.Collector)

Register all metrics.

func RegisterGRPCStreamMetrics ¶

func RegisterGRPCStreamMetrics()

RegisterGRPCStreamMetrics registers the ext_proc stream metrics; called only when enabled.

func Reset ¶

func Reset()

Just for integration test

func SubFlowControlQueueBytes ¶

func SubFlowControlQueueBytes(fairnessID, priority, inferencePool, modelName, targetModelName string, bytes uint64)

SubFlowControlQueueBytes decrements the Flow Control queue bytes gauge.

Types ¶

This section is empty.

Source Files ¶

View all Source files

Directories ¶

Path	Synopsis
collectors

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL