OpenTelemetry Rust Tracing API

This document teaches you how to use the OpenTelemetry Rust API to instrument your applications with distributed tracing. To learn how to install and configure the OpenTelemetry Rust SDK, see Getting started with OpenTelemetry Rust.

Installation

Add the required dependencies to your Cargo.toml:

toml

[dependencies]
opentelemetry = "0.21"
opentelemetry_sdk = { version = "0.21", features = ["rt-tokio"] }
tokio = { version = "1.0", features = ["full"] }

Quickstart

Step 1. Let's instrument this async function:

rust

async fn create_user(name: String, email: String) -> Result<User, UserError> {
    User::create(name, email).await
}

Step 2. Wrap the operation with a span:

rust

use opentelemetry::{global, trace::{TraceContextExt, Tracer}, KeyValue};

async fn create_user(name: String, email: String) -> Result<User, UserError> {
    let tracer = global::tracer("my-rust-app");

    tracer.in_span("create-user", |_cx| async move {
        User::create(name, email).await
    }).await
}

Step 3. Add error handling and status:

rust

use opentelemetry::trace::Status;

async fn create_user(name: String, email: String) -> Result<User, UserError> {
    let tracer = global::tracer("my-rust-app");

    tracer.in_span("create-user", |cx| async move {
        let span = cx.span();

        match User::create(name, email).await {
            Ok(user) => {
                span.set_status(Status::Ok);
                Ok(user)
            }
            Err(e) => {
                span.set_status(Status::error(format!("User creation failed: {}", e)));
                Err(e)
            }
        }
    }).await
}

Step 4. Add contextual attributes:

rust

async fn create_user(name: String, email: String) -> Result<User, UserError> {
    let tracer = global::tracer("my-rust-app");

    tracer.in_span("create-user", |cx| async move {
        let span = cx.span();

        if span.is_recording() {
            span.set_attribute(KeyValue::new("user.name", name.clone()));
            span.set_attribute(KeyValue::new("user.email", email.clone()));
        }

        match User::create(name, email).await {
            Ok(user) => {
                if span.is_recording() {
                    span.set_attribute(KeyValue::new("user.id", user.id.to_string()));
                }
                span.set_status(Status::Ok);
                Ok(user)
            }
            Err(e) => {
                span.set_status(Status::error(format!("Failed: {}", e)));
                Err(e)
            }
        }
    }).await
}

Tracer

Create a tracer to start generating spans:

rust

use opentelemetry::{global, trace::Tracer};

// Basic tracer
let tracer = global::tracer("my-rust-app");

// With version
let tracer = global::tracer_with_version("my-rust-app", "1.0.0");

Global Tracer Pattern

rust

use std::sync::OnceLock;

static TRACER: OnceLock<Box<dyn Tracer + Send + Sync>> = OnceLock::new();

pub fn tracer() -> &'static dyn Tracer {
    TRACER.get_or_init(|| global::tracer("my-rust-app")).as_ref()
}

Creating Spans

Basic Spans

rust

async fn handle_request() -> Result<String, MyError> {
    let tracer = global::tracer("my-app");

    tracer.in_span("handle-request", |cx| async move {
        let span = cx.span();
        span.set_attribute(KeyValue::new("operation.type", "request"));

        do_work().await
    }).await
}

Span Kinds

rust

use opentelemetry::trace::{SpanBuilder, SpanKind};

// Server span (incoming requests)
tracer.span_builder("handle-request")
    .with_kind(SpanKind::Server)
    .start(&tracer);

// Client span (outgoing requests)
tracer.span_builder("http-request")
    .with_kind(SpanKind::Client)
    .start(&tracer);

// Internal operations (default)
tracer.in_span("internal-work", |_cx| async move {
    process_data().await
}).await

Manual Span Management

rust

async fn manual_span() -> Result<(), MyError> {
    let tracer = global::tracer("my-app");

    let span = tracer.span_builder("manual-operation").start(&tracer);
    let cx = opentelemetry::Context::current_with_span(span);

    async move {
        let span = cx.span();
        span.set_status(Status::Ok);
        Ok(())
    }.with_context(cx).await
}

Adding Span Attributes

rust

async fn with_attributes() -> Result<(), MyError> {
    let tracer = global::tracer("my-app");

    tracer.in_span("operation", |cx| async move {
        let span = cx.span();

        if span.is_recording() {
            // Single attribute
            span.set_attribute(KeyValue::new("user.id", "12345"));

            // Multiple attributes
            span.set_attributes(vec![
                KeyValue::new("http.method", "GET"),
                KeyValue::new("http.status_code", 200),
                KeyValue::new("user.role", "admin"),
            ]);
        }

        Ok(())
    }).await
}

Adding Span Events

rust

async fn with_events() -> Result<(), MyError> {
    let tracer = global::tracer("my-app");

    tracer.in_span("file-processing", |cx| async move {
        let span = cx.span();

        // Simple event
        span.add_event("processing.started", vec![]);

        // Event with attributes
        span.add_event("user.authenticated", vec![
            KeyValue::new("user.id", "12345"),
            KeyValue::new("auth.method", "oauth"),
        ]);

        // Process file...
        Ok(())
    }).await
}

Setting Span Status

rust

use opentelemetry::trace::Status;

async fn with_status() -> Result<String, MyError> {
    let tracer = global::tracer("my-app");

    tracer.in_span("database-query", |cx| async move {
        let span = cx.span();

        match perform_query().await {
            Ok(result) => {
                span.set_status(Status::Ok);
                Ok(result)
            }
            Err(e) => {
                span.set_status(Status::error(format!("Query failed: {}", e)));
                Err(e)
            }
        }
    }).await
}

Current Span and Context

Getting Current Span

rust

use opentelemetry::{Context, trace::TraceContextExt};

async fn add_user_info(user: &User) {
    let current_context = Context::current();
    let span = current_context.span();

    if span.is_recording() {
        span.set_attribute(KeyValue::new("user.id", user.id.to_string()));
    }
}

Context Nesting

rust

async fn nested_operations() -> Result<(), MyError> {
    let tracer = global::tracer("my-app");

    // Parent span
    tracer.in_span("parent-operation", |_parent_cx| async move {
        // Child span - automatically nested
        tracer.in_span("child-operation", |_child_cx| async move {
            // Both spans are properly linked
            Ok(())
        }).await
    }).await
}

Async Context Propagation

rust

use tokio::task;

async fn spawn_with_context() -> Result<(), MyError> {
    let tracer = global::tracer("my-app");

    tracer.in_span("parent-task", |_cx| async move {
        // Context is automatically inherited in spawned tasks
        let handle = task::spawn(async move {
            let current_context = Context::current();
            let span = current_context.span();

            if span.is_recording() {
                span.add_event("spawned-task-running", vec![]);
            }

            "Task completed"
        });

        let result = handle.await?;
        println!("Result: {}", result);
        Ok(())
    }).await
}

Best Practices

Error Handling Pattern

rust

async fn service_operation(params: &ServiceParams) -> Result<ServiceResult, ServiceError> {
    let tracer = global::tracer("service");

    tracer.in_span("service-operation", |cx| async move {
        let span = cx.span();

        if span.is_recording() {
            span.set_attribute(KeyValue::new("operation.type", "service_call"));
        }

        match perform_operation(params).await {
            Ok(result) => {
                span.set_status(Status::Ok);
                if span.is_recording() {
                    span.set_attribute(KeyValue::new("operation.result", "success"));
                }
                Ok(result)
            }
            Err(e) => {
                span.set_status(Status::error(format!("Operation failed: {}", e)));
                if span.is_recording() {
                    span.set_attribute(KeyValue::new("error.type", e.error_type()));
                }
                Err(e)
            }
        }
    }).await
}

Attribute Optimization

rust

async fn optimized_spans() -> Result<(), MyError> {
    let tracer = global::tracer("my-app");

    tracer.in_span("expensive-operation", |cx| async move {
        let span = cx.span();

        // Only do expensive work if span is being recorded
        if span.is_recording() {
            let expensive_data = calculate_metrics().await;
            span.set_attribute(KeyValue::new("computed.value", expensive_data));
        }

        // Always do the main work
        perform_operation().await
    }).await
}

Span Naming

rust

// Good: Low cardinality, descriptive
tracer.in_span("GET /users/:id", |cx| async move {
    let span = cx.span();
    if span.is_recording() {
        span.set_attribute(KeyValue::new("user.id", user_id.to_string()));
    }
    get_user(user_id).await
}).await

// Avoid: High cardinality
// tracer.in_span(&format!("GET /users/{}", user_id), |_cx| async move { ... })

Integration Examples

HTTP Client

rust

use reqwest::Client;

pub struct TracedHttpClient {
    client: Client,
    tracer: Box<dyn opentelemetry::trace::Tracer + Send + Sync>,
}

impl TracedHttpClient {
    pub fn new() -> Self {
        Self {
            client: Client::new(),
            tracer: global::tracer("http-client"),
        }
    }

    pub async fn get(&self, url: &str) -> Result<reqwest::Response, reqwest::Error> {
        self.tracer.in_span("HTTP GET", |cx| async move {
            let span = cx.span();

            if span.is_recording() {
                span.set_attributes(vec![
                    KeyValue::new("http.method", "GET"),
                    KeyValue::new("http.url", url.to_string()),
                ]);
            }

            match self.client.get(url).send().await {
                Ok(response) => {
                    if span.is_recording() {
                        span.set_attribute(KeyValue::new("http.status_code", response.status().as_u16() as i64));
                    }
                    span.set_status(Status::Ok);
                    Ok(response)
                }
                Err(e) => {
                    span.set_status(Status::error(format!("HTTP request failed: {}", e)));
                    Err(e)
                }
            }
        }).await
    }
}

Axum Integration

rust

use axum::{extract::Path, response::Json, routing::get, Router};
use serde_json::{json, Value};
use tracing::{info, instrument};

// Use #[instrument] for automatic span creation
#[instrument(skip_all, fields(user_id = %user_id))]
async fn get_user(Path(user_id): Path<u32>) -> Result<Json<Value>, axum::http::StatusCode> {
    info!("Fetching user data");

    match fetch_user(user_id).await {
        Ok(user) => Ok(Json(json!({ "id": user_id, "name": user.name }))),
        Err(_) => Err(axum::http::StatusCode::NOT_FOUND),
    }
}

#[instrument]
async fn health_check() -> Json<Value> {
    Json(json!({ "status": "healthy" }))
}

pub fn create_app() -> Router {
    Router::new()
        .route("/users/:id", get(get_user))
        .route("/health", get(health_check))
}

Performance Considerations

Conditional Instrumentation

rust

pub struct PerformanceCriticalService {
    enable_tracing: bool,
    tracer: Option<Box<dyn opentelemetry::trace::Tracer + Send + Sync>>,
}

impl PerformanceCriticalService {
    pub fn new(enable_tracing: bool) -> Self {
        let tracer = if enable_tracing {
            Some(global::tracer("critical-service"))
        } else {
            None
        };

        Self { enable_tracing, tracer }
    }

    pub async fn perform_operation(&self) -> Result<String, MyError> {
        if let Some(tracer) = &self.tracer {
            tracer.in_span("critical-operation", |_cx| async move {
                self.do_work().await
            }).await
        } else {
            self.do_work().await
        }
    }

    async fn do_work(&self) -> Result<String, MyError> {
        Ok("Operation completed".to_string())
    }
}

Sampling Awareness

rust

async fn sampling_aware() -> Result<(), MyError> {
    let tracer = global::tracer("my-app");

    tracer.in_span("operation", |cx| async move {
        let span = cx.span();

        // Only do expensive work if span is being recorded
        if span.is_recording() {
            let expensive_data = calculate_expensive_metrics().await;
            span.set_attribute(KeyValue::new("expensive.data", expensive_data));
        }

        perform_main_operation().await
    }).await
}

Auto-instrumentation

When possible, prefer explicit instrumentation for better control. Auto-instrumentation works well for common frameworks and libraries.

OpenTelemetry Rust supports automatic instrumentation through the tracing ecosystem.

Installation

toml

[dependencies]
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
tracing-opentelemetry = "0.22"
opentelemetry = "0.21"
opentelemetry_sdk = { version = "0.21", features = ["rt-tokio"] }
opentelemetry-otlp = "0.14"

Supported Libraries

Library	Instrumentation
`reqwest`	Automatic HTTP request spans
`sqlx`	Database query tracing
`axum`	HTTP request/response tracing
`tokio`	Task and runtime tracing
`tower-http`	HTTP middleware tracing

Setup

rust

use opentelemetry::{global, KeyValue};
use opentelemetry_sdk::{trace, Resource};
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};

#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync + 'static>> {
    // Initialize OpenTelemetry tracer
    let tracer = opentelemetry_otlp::new_pipeline()
        .tracing()
        .with_exporter(
            opentelemetry_otlp::new_exporter()
                .tonic()
                .with_endpoint("https://api.uptrace.dev:4317")
        )
        .with_trace_config(
            trace::config()
                .with_resource(Resource::new(vec![
                    KeyValue::new("service.name", "rust-app"),
                    KeyValue::new("service.version", "1.0.0"),
                ]))
        )
        .install_batch(opentelemetry_sdk::runtime::Tokio)?;

    // Create tracing layer
    let telemetry = tracing_opentelemetry::layer().with_tracer(tracer);

    // Initialize subscriber
    tracing_subscriber::registry()
        .with(telemetry)
        .with(tracing_subscriber::EnvFilter::from_default_env())
        .init();

    // Your application code - automatically traced
    run_app().await?;

    global::shutdown_tracer_provider();
    Ok(())
}

Usage

rust

use tracing::{info, instrument};

#[instrument(skip_all, fields(user_id = %user_id))]
async fn get_user(user_id: u32) -> Result<User, UserError> {
    info!("Fetching user data");

    // SQLx queries will be automatically traced
    let user = sqlx::query_as::<_, User>("SELECT * FROM users WHERE id = $1")
        .bind(user_id)
        .fetch_one(&pool)
        .await?;

    Ok(user)
}

OpenTelemetry APM

Uptrace is a OpenTelemetry APM that supports distributed tracing, metrics, and logs. You can use it to monitor applications and troubleshoot issues.

Uptrace Overview

Uptrace comes with an intuitive query builder, rich dashboards, alerting rules with notifications, and integrations for most languages and frameworks.

Uptrace can process billions of spans and metrics on a single server and allows you to monitor your applications at 10x lower cost.

In just a few minutes, you can try Uptrace by visiting the cloud demo (no login required) or running it locally with Docker. The source code is available on GitHub.

What's next?

Get startedStep-by-step guide to install and configure OpenTelemetry Rust SDKs, export telemetry to Uptrace, and verify that traces, metrics, and logs arrive via OTLP.

Metrics APICollect Rust application metrics with the OpenTelemetry Meter API, configure instruments, and forward data to Uptrace dashboards.

Adding Span Attributes

Adding Span Events

Setting Span Status

Current Span and Context

Best Practices

Integration Examples

Performance Considerations

Auto-instrumentation

OpenTelemetry APM

What's next?