OpenTelemetry Sampling [Erlang/Elixir]

What is sampling?

Sampling is a process that restricts the amount of traces that are generated by a system. In high-volume applications, collecting 100% of traces can be expensive and unnecessary. Sampling allows you to collect a representative subset of traces while reducing costs and performance overhead.

Erlang/Elixir sampling

OpenTelemetry Erlang/Elixir SDK provides head-based sampling capabilities where the sampling decision is made at the beginning of a trace. By default, all spans are sampled (100% sampling rate). The SDK offers several built-in samplers that can be configured through application configuration or environment variables.

Built-in samplers

ParentBasedSampler

The most commonly used sampler for head sampling. It uses the sampling decision of the span's parent, or uses a root sampler when there's no parent:

elixir Elixir

# config/runtime.exs
config :opentelemetry,
  sampler: {:parent_based, %{
    root: {:trace_id_ratio_based, 0.10},
    remote_parent_sampled: :always_on,
    remote_parent_not_sampled: :always_off,
    local_parent_sampled: :always_on,
    local_parent_not_sampled: :always_off
  }}

TraceIdRatioBasedSampler

Deterministically samples a percentage of traces based on the trace ID:

elixir Elixir

# config/runtime.exs
config :opentelemetry,
  sampler: {:trace_id_ratio_based, 0.25}  # Sample 25% of traces

AlwaysOnSampler

Samples 100% of traces. This is the default behavior:

elixir Elixir

# config/runtime.exs
config :opentelemetry,
  sampler: :always_on

AlwaysOffSampler

Disables tracing completely (0% sampling):

elixir Elixir

# config/runtime.exs
config :opentelemetry,
  sampler: :always_off

Configuration in Erlang/Elixir

Environment variable

You can configure sampling using environment variables instead of application configuration:

bash

# ParentBased with TraceIdRatio - 10% sampling
export OTEL_TRACES_SAMPLER="parentbased_traceidratio"
export OTEL_TRACES_SAMPLER_ARG="0.1"

# ParentBased with AlwaysOn
export OTEL_TRACES_SAMPLER="parentbased_always_on"

# Simple TraceIdRatio - 5% sampling
export OTEL_TRACES_SAMPLER="traceidratio"
export OTEL_TRACES_SAMPLER_ARG="0.05"

# AlwaysOn/AlwaysOff
export OTEL_TRACES_SAMPLER="always_on"

Programmatic configuration

elixir

# config/runtime.exs
import Config

# Different sampling for different environments
sampling_config = case config_env() do
  :dev ->
    # Sample everything in development
    :always_on

  :test ->
    # Disable sampling in tests
    :always_off

  :prod ->
    # Use parent-based sampling with 5% for root spans
    {:parent_based, %{
      root: {:trace_id_ratio_based, 0.05},
      remote_parent_sampled: :always_on,
      remote_parent_not_sampled: :always_off,
      local_parent_sampled: :always_on,
      local_parent_not_sampled: :always_off
    }}
end

config :opentelemetry,
  sampler: sampling_config,
  span_processor: :batch,
  traces_exporter: :otlp

Custom sampler

Create custom sampling logic by implementing the :otel_sampler behaviour:

elixir Elixir

defmodule MyApp.CustomSampler do
  require OpenTelemetry.Tracer, as: Tracer
  @behaviour :otel_sampler

  @impl :otel_sampler
  def setup(_sampler_opts), do: []

  @impl :otel_sampler
  def description(_sampler_config), do: "MyApp.CustomSampler"

  @impl :otel_sampler
  def should_sample(ctx, _trace_id, _links, span_name, _span_kind, attributes, _sampler_config) do
    tracestate = Tracer.current_span_ctx(ctx) |> OpenTelemetry.Span.tracestate()

    sample_decision = cond do
      # Always sample error traces
      has_error_attribute?(attributes) -> true
      # Always sample critical operations
      String.contains?(span_name, ["critical", "payment", "auth"]) -> true
      # Sample 10% of regular traces
      :rand.uniform() < 0.1 -> true
      true -> false
    end

    case sample_decision do
      true -> {:record_and_sample, [], tracestate}
      false -> {:drop, [], tracestate}
    end
  end

  defp has_error_attribute?(attributes) do
    Enum.any?(attributes, fn {key, value} -> key == "error" and value == true end)
  end
end

# Configuration
config :opentelemetry,
  sampler: {:parent_based, %{root: {MyApp.CustomSampler, %{custom_arg: "value"}}}},
  span_processor: :batch,
  traces_exporter: :otlp

Advanced scenarios

Sampling based on attributes

elixir Elixir

defmodule MyApp.ConditionalSampler do
  @behaviour :otel_sampler

  @impl :otel_sampler
  def setup(_opts), do: []

  @impl :otel_sampler
  def description(_config), do: "ConditionalSampler"

  @impl :otel_sampler
  def should_sample(ctx, _trace_id, _links, span_name, _span_kind, _attributes, _config) do
    tracestate = OpenTelemetry.Tracer.current_span_ctx(ctx)
                 |> OpenTelemetry.Span.tracestate()

    should_sample = cond do
      # Always sample requests to admin endpoints
      String.contains?(span_name, "/admin") -> true
      # Sample 50% of API requests
      String.starts_with?(span_name, "/api") -> :rand.uniform() < 0.5
      # Don't sample health checks
      span_name == "/health" -> false
      # Default sampling rate
      true -> :rand.uniform() < 0.1
    end

    case should_sample do
      true -> {:record_and_sample, [], tracestate}
      false -> {:drop, [], tracestate}
    end
  end
end

Production deployment

Phoenix application with sampling

elixir Application

# lib/my_app/application.ex
defmodule MyApp.Application do
  use Application

  def start(_type, _args) do
    # Setup OpenTelemetry instrumentations
    :opentelemetry_cowboy.setup()
    OpentelemetryPhoenix.setup(adapter: :cowboy2)
    OpentelemetryEcto.setup([:my_app, :repo])

    children = [
      MyApp.Repo,
      {Phoenix.PubSub, name: MyApp.PubSub},
      MyAppWeb.Endpoint
    ]

    opts = [strategy: :one_for_one, name: MyApp.Supervisor]
    Supervisor.start_link(children, opts)
  end
end

Manual span creation with sampling checks

elixir Elixir

defmodule MyApp.OrderService do
  require OpenTelemetry.Tracer

  def process_order(order_data) do
    OpenTelemetry.Tracer.with_span "process_order" do
      # Check if span is being recorded to avoid expensive operations
      current_span = OpenTelemetry.Tracer.current_span_ctx()

      if OpenTelemetry.Span.is_recording(current_span) do
        OpenTelemetry.Tracer.set_attributes(%{
          "order.id" => order_data.id,
          "order.amount" => order_data.amount,
          "order.customer_type" => order_data.customer_type
        })
      end

      # Business logic
      result = perform_order_processing(order_data)

      # Only add expensive attributes if recording
      if OpenTelemetry.Span.is_recording(current_span) do
        OpenTelemetry.Tracer.add_event("order_processed", %{
          "result.status" => result.status
        })
      end

      result
    end
  end

  defp perform_order_processing(_order_data) do
    # Simulate processing
    :timer.sleep(100)
    %{status: "completed"}
  end
end

Monitoring sampling

Check sampling effectiveness

elixir

defmodule MyApp.SamplingChecker do
  require Logger

  def check_sampling_rate do
    # Get span statistics from your monitoring system
    stats = get_span_statistics()

    if stats.total > 0 do
      sampling_rate = stats.sampled / stats.total * 100

      Logger.info("Current sampling rate: #{Float.round(sampling_rate, 2)}% " <>
                  "(#{stats.sampled}/#{stats.total} spans)")

      # Alert if sampling rate is unexpected
      expected_rate = get_expected_sampling_rate()

      if abs(sampling_rate - expected_rate) > 5.0 do
        Logger.warning("Sampling rate #{Float.round(sampling_rate, 2)}% " <>
                       "differs from expected #{expected_rate}%")
      end
    end
  end

  defp get_span_statistics do
    # This would be implemented based on your monitoring setup
    %{total: 1000, sampled: 100}
  end

  defp get_expected_sampling_rate do
    case Application.get_env(:my_app, :environment) do
      :prod -> 10.0
      :staging -> 50.0
      _ -> 100.0
    end
  end
end

OpenTelemetry APM

Uptrace is a OpenTelemetry backend that supports distributed tracing, metrics, and logs. You can use it to monitor applications and troubleshoot issues.

Uptrace Overview

Uptrace comes with an intuitive query builder, rich dashboards, alerting rules with notifications, and integrations for most languages and frameworks.

Uptrace can process billions of spans and metrics on a single server and allows you to monitor your applications at 10x lower cost.

In just a few minutes, you can try Uptrace by visiting the cloud demo (no login required) or running it locally with Docker. The source code is available on GitHub.

What's next?

PropagationLearn how to implement context propagation for distributed tracing in Erlang/Elixir OpenTelemetry applications.

Get startedStep-by-step guide to install and configure OpenTelemetry Go SDKs, export telemetry to Uptrace, and verify that traces, metrics, and logs arrive via OTLP.