[{"data":1,"prerenderedAt":791},["ShallowReactive",2],{"\u002Ffeatures\u002Ftraces\u002Fquerying-traces-navigation":3,"\u002Ffeatures\u002Ftraces\u002Fquerying-traces":116},[4,8,28,38,52,56,60,70,74,78,82,86,90,94,98],{"title":5,"path":6,"stem":7},"Get started","\u002Ffeatures","features\u002Findex",{"title":9,"closed":10,"path":11,"stem":12,"children":13,"page":-1},"Traces",true,"\u002Ffeatures\u002Ftraces","features\u002F01.traces\u002Findex",[14,16,20,24],{"title":15,"path":11,"stem":12},"Introduction",{"title":17,"path":18,"stem":19},"Grouping & systems","\u002Ffeatures\u002Ftraces\u002Fgrouping","features\u002F01.traces\u002Fgrouping",{"title":21,"path":22,"stem":23},"Querying spans","\u002Ffeatures\u002Ftraces\u002Fquerying-spans","features\u002F01.traces\u002Fquerying-spans",{"title":25,"path":26,"stem":27},"Querying traces","\u002Ffeatures\u002Ftraces\u002Fquerying-traces","features\u002F01.traces\u002Fquerying-traces",{"title":29,"closed":10,"path":30,"stem":31,"children":32,"page":-1},"Logs","\u002Ffeatures\u002Flogs","features\u002F02.logs\u002Findex",[33,34],{"title":15,"path":30,"stem":31},{"title":35,"path":36,"stem":37},"Grouping rules","\u002Ffeatures\u002Flogs\u002Fgrouping","features\u002F02.logs\u002Fgrouping",{"title":39,"closed":10,"path":40,"stem":41,"children":42,"page":-1},"Metrics","\u002Ffeatures\u002Fmetrics","features\u002F03.metrics\u002Findex",[43,44,48],{"title":15,"path":40,"stem":41},{"title":45,"path":46,"stem":47},"Querying","\u002Ffeatures\u002Fmetrics\u002Fquerying","features\u002F03.metrics\u002F02.querying",{"title":49,"path":50,"stem":51},"PromQL compat","\u002Ffeatures\u002Fmetrics\u002Fpromql-compat","features\u002F03.metrics\u002F03.promql-compat",{"title":53,"path":54,"stem":55},"Alerts and Notifications","\u002Ffeatures\u002Falerting","features\u002F04.alerting",{"title":57,"path":58,"stem":59},"Chart annotations","\u002Ffeatures\u002Fannotations","features\u002F05.annotations",{"title":61,"path":62,"stem":63,"children":64,"closed":10},"Dashboards","\u002Ffeatures\u002Fdashboards","features\u002F06.dashboards\u002F1.index",[65,66],{"title":61,"path":62,"stem":63},{"title":67,"path":68,"stem":69},"YAML Templates","\u002Ffeatures\u002Fdashboards\u002Fyaml","features\u002F06.dashboards\u002F2.yaml",{"title":71,"path":72,"stem":73},"Transformations","\u002Ffeatures\u002Ftransformations","features\u002F07.transformations",{"title":75,"path":76,"stem":77},"Service graph","\u002Ffeatures\u002Fservice-graph","features\u002F08.service-graph",{"title":79,"path":80,"stem":81},"Grafana integration","\u002Ffeatures\u002Fgrafana","features\u002F09.grafana",{"title":83,"path":84,"stem":85},"Data fixtures","\u002Ffeatures\u002Ffixtures","features\u002F10.fixtures",{"title":87,"path":88,"stem":89},"JavaScript Source Maps","\u002Ffeatures\u002Fsourcemaps","features\u002F11.sourcemaps",{"title":91,"path":92,"stem":93},"MCP Server","\u002Ffeatures\u002Fmcp","features\u002F12.mcp",{"title":95,"path":96,"stem":97},"Searching","\u002Ffeatures\u002Fsearching","features\u002F4.searching",{"title":99,"closed":10,"path":100,"stem":101,"children":102,"page":115},"SSO","\u002Ffeatures\u002Fsso","features\u002Fsso",[103,107,111],{"title":104,"path":105,"stem":106},"Google","\u002Ffeatures\u002Fsso\u002Fgoogle","features\u002Fsso\u002F1.google",{"title":108,"path":109,"stem":110},"Okta","\u002Ffeatures\u002Fsso\u002Fokta","features\u002Fsso\u002F2.okta",{"title":112,"path":113,"stem":114},"Keycloak","\u002Ffeatures\u002Fsso\u002Fkeycloak","features\u002Fsso\u002F3.keycloak",false,{"page":117,"surround":786},{"id":118,"title":25,"body":119,"description":775,"extension":776,"image":777,"meta":778,"navigation":10,"path":26,"seo":784,"stem":27,"surround_disabled":115,"__hash__":785},"features\u002Ffeatures\u002F01.traces\u002Fquerying-traces.md",{"type":120,"value":121,"toc":756},"minimark",[122,136,139,146,151,158,169,175,181,275,279,290,414,418,423,426,432,436,439,445,449,452,458,461,465,468,474,478,481,487,491,494,500,504,507,513,517,524,530,533,537,543,717,724,730,734],[123,124,125,126,130,131,135],"p",{},"While the ",[127,128,129],"a",{"href":22},"span query language"," operates on individual spans and logs, ",[132,133,134],"strong",{},"trace queries"," allow you to select entire traces based on the spans, logs, and events they contain. This is useful for finding traces that match complex criteria across multiple services and operations.",[123,137,138],{},"For example, you can find all traces that contain a slow database query, have an error in a specific service, or involve a particular combination of microservices.",[123,140,141],{},[142,143],"img",{"alt":144,"src":145},"Querying Traces UI","\u002Ffeatures\u002Fquerying-traces\u002Fui.png",[147,148,150],"h2",{"id":149},"query-structure","Query Structure",[123,152,153,154,157],{},"A trace query consists of multiple rows, where each row targets a different part of the trace. The first row is always the ",[132,155,156],{},"root"," row, and subsequent rows define conditions on child spans, logs, or events.",[159,160,165],"pre",{"className":161,"code":163,"language":164},[162],"language-text","root: \u003Cquery for the root span>\n\u003Csystem> as \u003Calias>: \u003Cquery for matching spans>\n","text",[166,167,163],"code",{"__ignoreMap":168},"",[123,170,171,172,174],{},"Each row accepts the full ",[127,173,129],{"href":22}," including filters, aggregations, and groupings. A typical root row query looks like:",[159,176,179],{"className":177,"code":178,"language":164},[162],"root: perMin(count()) | quantiles(_dur_ms) | group by _group_id\n",[166,180,178],{"__ignoreMap":168},[182,183,184,197],"table",{},[185,186,187],"thead",{},[188,189,190,194],"tr",{},[191,192,193],"th",{},"Component",[191,195,196],{},"Description",[198,199,200,214,241,251],"tbody",{},[188,201,202,207],{},[203,204,205],"td",{},[166,206,156],{},[203,208,209,210,213],{},"Selects the root span of the trace. Use ",[166,211,212],{},"\u003Cempty>"," to match all root spans.",[188,215,216,221],{},[203,217,218],{},[166,219,220],{},"\u003Csystem>",[203,222,223,224,228,229,232,233,236,237,240],{},"A ",[127,225,227],{"href":226},"\u002Ffeatures\u002Ftraces\u002Fgrouping#span-systems","span system"," like ",[166,230,231],{},"db:postgresql",", ",[166,234,235],{},"rpc:all",", or ",[166,238,239],{},"log:error",".",[188,242,243,248],{},[203,244,245],{},[166,246,247],{},"as \u003Calias>",[203,249,250],{},"An optional alias for readability.",[188,252,253,258],{},[203,254,255],{},[166,256,257],{},"\u003Cquery>",[203,259,260,261,232,265,269,270,274],{},"Full query with ",[127,262,264],{"href":263},"\u002Ffeatures\u002Ftraces\u002Fquerying-spans#filters","where",[127,266,268],{"href":267},"\u002Ffeatures\u002Ftraces\u002Fquerying-spans#groupings","group by",", and ",[127,271,273],{"href":272},"\u002Ffeatures\u002Ftraces\u002Fquerying-spans#aggregations","aggregate"," clauses.",[147,276,278],{"id":277},"filtering-by-system","Filtering by System",[123,280,281,282,285,286,289],{},"Each non-root row targets spans belonging to a specific ",[127,283,284],{"href":226},"system",". You can use exact systems or wildcard systems with ",[166,287,288],{},":all",":",[182,291,292,302],{},[185,293,294],{},[188,295,296,299],{},[191,297,298],{},"System",[191,300,301],{},"Matches",[198,303,304,313,323,333,343,352,365,375,385,395,404],{},[188,305,306,310],{},[203,307,308],{},[166,309,231],{},[203,311,312],{},"PostgreSQL database spans only",[188,314,315,320],{},[203,316,317],{},[166,318,319],{},"db:mysql",[203,321,322],{},"MySQL database spans only",[188,324,325,330],{},[203,326,327],{},[166,328,329],{},"db:all",[203,331,332],{},"All database spans (PostgreSQL, MySQL, etc)",[188,334,335,340],{},[203,336,337],{},[166,338,339],{},"rpc:grpc",[203,341,342],{},"gRPC spans only",[188,344,345,349],{},[203,346,347],{},[166,348,235],{},[203,350,351],{},"All RPC spans",[188,353,354,359],{},[203,355,356],{},[166,357,358],{},"http:service1",[203,360,361,362],{},"HTTP spans for ",[166,363,364],{},"service1",[188,366,367,372],{},[203,368,369],{},[166,370,371],{},"httpserver:all",[203,373,374],{},"All HTTP server spans",[188,376,377,382],{},[203,378,379],{},[166,380,381],{},"messaging:kafka",[203,383,384],{},"Kafka messaging spans only",[188,386,387,392],{},[203,388,389],{},[166,390,391],{},"messaging:all",[203,393,394],{},"All messaging spans",[188,396,397,401],{},[203,398,399],{},[166,400,239],{},[203,402,403],{},"Error-level logs",[188,405,406,411],{},[203,407,408],{},[166,409,410],{},"log:warn",[203,412,413],{},"Warning-level logs",[147,415,417],{"id":416},"examples","Examples",[419,420,422],"h3",{"id":421},"traces-with-many-child-spans","Traces with Many Child Spans",[123,424,425],{},"Find traces that contain more than 1000 child spans, which can indicate runaway loops or excessive instrumentation:",[159,427,430],{"className":428,"code":429,"language":164},[162],"root: perMin(count()) | quantiles(_dur_ms) | _error_rate | group by _group_id\nspans:all as db: count() | having count() > 1000\n",[166,431,429],{"__ignoreMap":168},[419,433,435],{"id":434},"traces-with-the-number-of-database-queries","Traces with the Number of Database Queries",[123,437,438],{},"Return traces along with how many database queries each trace contains:",[159,440,443],{"className":441,"code":442,"language":164},[162],"root: perMin(count()) | quantiles(_dur_ms) | _error_rate | group by _group_id\ndb:all as db: count()\n",[166,444,442],{"__ignoreMap":168},[419,446,448],{"id":447},"traces-with-multiple-database-queries","Traces with Multiple Database Queries",[123,450,451],{},"Find traces that contain at least 2 PostgreSQL queries:",[159,453,456],{"className":454,"code":455,"language":164},[162],"root: perMin(count()) | quantiles(_dur_ms) | _error_rate | group by _group_id\ndb:postgresql as pg: having count() >= 2\n",[166,457,455],{"__ignoreMap":168},[123,459,460],{},"This is useful for identifying N+1 query problems or traces with excessive database calls.",[419,462,464],{"id":463},"traces-with-slow-service-calls","Traces with Slow Service Calls",[123,466,467],{},"Find traces where a specific service takes longer than 1 second to respond:",[159,469,472],{"className":470,"code":471,"language":164},[162],"root: perMin(count()) | quantiles(_dur_ms) | _error_rate | group by _group_id\nrpc:all as rpc: where service_name = \"foo\" | where _kind = \"server\" | where _dur_ms >= 1000\n",[166,473,471],{"__ignoreMap":168},[419,475,477],{"id":476},"traces-with-error-logs","Traces with Error Logs",[123,479,480],{},"Find traces that contain error logs mentioning \"timeout\":",[159,482,485],{"className":483,"code":484,"language":164},[162],"root: perMin(count()) | quantiles(_dur_ms) | _error_rate | group by _group_id\nlog:error as err: _display_name contains \"timeout\"\n",[166,486,484],{"__ignoreMap":168},[419,488,490],{"id":489},"traces-with-errors-in-a-specific-service","Traces with Errors in a Specific Service",[123,492,493],{},"Find traces where a particular service returned an error:",[159,495,498],{"className":496,"code":497,"language":164},[162],"root: perMin(count()) | quantiles(_dur_ms) | _error_rate | group by _group_id\nrpc:all as rpc: where service_name = \"payment-service\" | where _status_code = \"error\"\n",[166,499,497],{"__ignoreMap":168},[419,501,503],{"id":502},"traces-with-slow-root-spans","Traces with Slow Root Spans",[123,505,506],{},"Find traces where the root span exceeds 5 seconds:",[159,508,511],{"className":509,"code":510,"language":164},[162],"root: where _dur_ms >= 5000\n",[166,512,510],{"__ignoreMap":168},[419,514,516],{"id":515},"combining-multiple-conditions","Combining Multiple Conditions",[123,518,519,520,523],{},"Find traces that are slow ",[132,521,522],{},"and"," contain database errors:",[159,525,528],{"className":526,"code":527,"language":164},[162],"root: where _dur_ms >= 3000\ndb:all as db: where _status_code = \"error\"\n",[166,529,527],{"__ignoreMap":168},[123,531,532],{},"This selects traces where the overall duration exceeds 3 seconds and at least one database span has an error status.",[147,534,536],{"id":535},"available-clauses","Available Clauses",[123,538,539,540,542],{},"Each row — both root and child — supports the full ",[127,541,129],{"href":22}," including filters, aggregations, and groupings:",[182,544,545,557],{},[185,546,547],{},[188,548,549,552,555],{},[191,550,551],{},"Clause",[191,553,554],{},"Example",[191,556,196],{},[198,558,559,573,587,601,615,629,643,657,673,688,703],{},[188,560,561,565,570],{},[203,562,563],{},[166,564,264],{},[203,566,567],{},[166,568,569],{},"where service_name = \"foo\"",[203,571,572],{},"Filter spans by attribute values",[188,574,575,579,584],{},[203,576,577],{},[166,578,264],{},[203,580,581],{},[166,582,583],{},"where _dur_ms >= 1000",[203,585,586],{},"Filter by span duration",[188,588,589,593,598],{},[203,590,591],{},[166,592,264],{},[203,594,595],{},[166,596,597],{},"where _status_code = \"error\"",[203,599,600],{},"Filter by status code",[188,602,603,607,612],{},[203,604,605],{},[166,606,264],{},[203,608,609],{},[166,610,611],{},"where _kind = \"server\"",[203,613,614],{},"Filter by span kind",[188,616,617,621,626],{},[203,618,619],{},[166,620,264],{},[203,622,623],{},[166,624,625],{},"where _display_name contains \"text\"",[203,627,628],{},"Filter by display name",[188,630,631,635,640],{},[203,632,633],{},[166,634,268],{},[203,636,637],{},[166,638,639],{},"group by _group_id",[203,641,642],{},"Group spans by attribute",[188,644,645,649,654],{},[203,646,647],{},[166,648,268],{},[203,650,651],{},[166,652,653],{},"group by service_name",[203,655,656],{},"Group by service",[188,658,659,662,670],{},[203,660,661],{},"Aggregates",[203,663,664,232,667],{},[166,665,666],{},"count()",[166,668,669],{},"perMin(count())",[203,671,672],{},"Count and rate aggregations",[188,674,675,677,685],{},[203,676,661],{},[203,678,679,232,682],{},[166,680,681],{},"p50(_dur_ms)",[166,683,684],{},"quantiles(_dur_ms)",[203,686,687],{},"Duration percentiles",[188,689,690,695,700],{},[203,691,692],{},[166,693,694],{},"having",[203,696,697],{},[166,698,699],{},"having count() >= 2",[203,701,702],{},"Filter by aggregated span count",[188,704,705,709,714],{},[203,706,707],{},[166,708,694],{},[203,710,711],{},[166,712,713],{},"having p99(_dur_ms) >= 500",[203,715,716],{},"Filter by percentile duration",[123,718,719,720,723],{},"Multiple clauses can be chained with the pipe ",[166,721,722],{},"|"," operator:",[159,725,728],{"className":726,"code":727,"language":164},[162],"rpc:all as rpc: where service_name = \"foo\" | where _kind = \"server\" | where _dur_ms >= 1000\n",[166,729,727],{"__ignoreMap":168},[147,731,733],{"id":732},"see-also","See Also",[735,736,737,744,750],"ul",{},[738,739,740,743],"li",{},[127,741,742],{"href":22},"Querying spans and logs"," - Query language fundamentals",[738,745,746,749],{},[127,747,748],{"href":96},"Searching spans and logs"," - Simpler search syntax",[738,751,752,755],{},[127,753,754],{"href":18},"Grouping and systems"," - How span systems work",{"title":168,"searchDepth":757,"depth":758,"links":759},4,2,[760,761,762,773,774],{"id":149,"depth":758,"text":150},{"id":277,"depth":758,"text":278},{"id":416,"depth":758,"text":417,"children":763},[764,766,767,768,769,770,771,772],{"id":421,"depth":765,"text":422},3,{"id":434,"depth":765,"text":435},{"id":447,"depth":765,"text":448},{"id":463,"depth":765,"text":464},{"id":476,"depth":765,"text":477},{"id":489,"depth":765,"text":490},{"id":502,"depth":765,"text":503},{"id":515,"depth":765,"text":516},{"id":535,"depth":758,"text":536},{"id":732,"depth":758,"text":733},"Query distributed traces by filtering spans, logs, and events across services using the multi-row trace query syntax.","md",null,{"readingTime":779},{"text":780,"minutes":781,"time":782,"words":783},"3 min read",2.94,176400,588,{"title":25,"description":775},"5k5orn8DmT7AOdqOrFenFy3iFN3Aknr6Uy9OBU7D95g",[787,789],{"title":21,"path":22,"stem":23,"description":788,"children":-1},"Master the Uptrace span query language including identifiers, filters, grouping, and aggregates for traces, logs, and events.",{"title":15,"path":30,"stem":31,"description":790,"children":-1},"Browse grouped logs and errors in Uptrace, filter by service or severity, search by text or attribute, and inspect individual records with aggregated stats and per-entry details.",1778588617786]