Filter by trace ID, model spans, metadata, MCP, error type, client disconnects, and TTFT timeouts.
Advanced Use Cases
Filter by Specific Trace ID
Fetch spans of a specific traceId
from truefoundry import clientfrom truefoundry_sdk import SortDirectionspans = client.traces.query_spans( data_routing_destination="default", start_time="2025-10-08T00:00:00.000Z", end_time="2025-10-08T23:59:59.999Z", trace_ids=[ "0199c25e124a70989b0455584fbbf7b7" ], limit=200, sort_direction=SortDirection.DESC)# Process all spans across all pagesfor span in spans: print(span.span_name, span.duration, span.span_attributes.get("tfy.span_type"))
import requestspage_token = Nonedone = Falsewhile not done: # Make API request response = requests.post( "https://{control_plane_url}/api/svc/v1/spans/query", headers={ "Authorization": "Bearer YOUR_API_TOKEN", "Content-Type": "application/json" }, json={ "dataRoutingDestination": "default", "startTime": "2025-10-08T00:00:00.000Z", "endTime": "2025-10-08T23:59:59.999Z", "traceIds": ["0199c25e124a70989b0455584fbbf7b7"], "limit": 200, "sortDirection": "desc", "pageToken": page_token } ) response.raise_for_status() data = response.json() for span in data['data']: print(span['spanName'], span['duration'], span['spanAttributes'].get("tfy.span_type")) page_token = data['pagination'].get("nextPageToken") done = page_token is Noneprint("Fetch spans by trace ID completed!")
Fetch all model spans
Filter spans by model spanType using the tfy.span_type span attribute filter with value Model.
from truefoundry import clientfrom truefoundry_sdk import SortDirection, SpanAttributeFilterfrom truefoundry_sdk.types.span_attribute_filter_operator import SpanAttributeFilterOperatorspans = client.traces.query_spans( data_routing_destination="default", start_time="2025-10-08T00:00:00.000Z", end_time="2025-10-08T23:59:59.999Z", filters=[ SpanAttributeFilter(span_attribute_key="tfy.span_type", operator=SpanAttributeFilterOperator.EQUAL, value="Model"), ], limit=200, sort_direction=SortDirection.DESC)# Process all spans across all pagesfor span in spans: print(span.span_name, span.span_attributes.get("tfy.span_type"))
import requestspage_token = Nonedone = Falsewhile not done: # Make API request response = requests.post( "https://{control_plane_url}/api/svc/v1/spans/query", headers={ "Authorization": "Bearer YOUR_API_TOKEN", "Content-Type": "application/json" }, json={ "dataRoutingDestination": "default", "startTime": "2025-10-08T00:00:00.000Z", "endTime": "2025-10-08T23:59:59.999Z", "filters": [ {"spanAttributeKey": "tfy.span_type", "operator": "EQUAL", "value": "Model"}, ], "limit": 200, "sortDirection": "desc", "pageToken": page_token } ) response.raise_for_status() data = response.json() for span in data['data']: print(span['spanName'], span['spanAttributes'].get("tfy.span_type")) page_token = data['pagination'].get("nextPageToken") done = page_token is Noneprint("Fetch spans by model spanType completed!")
Fetch requests with Gateway request metadata
Filter spans based on custom metadata keys and values that were passed to Gateway requests using the X-TFY-METADATA header.
from truefoundry import clientfrom truefoundry_sdk import SortDirection, GatewayRequestMetadataFilterfrom truefoundry_sdk.types.gateway_request_metadata_filter_operator import GatewayRequestMetadataFilterOperatorspans = client.traces.query_spans( data_routing_destination="default", start_time="2025-10-08T00:00:00.000Z", end_time="2025-10-08T23:59:59.999Z", filters=[ GatewayRequestMetadataFilter(gateway_request_metadata_key="application", operator=GatewayRequestMetadataFilterOperator.EQUAL, value="booking-bot"), GatewayRequestMetadataFilter(gateway_request_metadata_key="environment", operator=GatewayRequestMetadataFilterOperator.IN, value=["staging", "production"]), ], limit=200, sort_direction=SortDirection.DESC)# Process all spans across all pagesfor span in spans: print(span.span_name, span.span_attributes.get("tfy.span_type"))
import requestspage_token = Nonedone = Falsewhile not done: # Make API request response = requests.post( "https://{control_plane_url}/api/svc/v1/spans/query", headers={ "Authorization": "Bearer YOUR_API_TOKEN", "Content-Type": "application/json" }, json={ "dataRoutingDestination": "default", "startTime": "2025-10-08T00:00:00.000Z", "endTime": "2025-10-08T23:59:59.999Z", "filters": [ {"gatewayRequestMetadataKey": "application", "operator": "EQUAL", "value": "booking-bot"}, {"gatewayRequestMetadataKey": "environment", "operator": "IN", "value": ["staging", "production"]}, ], "limit": 200, "sortDirection": "desc", "pageToken": page_token } ) response.raise_for_status() data = response.json() for span in data['data']: print(span['spanName'], span['spanAttributes'].get("tfy.span_type")) page_token = data['pagination'].get("nextPageToken") done = page_token is Noneprint("Fetch spans with Gateway request metadata completed!")
Fetch spans with MCP in span name
Filter spans that have MCP in the span name.
from truefoundry import clientfrom truefoundry_sdk import SortDirection, SpanFieldFilterfrom truefoundry_sdk.types.span_field_filter_span_field_name import SpanFieldFilterSpanFieldNamefrom truefoundry_sdk.types.span_field_filter_operator import SpanFieldFilterOperatorspans = client.traces.query_spans( data_routing_destination="default", start_time="2025-10-08T00:00:00.000Z", end_time="2025-10-08T23:59:59.999Z", filters=[ SpanFieldFilter(span_field_name=SpanFieldFilterSpanFieldName.SPAN_NAME, operator=SpanFieldFilterOperator.STRING_CONTAINS, value="MCP"), ], limit=200, sort_direction=SortDirection.DESC)# Process all spans across all pagesfor span in spans: print(span.span_name, span.span_attributes.get("tfy.span_type"))
import requestspage_token = Nonedone = Falsewhile not done: # Make API request response = requests.post( "https://{control_plane_url}/api/svc/v1/spans/query", headers={ "Authorization": "Bearer YOUR_API_TOKEN", "Content-Type": "application/json" }, json={ "dataRoutingDestination": "default", "startTime": "2025-10-08T00:00:00.000Z", "endTime": "2025-10-08T23:59:59.999Z", "filters": [ {"spanFieldName": "spanName", "operator": "STRING_CONTAINS", "value": "MCP"}, ], "limit": 200, "sortDirection": "desc", "pageToken": page_token } ) response.raise_for_status() data = response.json() for span in data['data']: print(span['spanName'], span['spanAttributes'].get("tfy.span_type")) page_token = data['pagination'].get("nextPageToken") done = page_token is Noneprint("Fetch MCP spans completed!")
Filter model spans by model name
Fetch model spans by filtering with tfy.span_type span attribute filter with value Model, and then further filter by model name using the tfy.model.name span attribute filter.
from truefoundry import clientfrom truefoundry_sdk import SortDirection, SpanAttributeFilterfrom truefoundry_sdk.types.span_attribute_filter_operator import SpanAttributeFilterOperatorspans = client.traces.query_spans( data_routing_destination="default", start_time="2025-10-08T00:00:00.000Z", end_time="2025-10-08T23:59:59.999Z", filters=[ SpanAttributeFilter(span_attribute_key="tfy.model.name", operator=SpanAttributeFilterOperator.EQUAL, value="openai-main/gpt-4"), SpanAttributeFilter(span_attribute_key="tfy.span_type", operator=SpanAttributeFilterOperator.EQUAL, value="Model"), ], limit=200, sort_direction=SortDirection.DESC)# Process all spans across all pagesfor span in spans: print(span.span_name, span.span_attributes.get("tfy.model.name"))
import requestspage_token = Nonedone = Falsewhile not done: # Make API request response = requests.post( "https://{control_plane_url}/api/svc/v1/spans/query", headers={ "Authorization": "Bearer YOUR_API_TOKEN", "Content-Type": "application/json" }, json={ "dataRoutingDestination": "default", "startTime": "2025-10-08T00:00:00.000Z", "endTime": "2025-10-08T23:59:59.999Z", "filters": [ {"spanAttributeKey": "tfy.model.name", "operator": "EQUAL", "value": "openai-main/gpt-4"}, {"spanAttributeKey": "tfy.span_type", "operator": "EQUAL", "value": "Model"}, ], "limit": 200, "sortDirection": "desc", "pageToken": page_token } ) response.raise_for_status() data = response.json() for span in data['data']: print(span['spanName'], span['spanAttributes'].get("tfy.model.name")) page_token = data['pagination'].get("nextPageToken") done = page_token is Noneprint("Fetch spans by model name completed!")
Filter missing-model vs forbidden spans
Requests for a model that does not exist and requests the caller is not allowed to use both return HTTP 403 with the same client message. Filter on tfy.error_type (tfy-model-not-found vs tfy-forbidden). The attribute is on the Model span. See Authorization vs missing model.
A client that hangs up mid-stream is recorded as tfy.error_type=client-disconnected with HTTP 499, and may still carry cost. Filter on the Model span. See Cancelled and disconnected streams.
A streaming request that never produced a first chunk within x-tfy-ttft-timeout-ms is recorded as tfy.error_type=ttft-timeout with HTTP 408. Filter on the Model span. See First Chunk Gate.