{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "Traceloop API Schemas",
"description": "JSON Schema definitions for Traceloop REST API request and response objects",
"definitions": {
"evaluator_Property": {
"properties": {
"description": {
"type": "string"
},
"enum_values": {
"description": "The allowed values for this property. Only applicable when `type` is `enum`.",
"items": {
"type": "string"
},
"type": "array"
},
"label": {
"type": "string"
},
"name": {
"type": "string"
},
"type": {
"$ref": "#/components/schemas/evaluator.PropertyType"
}
},
"required": [
"name",
"type"
],
"type": "object"
},
"evaluator_PropertyType": {
"enum": [
"string",
"int",
"float",
"boolean",
"enum"
],
"type": "string",
"x-enum-varnames": [
"StringType",
"IntType",
"FloatType",
"BooleanType",
"EnumType"
]
},
"model_LLMMessage": {
"properties": {
"content": {
"type": "string"
},
"role": {
"type": "string"
},
"tool_call_id": {
"type": "string"
}
},
"type": "object"
},
"request_AgentEfficiencyInput": {
"properties": {
"trajectory_completions": {
"example": "[\"User found\", \"Email updated\", \"Changes saved\"]",
"type": "string"
},
"trajectory_prompts": {
"example": "[\"Find user info\", \"Update email\", \"Save changes\"]",
"type": "string"
}
},
"required": [
"trajectory_completions",
"trajectory_prompts"
],
"type": "object"
},
"request_AgentEfficiencyRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.AgentEfficiencyInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request_AgentFlowQualityConfigRequest": {
"properties": {
"conditions": {
"example": [
"no tools called",
"agent completed task"
],
"items": {
"type": "string"
},
"type": "array"
},
"threshold": {
"example": 0.5,
"type": "number"
}
},
"required": [
"conditions",
"threshold"
],
"type": "object"
},
"request_AgentFlowQualityInput": {
"properties": {
"trajectory_completions": {
"example": "[\"Found 5 flights\", \"Selected $299 flight\", \"Booking confirmed\"]",
"type": "string"
},
"trajectory_prompts": {
"example": "[\"Search for flights\", \"Select the cheapest option\", \"Confirm booking\"]",
"type": "string"
}
},
"required": [
"trajectory_completions",
"trajectory_prompts"
],
"type": "object"
},
"request_AgentFlowQualityRequest": {
"properties": {
"config": {
"$ref": "#/components/schemas/request.AgentFlowQualityConfigRequest"
},
"input": {
"$ref": "#/components/schemas/request.AgentFlowQualityInput"
}
},
"required": [
"config",
"input"
],
"type": "object"
},
"request_AgentGoalAccuracyInput": {
"properties": {
"completion": {
"example": "I have booked your flight from New York to Los Angeles departing Monday at 9am.",
"type": "string"
},
"question": {
"example": "Book a flight from NYC to LA for next Monday",
"type": "string"
},
"reference": {
"example": "Flight booked: NYC to LA, Monday departure",
"type": "string"
}
},
"required": [
"completion",
"question",
"reference"
],
"type": "object"
},
"request_AgentGoalAccuracyRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.AgentGoalAccuracyInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request_AgentGoalCompletenessConfigRequest": {
"properties": {
"threshold": {
"example": 0.5,
"type": "number"
}
},
"required": [
"threshold"
],
"type": "object"
},
"request_AgentGoalCompletenessInput": {
"properties": {
"trajectory_completions": {
"example": "[\"Account created\", \"Preferences saved\", \"Notifications enabled\"]",
"type": "string"
},
"trajectory_prompts": {
"example": "[\"Create new account\", \"Set preferences\", \"Enable notifications\"]",
"type": "string"
}
},
"required": [
"trajectory_completions",
"trajectory_prompts"
],
"type": "object"
},
"request_AgentGoalCompletenessRequest": {
"properties": {
"config": {
"$ref": "#/components/schemas/request.AgentGoalCompletenessConfigRequest"
},
"input": {
"$ref": "#/components/schemas/request.AgentGoalCompletenessInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request_AgentToolErrorDetectorInput": {
"properties": {
"tool_input": {
"example": "{\"action\": \"search\", \"query\": \"flights to Paris\"}",
"type": "string"
},
"tool_output": {
"example": "{\"status\": \"success\", \"results\": [{\"flight\": \"AF123\", \"price\": 450}]}",
"type": "string"
}
},
"required": [
"tool_input",
"tool_output"
],
"type": "object"
},
"request_AgentToolErrorDetectorRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.AgentToolErrorDetectorInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request_AgentToolTrajectoryConfigRequest": {
"properties": {
"input_params_sensitive": {
"example": true,
"type": "boolean"
},
"mismatch_sensitive": {
"example": false,
"type": "boolean"
},
"order_sensitive": {
"example": false,
"type": "boolean"
},
"threshold": {
"example": 0.5,
"type": "number"
}
},
"type": "object"
},
"request_AgentToolTrajectoryInput": {
"properties": {
"executed_tool_calls": {
"example": "[{\"name\": \"search\", \"input\": {\"query\": \"weather\"}}]",
"type": "string"
},
"expected_tool_calls": {
"example": "[{\"name\": \"search\", \"input\": {\"query\": \"weather\"}}]",
"type": "string"
}
},
"required": [
"executed_tool_calls",
"expected_tool_calls"
],
"type": "object"
},
"request_AgentToolTrajectoryRequest": {
"properties": {
"config": {
"$ref": "#/components/schemas/request.AgentToolTrajectoryConfigRequest"
},
"input": {
"$ref": "#/components/schemas/request.AgentToolTrajectoryInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request_AnswerCompletenessInput": {
"properties": {
"completion": {
"example": "Paris.",
"type": "string"
},
"context": {
"example": "The capital of France is Paris.",
"type": "string"
},
"question": {
"example": "What is the capital of France?",
"type": "string"
}
},
"required": [
"completion",
"context",
"question"
],
"type": "object"
},
"request_AnswerCompletenessRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.AnswerCompletenessInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request_AnswerCorrectnessInput": {
"properties": {
"completion": {
"example": "World War II ended in 1945.",
"type": "string"
},
"ground_truth": {
"example": "1945",
"type": "string"
},
"question": {
"example": "What year did World War II end?",
"type": "string"
}
},
"required": [
"completion",
"ground_truth",
"question"
],
"type": "object"
},
"request_AnswerCorrectnessRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.AnswerCorrectnessInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request_AnswerRelevancyInput": {
"properties": {
"answer": {
"example": "The capital of France is Paris.",
"type": "string"
},
"question": {
"example": "What is the capital of France?",
"type": "string"
}
},
"required": [
"answer",
"question"
],
"type": "object"
},
"request_AnswerRelevancyRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.AnswerRelevancyInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request_CharCountInput": {
"properties": {
"text": {
"example": "Hello, world! This is a sample text.",
"type": "string"
}
},
"required": [
"text"
],
"type": "object"
},
"request_CharCountRatioInput": {
"properties": {
"denominator_text": {
"example": "This is a longer text for comparison",
"type": "string"
},
"numerator_text": {
"example": "Short text",
"type": "string"
}
},
"required": [
"denominator_text",
"numerator_text"
],
"type": "object"
},
"request_CharCountRatioRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.CharCountRatioInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request_CharCountRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.CharCountInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request_ContextRelevanceConfigRequest": {
"properties": {
"model": {
"example": "gpt-4o",
"type": "string"
}
},
"type": "object"
},
"request_ContextRelevanceInput": {
"properties": {
"completion": {
"example": "Our store is open Monday to Friday from 9am to 6pm, and Saturday from 10am to 4pm. We are closed on Sundays.",
"type": "string"
},
"context": {
"example": "Our store is open Monday to Friday from 9am to 6pm, and Saturday from 10am to 4pm. We are closed on Sundays.",
"type": "string"
},
"query": {
"example": "What are the business hours?",
"type": "string"
}
},
"required": [
"completion",
"context",
"query"
],
"type": "object"
},
"request_ContextRelevanceRequest": {
"properties": {
"config": {
"$ref": "#/components/schemas/request.ContextRelevanceConfigRequest"
},
"input": {
"$ref": "#/components/schemas/request.ContextRelevanceInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request_ConversationQualityInput": {
"properties": {
"completions": {
"example": "[\"Hi! I'd be happy to assist you today.\", \"We offer consulting, development, and support services.\"]",
"type": "string"
},
"prompts": {
"example": "[\"Hello, how can I help?\", \"What services do you offer?\"]",
"type": "string"
}
},
"required": [
"completions",
"prompts"
],
"type": "object"
},
"request_ConversationQualityRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.ConversationQualityInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request_CreateAutoMonitorSetupInput": {
"properties": {
"evaluators": {
"description": "List of evaluator slugs to run on matched spans",
"example": [
"hallucination",
"toxicity"
],
"items": {
"type": "string"
},
"minItems": 1,
"type": "array"
},
"external_id": {
"description": "Unique identifier for the auto monitor setup, used to reference it in future requests",
"example": "my-agent-monitor-1",
"type": "string"
},
"selector": {
"description": "Map of span attributes to filter which spans this monitor applies to.\nKeys are span attribute names (e.g. gen_ai.system, gen_ai.request.model) and\nvalues can be strings, numbers, or booleans.\nExample: {\"gen_ai.system\": \"openai\", \"gen_ai.request.model\": \"gpt-4o\", \"gen_ai.request.max_tokens\": 1000}",
"type": "object"
}
},
"required": [
"evaluators",
"external_id"
],
"type": "object"
},
"request_CreateCustomEvaluatorInput": {
"properties": {
"description": {
"type": "string"
},
"frequency_penalty": {
"type": "number"
},
"input_schema": {
"description": "Schemas",
"items": {
"$ref": "#/components/schemas/evaluator.Property"
},
"type": "array"
},
"logprobs": {
"type": "boolean"
},
"max_tokens": {
"type": "integer"
},
"messages": {
"description": "LLM Message configuration",
"items": {
"$ref": "#/components/schemas/model.LLMMessage"
},
"type": "array"
},
"mode": {
"type": "string"
},
"model": {
"description": "LLM parameters (from BaseLLMConfig)",
"type": "string"
},
"name": {
"description": "Basic fields",
"type": "string"
},
"output_schema": {
"items": {
"$ref": "#/components/schemas/evaluator.Property"
},
"type": "array"
},
"presence_penalty": {
"type": "number"
},
"provider": {
"description": "Provider configuration",
"type": "string"
},
"slug": {
"type": "string"
},
"stop": {
"items": {
"type": "string"
},
"type": "array"
},
"temperature": {
"type": "number"
},
"top_p": {
"type": "number"
}
},
"required": [
"input_schema",
"messages",
"model",
"name",
"output_schema",
"provider"
],
"type": "object"
},
"request_CreateOrganizationRequest": {
"properties": {
"envs": {
"items": {
"type": "string"
},
"type": "array"
},
"org_name": {
"type": "string"
},
"retention_hours": {
"type": "integer",
"description": "Data retention period in hours (1-720). Defaults to 720 (1 month).",
"minimum": 1,
"maximum": 720,
"default": 720
}
},
"required": [
"org_name"
],
"type": "object"
},
"request_FaithfulnessInput": {
"properties": {
"completion": {
"example": "The Eiffel Tower is located in Paris and was built in 1889.",
"type": "string"
},
"context": {
"example": "The Eiffel Tower is a wrought-iron lattice tower on the Champ de Mars in Paris, France. It was constructed from 1887 to 1889.",
"type": "string"
},
"question": {
"example": "When was the Eiffel Tower built?",
"type": "string"
}
},
"required": [
"completion",
"context",
"question"
],
"type": "object"
},
"request_FaithfulnessRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.FaithfulnessInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request_GetMetricsRequest": {
"properties": {
"from_timestamp_sec": {
"type": "integer",
"description": "Start time in Unix seconds timestamp.",
"example": 1772905218
},
"to_timestamp_sec": {
"type": "integer",
"description": "End time in Unix seconds timestamp.",
"example": 1772905218
},
"cursor": {
"type": "integer",
"description": "Pagination cursor for fetching the next page of results. The cursor is returned in the response to the previous request.",
"example": 1772905218000
},
"environments": {
"items": {
"type": "string"
},
"type": "array",
"description": "List of environments to filter by",
"example": [
"prd",
"stg",
"dev"
]
},
"filters": {
"items": {
"$ref": "#/components/schemas/shared.FilterCondition"
},
"type": "array"
},
"limit": {
"type": "integer",
"description": "Maximum number of metrics to return per page.",
"example": 50
},
"logical_operator": {
"$ref": "#/components/schemas/types.LogicalOperator"
},
"metric_name": {
"type": "string",
"description": "Metric name to filter by. The metric_name is a metric attribute that is set on the metric object.",
"example": "answer relevancy"
},
"metric_source": {
"type": "string",
"description": "Metric source to filter by. The metric_source is a metric attribute that is set on the metric object.",
"example": "llm_as_a_judge"
},
"sort_by": {
"type": "string",
"description": "Field to sort by.",
"example": "timestamp"
},
"sort_order": {
"type": "string",
"description": "Sort order.",
"example": "ASC"
}
},
"type": "object"
},
"request_HtmlComparisonInput": {
"properties": {
"html1": {
"example": "Hello, world!
",
"type": "string"
},
"html2": {
"example": "Hello, world!
",
"type": "string"
}
},
"required": [
"html1",
"html2"
],
"type": "object"
},
"request_HtmlComparisonRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.HtmlComparisonInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request_InstructionAdherenceInput": {
"properties": {
"instructions": {
"example": "Respond in exactly 3 bullet points and use formal language.",
"type": "string"
},
"response": {
"example": "- First point about the topic\n- Second relevant consideration\n- Final concluding thought",
"type": "string"
}
},
"required": [
"instructions",
"response"
],
"type": "object"
},
"request_InstructionAdherenceRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.InstructionAdherenceInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request_IntentChangeInput": {
"properties": {
"completions": {
"example": "[\"Sure, I can help with hotel booking\", \"No problem, let me search for flights\"]",
"type": "string"
},
"prompts": {
"example": "[\"I want to book a hotel\", \"Actually, I need a flight instead\"]",
"type": "string"
}
},
"required": [
"completions",
"prompts"
],
"type": "object"
},
"request_IntentChangeRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.IntentChangeInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request_JSONValidatorConfigRequest": {
"properties": {
"enable_schema_validation": {
"example": true,
"type": "boolean"
},
"schema_string": {
"example": "{}",
"type": "string"
}
},
"type": "object"
},
"request_JSONValidatorInput": {
"properties": {
"text": {
"example": "{\"name\": \"John\", \"age\": 30}",
"type": "string"
}
},
"required": [
"text"
],
"type": "object"
},
"request_JSONValidatorRequest": {
"properties": {
"config": {
"$ref": "#/components/schemas/request.JSONValidatorConfigRequest"
},
"input": {
"$ref": "#/components/schemas/request.JSONValidatorInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request_PIIDetectorConfigRequest": {
"properties": {
"probability_threshold": {
"example": 0.8,
"type": "number"
}
},
"type": "object"
},
"request_PIIDetectorInput": {
"properties": {
"text": {
"example": "Please contact John Smith at john.smith@email.com or call 555-123-4567.",
"type": "string"
}
},
"required": [
"text"
],
"type": "object"
},
"request_PIIDetectorRequest": {
"properties": {
"config": {
"$ref": "#/components/schemas/request.PIIDetectorConfigRequest"
},
"input": {
"$ref": "#/components/schemas/request.PIIDetectorInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request_PerplexityInput": {
"properties": {
"logprobs": {
"example": "[-2.3, -1.5, -0.8, -1.2, -0.5]",
"type": "string"
}
},
"required": [
"logprobs"
],
"type": "object"
},
"request_PerplexityRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.PerplexityInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request_PlaceholderRegexConfigRequest": {
"properties": {
"case_sensitive": {
"example": true,
"type": "boolean"
},
"dot_include_nl": {
"example": true,
"type": "boolean"
},
"multi_line": {
"example": true,
"type": "boolean"
},
"should_match": {
"example": true,
"type": "boolean"
}
},
"type": "object"
},
"request_PlaceholderRegexInput": {
"properties": {
"placeholder_name": {
"example": "name",
"type": "string"
},
"placeholder_value": {
"example": "Hello ${name}",
"type": "string"
},
"text": {
"example": "Hello John Doe",
"type": "string"
}
},
"required": [
"placeholder_name",
"placeholder_value",
"text"
],
"type": "object"
},
"request_PlaceholderRegexRequest": {
"properties": {
"config": {
"$ref": "#/components/schemas/request.PlaceholderRegexConfigRequest"
},
"input": {
"$ref": "#/components/schemas/request.PlaceholderRegexInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request_ProfanityDetectorInput": {
"properties": {
"text": {
"example": "This is a clean and professional message.",
"type": "string"
}
},
"required": [
"text"
],
"type": "object"
},
"request_ProfanityDetectorRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.ProfanityDetectorInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request_PromptInjectionConfigRequest": {
"properties": {
"threshold": {
"example": 0.5,
"type": "number"
}
},
"type": "object"
},
"request_PromptInjectionInput": {
"properties": {
"prompt": {
"example": "What is the weather like today?",
"type": "string"
}
},
"required": [
"prompt"
],
"type": "object"
},
"request_PromptInjectionRequest": {
"properties": {
"config": {
"$ref": "#/components/schemas/request.PromptInjectionConfigRequest"
},
"input": {
"$ref": "#/components/schemas/request.PromptInjectionInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request_PromptPerplexityInput": {
"properties": {
"prompt": {
"example": "What is the capital of France?",
"type": "string"
}
},
"required": [
"prompt"
],
"type": "object"
},
"request_PromptPerplexityRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.PromptPerplexityInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request_RegexValidatorConfigRequest": {
"properties": {
"case_sensitive": {
"example": true,
"type": "boolean"
},
"dot_include_nl": {
"example": true,
"type": "boolean"
},
"multi_line": {
"example": true,
"type": "boolean"
},
"regex": {
"example": ".*",
"type": "string"
},
"should_match": {
"example": true,
"type": "boolean"
}
},
"type": "object"
},
"request_RegexValidatorInput": {
"properties": {
"text": {
"example": "user@example.com",
"type": "string"
}
},
"required": [
"text"
],
"type": "object"
},
"request_RegexValidatorRequest": {
"properties": {
"config": {
"$ref": "#/components/schemas/request.RegexValidatorConfigRequest"
},
"input": {
"$ref": "#/components/schemas/request.RegexValidatorInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request_SQLValidatorInput": {
"properties": {
"text": {
"example": "SELECT * FROM users WHERE id = 1;",
"type": "string"
}
},
"required": [
"text"
],
"type": "object"
},
"request_SQLValidatorRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.SQLValidatorInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request_SecretsDetectorInput": {
"properties": {
"text": {
"example": "Here is some text without any API keys or passwords.",
"type": "string"
}
},
"required": [
"text"
],
"type": "object"
},
"request_SecretsDetectorRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.SecretsDetectorInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request_SemanticSimilarityInput": {
"properties": {
"completion": {
"example": "The cat sat on the mat.",
"type": "string"
},
"reference": {
"example": "A feline was resting on the rug.",
"type": "string"
}
},
"required": [
"completion",
"reference"
],
"type": "object"
},
"request_SemanticSimilarityRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.SemanticSimilarityInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request_SexismDetectorConfigRequest": {
"properties": {
"threshold": {
"example": 0.5,
"type": "number"
}
},
"type": "object"
},
"request_SexismDetectorInput": {
"properties": {
"text": {
"example": "All team members should be treated equally regardless of gender.",
"type": "string"
}
},
"required": [
"text"
],
"type": "object"
},
"request_SexismDetectorRequest": {
"properties": {
"config": {
"$ref": "#/components/schemas/request.SexismDetectorConfigRequest"
},
"input": {
"$ref": "#/components/schemas/request.SexismDetectorInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request_ToneDetectionInput": {
"properties": {
"text": {
"example": "The capital of France is Paris.",
"type": "string"
}
},
"required": [
"text"
],
"type": "object"
},
"request_ToneDetectionRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.ToneDetectionInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request_TopicAdherenceInput": {
"properties": {
"completion": {
"example": "Machine learning is a subset of AI that enables systems to learn from data.",
"type": "string"
},
"question": {
"example": "Tell me about machine learning",
"type": "string"
},
"reference_topics": {
"example": "artificial intelligence, data science, algorithms",
"type": "string"
}
},
"required": [
"completion",
"question",
"reference_topics"
],
"type": "object"
},
"request_TopicAdherenceRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.TopicAdherenceInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request_ToxicityDetectorConfigRequest": {
"properties": {
"threshold": {
"example": 0.5,
"type": "number"
}
},
"type": "object"
},
"request_ToxicityDetectorInput": {
"properties": {
"text": {
"example": "Thank you for your help with this project.",
"type": "string"
}
},
"required": [
"text"
],
"type": "object"
},
"request_ToxicityDetectorRequest": {
"properties": {
"config": {
"$ref": "#/components/schemas/request.ToxicityDetectorConfigRequest"
},
"input": {
"$ref": "#/components/schemas/request.ToxicityDetectorInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request_UncertaintyDetectorInput": {
"properties": {
"prompt": {
"example": "I am not sure, I think the capital of France is Paris.",
"type": "string"
}
},
"required": [
"prompt"
],
"type": "object"
},
"request_UncertaintyDetectorRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.UncertaintyDetectorInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request_UpdateAutoMonitorSetupInput": {
"properties": {
"evaluators": {
"description": "List of evaluator slugs to run on matched spans",
"example": [
"hallucination",
"toxicity"
],
"items": {
"type": "string"
},
"type": "array"
},
"selector": {
"description": "Map of span attributes to filter which spans this monitor applies to.\nKeys are span attribute names (e.g. gen_ai.system, gen_ai.request.model) and\nvalues can be strings, numbers, or booleans.\nExample: {\"gen_ai.system\": \"openai\", \"gen_ai.request.model\": \"gpt-4o\", \"gen_ai.request.max_tokens\": 1000}",
"type": "object"
}
},
"type": "object"
},
"request_WordCountInput": {
"properties": {
"text": {
"example": "This is a sample text with several words.",
"type": "string"
}
},
"required": [
"text"
],
"type": "object"
},
"request_WordCountRatioInput": {
"properties": {
"denominator_text": {
"example": "This is a longer input text for comparison",
"type": "string"
},
"numerator_text": {
"example": "Short response",
"type": "string"
}
},
"required": [
"denominator_text",
"numerator_text"
],
"type": "object"
},
"request_WordCountRatioRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.WordCountRatioInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request_WordCountRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.WordCountInput"
}
},
"required": [
"input"
],
"type": "object"
},
"response_AgentEfficiencyResponse": {
"properties": {
"step_efficiency_reason": {
"example": "Agent completed task with minimal redundant steps",
"type": "string"
},
"step_efficiency_score": {
"example": 0.85,
"type": "number"
},
"task_completion_reason": {
"example": "All required tasks were completed successfully",
"type": "string"
},
"task_completion_score": {
"example": 0.92,
"type": "number"
}
},
"type": "object"
},
"response_AgentFlowQualityResponse": {
"properties": {
"reason": {
"example": "Agent followed the expected flow correctly",
"type": "string"
},
"score": {
"example": 0.89,
"type": "number"
},
"success": {
"example": true,
"type": "boolean"
}
},
"type": "object"
},
"response_AgentGoalAccuracyResponse": {
"properties": {
"accuracy_score": {
"example": 0.88,
"type": "number"
}
},
"type": "object"
},
"response_AgentGoalCompletenessResponse": {
"properties": {
"reason": {
"example": "All user goals were accomplished",
"type": "string"
},
"score": {
"example": 0.95,
"type": "number"
},
"success": {
"example": true,
"type": "boolean"
}
},
"type": "object"
},
"response_AgentToolErrorDetectorResponse": {
"properties": {
"reason": {
"example": "Tool executed successfully without errors",
"type": "string"
},
"success": {
"example": true,
"type": "boolean"
}
},
"type": "object"
},
"response_AgentToolTrajectoryResponse": {
"properties": {
"reason": {
"example": "Tool calls match the expected trajectory",
"type": "string"
},
"score": {
"example": 0.85,
"type": "number"
},
"success": {
"example": true,
"type": "boolean"
}
},
"type": "object"
},
"response_AnswerCompletenessResponse": {
"properties": {
"answer_completeness_score": {
"example": 0.95,
"type": "number"
}
},
"type": "object"
},
"response_AnswerCorrectnessResponse": {
"properties": {
"correctness_score": {
"example": 0.91,
"type": "number"
}
},
"type": "object"
},
"response_AnswerRelevancyResponse": {
"properties": {
"is_relevant": {
"example": true,
"type": "boolean"
}
},
"type": "object"
},
"response_AutoMonitorEvaluatorResponse": {
"properties": {
"binding_id": {
"type": "string"
},
"error_message": {
"type": "string"
},
"evaluator_id": {
"type": "string"
},
"evaluator_type": {
"type": "string"
},
"input_schema": {
"items": {
"$ref": "#/components/schemas/evaluator.Property"
},
"type": "array"
},
"output_schema": {
"items": {
"$ref": "#/components/schemas/evaluator.Property"
},
"type": "array"
},
"processed_at": {
"type": "string"
},
"status": {
"type": "string"
}
},
"type": "object"
},
"response_AutoMonitorSetupResponse": {
"properties": {
"created_at": {
"type": "string"
},
"env_project_id": {
"type": "string"
},
"evaluators": {
"items": {
"$ref": "#/components/schemas/response.AutoMonitorEvaluatorResponse"
},
"type": "array"
},
"external_id": {
"type": "string"
},
"id": {
"type": "string"
},
"init_rules": {
"items": {
"$ref": "#/components/schemas/types.Rule"
},
"type": "array"
},
"org_id": {
"type": "string"
},
"project_id": {
"type": "string"
},
"status": {
"type": "string"
},
"updated_at": {
"type": "string"
}
},
"type": "object"
},
"response_CharCountRatioResponse": {
"properties": {
"char_ratio": {
"example": 0.75,
"type": "number"
}
},
"type": "object"
},
"response_CharCountResponse": {
"properties": {
"char_count": {
"example": 42,
"type": "integer"
}
},
"type": "object"
},
"response_ContextRelevanceResponse": {
"properties": {
"relevance_score": {
"example": 0.88,
"type": "number"
}
},
"type": "object"
},
"response_ConversationQualityResponse": {
"properties": {
"conversation_quality_score": {
"example": 0.82,
"type": "number"
}
},
"type": "object"
},
"response_CreateEvaluatorResponse": {
"properties": {
"bindings": {
"items": {
"type": "string"
},
"type": "array"
},
"evaluator_id": {
"type": "string"
},
"slug": {
"type": "string"
},
"type": {
"$ref": "#/components/schemas/types.EvaluatorType"
},
"version_id": {
"type": "string"
}
},
"type": "object"
},
"response_CreateOrganizationResponse": {
"properties": {
"environments": {
"items": {
"$ref": "#/components/schemas/response.EnvironmentWithKeyResponse"
},
"type": "array"
},
"org_id": {
"type": "string"
}
},
"type": "object"
},
"response_EnvironmentWithKeyResponse": {
"properties": {
"api_key": {
"type": "string"
},
"slug": {
"type": "string"
}
},
"type": "object"
},
"response_ErrorResponse": {
"description": "Standard error response structure",
"properties": {
"error": {
"example": "error message",
"type": "string"
}
},
"type": "object"
},
"response_FaithfulnessResponse": {
"properties": {
"is_faithful": {
"example": true,
"type": "boolean"
}
},
"type": "object"
},
"response_GetMetricsResponse": {
"properties": {
"metrics": {
"$ref": "#/components/schemas/response.PaginatedMetricsResponse"
}
},
"type": "object"
},
"response_HtmlComparisonResponse": {
"properties": {
"similarity_score": {
"example": 0.92,
"type": "number"
}
},
"type": "object"
},
"response_InstructionAdherenceResponse": {
"properties": {
"instruction_adherence_score": {
"example": 0.87,
"type": "number"
}
},
"type": "object"
},
"response_IntentChangeResponse": {
"properties": {
"reason": {
"example": "User intent remained consistent throughout the conversation",
"type": "string"
},
"score": {
"example": 1,
"type": "integer"
},
"success": {
"example": true,
"type": "boolean"
}
},
"type": "object"
},
"response_JSONValidatorResponse": {
"properties": {
"is_valid_json": {
"example": true,
"type": "boolean"
}
},
"type": "object"
},
"response_MetricGroup": {
"properties": {
"metric_name": {
"type": "string",
"description": "Metric name.",
"example": "answer relevancy"
},
"organization_id": {
"type": "string"
},
"points": {
"items": {
"$ref": "#/components/schemas/response.MetricPoint"
},
"type": "array",
"description": "List of data points for this metric. The `unit` field in each point's `labels` indicates the data type and determines which value field is populated: `bool_value` for booleans, `numeric_value` for numbers, or `enum_value` for strings."
}
},
"type": "object"
},
"response_MetricPoint": {
"description": "A single metric data point. The `unit` field in `labels` indicates the data type and determines which value field is populated: `bool_value` for booleans unitless, `numeric_value` for number units, or `enum_value` for strings.",
"properties": {
"bool_value": {
"type": "boolean"
},
"enum_value": {
"type": "string"
},
"numeric_value": {
"type": "number"
},
"event_time": {
"type": "integer"
},
"labels": {
"description": "The labels are the attributes of the metric point. The labels are key-value pairs that are used to identify the metric point.",
"example": {
"agent_name": "Travel Planner Agent",
"entity_type": "span",
"env_project_id": "1111",
"environment": "dev",
"evaluator_name": "",
"insert_time": "1772616490000",
"metric_source": "word_count",
"metric_type": "numeric",
"model": "gpt-4o-2024-08-06",
"org_id": "12345",
"run_id": "54321",
"service_name": "travel-agent-demo",
"span_id": "e118a1985c9cff23",
"trace_id": "2c98a42d1225cb786f3395d97ca3014d",
"unit": "words",
"vendor": "openai"
},
"additionalProperties": {
"type": "string"
},
"type": "object"
}
},
"type": "object"
},
"response_MetricsHWMResponse": {
"properties": {
"high_water_mark": {
"type": "integer"
}
},
"type": "object"
},
"response_PIIDetectorResponse": {
"properties": {
"has_pii": {
"example": false,
"type": "boolean"
}
},
"type": "object"
},
"response_PaginatedMetricsResponse": {
"properties": {
"data": {
"items": {
"$ref": "#/components/schemas/response.MetricGroup"
},
"type": "array"
},
"next_cursor": {
"type": "string"
},
"total_points": {
"type": "integer"
},
"total_results": {
"type": "integer"
}
},
"type": "object"
},
"response_PerplexityResponse": {
"properties": {
"perplexity_score": {
"example": 12.5,
"type": "number"
}
},
"type": "object"
},
"response_PlaceholderRegexResponse": {
"properties": {
"is_valid_regex": {
"example": true,
"type": "boolean"
}
},
"type": "object"
},
"response_ProfanityDetectorResponse": {
"properties": {
"is_safe": {
"example": false,
"type": "boolean"
}
},
"type": "object"
},
"response_PromptInjectionResponse": {
"properties": {
"has_injection": {
"example": false,
"type": "boolean"
}
},
"type": "object"
},
"response_PromptPerplexityResponse": {
"properties": {
"perplexity_score": {
"example": 8.3,
"type": "number"
}
},
"type": "object"
},
"response_RegexValidatorResponse": {
"properties": {
"is_valid_regex": {
"example": true,
"type": "boolean"
}
},
"type": "object"
},
"response_SQLValidatorResponse": {
"properties": {
"is_valid_sql": {
"example": true,
"type": "boolean"
}
},
"type": "object"
},
"response_SecretsDetectorResponse": {
"properties": {
"has_secret": {
"example": false,
"type": "boolean"
}
},
"type": "object"
},
"response_SemanticSimilarityResponse": {
"properties": {
"similarity_score": {
"example": 0.92,
"type": "number"
}
},
"type": "object"
},
"response_SexismDetectorResponse": {
"properties": {
"is_safe": {
"example": false,
"type": "boolean"
}
},
"type": "object"
},
"response_ToneDetectionResponse": {
"properties": {
"score": {
"example": 0.95,
"type": "number"
},
"tone": {
"example": "neutral",
"type": "string"
}
},
"type": "object"
},
"response_TopicAdherenceResponse": {
"properties": {
"adherence_score": {
"example": 0.95,
"type": "number"
}
},
"type": "object"
},
"response_ToxicityDetectorResponse": {
"properties": {
"is_safe": {
"example": false,
"type": "boolean"
}
},
"type": "object"
},
"response_UncertaintyDetectorResponse": {
"properties": {
"answer": {
"example": "Paris",
"type": "string"
},
"uncertainty": {
"example": 0.95,
"type": "number"
}
},
"type": "object"
},
"response_WordCountRatioResponse": {
"properties": {
"word_ratio": {
"example": 0.85,
"type": "number"
}
},
"type": "object"
},
"response_WordCountResponse": {
"properties": {
"word_count": {
"example": 10,
"type": "integer"
}
},
"type": "object"
},
"shared_FilterCondition": {
"properties": {
"field": {
"type": "string",
"description": "Field key to filter by. The key is labels attribute on the metric point from the response.metrics object",
"example": "labels.agent_name"
},
"operator": {
"type": "string",
"description": "Comparison operator.",
"enum": [
"equals",
"not_equals",
"contains",
"not_contains",
"in",
"not_in"
],
"example": "equals"
},
"value": {
"type": "string",
"description": "Value to filter by.",
"example": "Travel Planner Agent"
},
"values": {
"items": {
"type": "string"
},
"type": "array",
"description": "List of values to filter by. This is only used for the in and not_in operators.",
"example": [
"Travel Planner Agent",
"Calendar Planner Agent"
]
}
},
"type": "object"
},
"types_ComparisonOperator": {
"enum": [
"equals",
"not_equals",
"contains",
"not_contains",
"in",
"not_in"
],
"type": "string",
"x-enum-varnames": [
"ComparisonOperatorEquals",
"ComparisonOperatorNotEquals",
"ComparisonOperatorContains",
"ComparisonOperatorNotContains",
"ComparisonOperatorIn",
"ComparisonOperatorNotIn"
]
},
"types_EvaluatorType": {
"enum": [
"AGENT_GOAL_ACCURACY",
"CHAR_COUNT",
"CHAR_COUNT_RATIO",
"ANSWER_COMPLETENESS",
"FAITHFULNESS",
"JSON_VALIDATOR",
"PERPLEXITY",
"PII_DETECTOR",
"PLACEHOLDER_REGEX",
"PROFANITY_DETECTOR",
"SEXISM_DETECTOR",
"PROMPT_INJECTION",
"TONE_DETECTION",
"PROMPT_PERPLEXITY",
"TOXICITY_DETECTOR",
"ANSWER_RELEVANCY",
"ANSWER_CORRECTNESS",
"REGEX_VALIDATOR",
"SECRETS_DETECTOR",
"SQL_VALIDATOR",
"SEMANTIC_SIMILARITY",
"TOPIC_ADHERENCE",
"UNCERTAINTY_DETECTOR",
"WORD_COUNT",
"WORD_COUNT_RATIO",
"INSTRUCTION_ADHERENCE",
"AGENT_FLOW_QUALITY",
"AGENT_EFFICIENCY",
"AGENT_GOAL_COMPLETENESS",
"CONVERSATION_QUALITY",
"INTENT_CHANGE",
"CONTEXT_RELEVANCE",
"AGENT_TOOL_ERROR_DETECTOR",
"AGENT_TOOL_TRAJECTORY",
"HTML_COMPARISON",
"LLM_AS_A_JUDGE"
],
"type": "string",
"x-enum-varnames": [
"AgentGoalAccuracy",
"CharCount",
"CharCountRatio",
"AnswerCompleteness",
"Faithfulness",
"JsonValidator",
"Perplexity",
"PIIDetector",
"PlaceholderRegex",
"ProfanityDetector",
"SexismDetector",
"PromptInjection",
"ToneDetection",
"PromptPerplexity",
"ToxicityDetector",
"AnswerRelevancy",
"AnswerCorrectness",
"RegexValidator",
"SecretsDetector",
"SQLValidator",
"SemanticSimilarity",
"TopicAdherence",
"UncertaintyDetector",
"WordCount",
"WordCountRatio",
"InstructionAdherence",
"AgentFlowQuality",
"AgentEfficiency",
"AgentGoalCompleteness",
"ConversationQuality",
"IdentIntentChange",
"ContextRelevance",
"AgentToolErrorDetector",
"AgentToolTrajectory",
"HtmlComparison",
"LLMAsAJudge"
]
},
"types_LogicalOperator": {
"enum": [
"AND",
"OR"
],
"type": "string",
"x-enum-varnames": [
"LogicalOperatorAnd",
"LogicalOperatorOr"
]
},
"types_Rule": {
"properties": {
"key": {
"type": "string"
},
"op": {
"$ref": "#/components/schemas/types.ComparisonOperator"
},
"source": {
"type": "string"
},
"value": {
"type": "string"
},
"value_type": {
"type": "string"
}
},
"type": "object"
}
}
}