{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://artifacts.apis.io/scale-ai/json-schema/scale-ai-create-batch-completions-v1-request-schema.json",
  "title": "CreateBatchCompletionsV1Request",
  "description": "Request object for batch completions.",
  "x-generated": "2026-10-27",
  "x-method": "derived",
  "x-generator": "derive-json-schema.py",
  "x-source": "openapi/scale-ai-llm-engine-openapi.yml#/components/schemas/CreateBatchCompletionsV1Request",
  "properties": {
    "input_data_path": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "title": "Input Data Path",
      "description": "Path to the input file. The input file should be a JSON file of type List[CreateBatchCompletionsRequestContent]."
    },
    "output_data_path": {
      "type": "string",
      "title": "Output Data Path",
      "description": "Path to the output file. The output file will be a JSON file of type List[CompletionOutput]."
    },
    "labels": {
      "additionalProperties": {
        "type": "string"
      },
      "type": "object",
      "title": "Labels",
      "description": "Labels to attach to the batch inference job.",
      "default": {}
    },
    "data_parallelism": {
      "anyOf": [
        {
          "type": "integer",
          "maximum": 64.0,
          "minimum": 1.0
        },
        {
          "type": "null"
        }
      ],
      "title": "Data Parallelism",
      "description": "Number of replicas to run the batch inference. More replicas are slower to schedule but faster to inference.",
      "default": 1
    },
    "max_runtime_sec": {
      "anyOf": [
        {
          "type": "integer",
          "maximum": 172800.0,
          "minimum": 1.0
        },
        {
          "type": "null"
        }
      ],
      "title": "Max Runtime Sec",
      "description": "Maximum runtime of the batch inference in seconds. Default to one day.",
      "default": 86400
    },
    "priority": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "title": "Priority",
      "description": "Priority of the batch inference job. Default to None."
    },
    "tool_config": {
      "anyOf": [
        {
          "$ref": "#/$defs/ToolConfig"
        },
        {
          "type": "null"
        }
      ],
      "description": "\nConfiguration for tool use.\nNOTE: this config is highly experimental and signature will change significantly in future iterations."
    },
    "cpus": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "integer"
        },
        {
          "type": "number"
        },
        {
          "type": "null"
        }
      ],
      "title": "Cpus",
      "description": "CPUs to use for the batch inference."
    },
    "gpus": {
      "anyOf": [
        {
          "type": "integer"
        },
        {
          "type": "null"
        }
      ],
      "title": "Gpus",
      "description": "Number of GPUs to use for the batch inference."
    },
    "memory": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "integer"
        },
        {
          "type": "number"
        },
        {
          "type": "null"
        }
      ],
      "title": "Memory",
      "description": "Amount of memory to use for the batch inference."
    },
    "gpu_type": {
      "anyOf": [
        {
          "$ref": "#/$defs/GpuType"
        },
        {
          "type": "null"
        }
      ],
      "description": "GPU type to use for the batch inference."
    },
    "storage": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "integer"
        },
        {
          "type": "number"
        },
        {
          "type": "null"
        }
      ],
      "title": "Storage",
      "description": "Storage to use for the batch inference."
    },
    "nodes_per_worker": {
      "anyOf": [
        {
          "type": "integer"
        },
        {
          "type": "null"
        }
      ],
      "title": "Nodes Per Worker",
      "description": "Number of nodes per worker for the batch inference."
    },
    "content": {
      "anyOf": [
        {
          "$ref": "#/$defs/CreateBatchCompletionsV1RequestContent"
        },
        {
          "type": "null"
        }
      ]
    },
    "model_config": {
      "$ref": "#/$defs/CreateBatchCompletionsV1ModelConfig"
    }
  },
  "type": "object",
  "required": [
    "output_data_path",
    "model_config"
  ],
  "$defs": {
    "CreateBatchCompletionsV1ModelConfig": {
      "properties": {
        "max_model_len": {
          "anyOf": [
            {
              "type": "integer"
            },
            {
              "type": "null"
            }
          ],
          "title": "Max Model Len",
          "description": "Model context length, If unspecified, will be automatically derived from the model config"
        },
        "max_num_seqs": {
          "anyOf": [
            {
              "type": "integer"
            },
            {
              "type": "null"
            }
          ],
          "title": "Max Num Seqs",
          "description": "Maximum number of sequences per iteration"
        },
        "enforce_eager": {
          "anyOf": [
            {
              "type": "boolean"
            },
            {
              "type": "null"
            }
          ],
          "title": "Enforce Eager",
          "description": "Always use eager-mode PyTorch. If False, will use eager mode and CUDA graph in hybrid for maximal perforamnce and flexibility"
        },
        "trust_remote_code": {
          "anyOf": [
            {
              "type": "boolean"
            },
            {
              "type": "null"
            }
          ],
          "title": "Trust Remote Code",
          "description": "Whether to trust remote code from Hugging face hub. This is only applicable to models whose code is not supported natively by the transformers library (e.g. deepseek). Default to False.",
          "default": false
        },
        "pipeline_parallel_size": {
          "anyOf": [
            {
              "type": "integer"
            },
            {
              "type": "null"
            }
          ],
          "title": "Pipeline Parallel Size",
          "description": "Number of pipeline stages. Default to None."
        },
        "tensor_parallel_size": {
          "anyOf": [
            {
              "type": "integer"
            },
            {
              "type": "null"
            }
          ],
          "title": "Tensor Parallel Size",
          "description": "Number of tensor parallel replicas. Default to None."
        },
        "quantization": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "title": "Quantization",
          "description": "Method used to quantize the weights. If None, we first check the `quantization_config` attribute in the model config file. If that is None, we assume the model weights are not quantized and use `dtype` to determine the data type of the weights."
        },
        "disable_log_requests": {
          "anyOf": [
            {
              "type": "boolean"
            },
            {
              "type": "null"
            }
          ],
          "title": "Disable Log Requests",
          "description": "Disable logging requests. Default to None."
        },
        "chat_template": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "title": "Chat Template",
          "description": "A Jinja template to use for this endpoint. If not provided, will use the chat template from the checkpoint"
        },
        "tool_call_parser": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "title": "Tool Call Parser",
          "description": "Tool call parser"
        },
        "enable_auto_tool_choice": {
          "anyOf": [
            {
              "type": "boolean"
            },
            {
              "type": "null"
            }
          ],
          "title": "Enable Auto Tool Choice",
          "description": "Enable auto tool choice"
        },
        "load_format": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "title": "Load Format",
          "description": "The format of the model weights to load.\n\n* \"auto\" will try to load the weights in the safetensors format and fall back to the pytorch bin format if safetensors format is not available.\n* \"pt\" will load the weights in the pytorch bin format.\n* \"safetensors\" will load the weights in the safetensors format.\n* \"npcache\" will load the weights in pytorch format and store a numpy cache to speed up the loading.\n* \"dummy\" will initialize the weights with random values, which is mainly for profiling.\n* \"tensorizer\" will load the weights using tensorizer from CoreWeave. See the Tensorize vLLM Model script in the Examples section for more information.\n* \"bitsandbytes\" will load the weights using bitsandbytes quantization.\n"
        },
        "config_format": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "title": "Config Format",
          "description": "The config format which shall be loaded.  Defaults to 'auto' which defaults to 'hf'."
        },
        "tokenizer_mode": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "title": "Tokenizer Mode",
          "description": "Tokenizer mode. 'auto' will use the fast tokenizer ifavailable, 'slow' will always use the slow tokenizer, and'mistral' will always use the tokenizer from `mistral_common`."
        },
        "limit_mm_per_prompt": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "title": "Limit Mm Per Prompt",
          "description": "Maximum number of data instances per modality per prompt. Only applicable for multimodal models."
        },
        "max_num_batched_tokens": {
          "anyOf": [
            {
              "type": "integer"
            },
            {
              "type": "null"
            }
          ],
          "title": "Max Num Batched Tokens",
          "description": "Maximum number of batched tokens per iteration"
        },
        "tokenizer": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "title": "Tokenizer",
          "description": "Name or path of the huggingface tokenizer to use."
        },
        "dtype": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "title": "Dtype",
          "description": "Data type for model weights and activations. The 'auto' option will use FP16 precision for FP32 and FP16 models, and BF16 precision for BF16 models."
        },
        "seed": {
          "anyOf": [
            {
              "type": "integer"
            },
            {
              "type": "null"
            }
          ],
          "title": "Seed",
          "description": "Random seed for the model."
        },
        "revision": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "title": "Revision",
          "description": "The specific model version to use. It can be a branch name, a tag name, or a commit id. If unspecified, will use the default version."
        },
        "code_revision": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "title": "Code Revision",
          "description": "The specific revision to use for the model code on Hugging Face Hub. It can be a branch name, a tag name, or a commit id. If unspecified, will use the default version."
        },
        "rope_scaling": {
          "anyOf": [
            {
              "additionalProperties": true,
              "type": "object"
            },
            {
              "type": "null"
            }
          ],
          "title": "Rope Scaling",
          "description": "Dictionary containing the scaling configuration for the RoPE embeddings. When using this flag, don't update `max_position_embeddings` to the expected new maximum."
        },
        "tokenizer_revision": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "title": "Tokenizer Revision",
          "description": "The specific tokenizer version to use. It can be a branch name, a tag name, or a commit id. If unspecified, will use the default version."
        },
        "quantization_param_path": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "title": "Quantization Param Path",
          "description": "Path to JSON file containing scaling factors. Used to load KV cache scaling factors into the model when KV cache type is FP8_E4M3 on ROCm (AMD GPU). In the future these will also be used to load activation and weight scaling factors when the model dtype is FP8_E4M3 on ROCm."
        },
        "max_seq_len_to_capture": {
          "anyOf": [
            {
              "type": "integer"
            },
            {
              "type": "null"
            }
          ],
          "title": "Max Seq Len To Capture",
          "description": "Maximum sequence len covered by CUDA graphs. When a sequence has context length larger than this, we fall back to eager mode. Additionally for encoder-decoder models, if the sequence length of the encoder input is larger than this, we fall back to the eager mode."
        },
        "disable_sliding_window": {
          "anyOf": [
            {
              "type": "boolean"
            },
            {
              "type": "null"
            }
          ],
          "title": "Disable Sliding Window",
          "description": "Whether to disable sliding window. If True, we will disable the sliding window functionality of the model. If the model does not support sliding window, this argument is ignored."
        },
        "skip_tokenizer_init": {
          "anyOf": [
            {
              "type": "boolean"
            },
            {
              "type": "null"
            }
          ],
          "title": "Skip Tokenizer Init",
          "description": "If true, skip initialization of tokenizer and detokenizer."
        },
        "served_model_name": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "title": "Served Model Name",
          "description": "The model name used in metrics tag `model_name`, matches the model name exposed via the APIs. If multiple model names provided, the first name will be used. If not specified, the model name will be the same as `model`."
        },
        "override_neuron_config": {
          "anyOf": [
            {
              "additionalProperties": true,
              "type": "object"
            },
            {
              "type": "null"
            }
          ],
          "title": "Override Neuron Config",
          "description": "Initialize non default neuron config or override default neuron config that are specific to Neuron devices, this argument will be used to configure the neuron config that can not be gathered from the vllm arguments."
        },
        "mm_processor_kwargs": {
          "anyOf": [
            {
              "additionalProperties": true,
              "type": "object"
            },
            {
              "type": "null"
            }
          ],
          "title": "Mm Processor Kwargs",
          "description": "Arguments to be forwarded to the model's processor for multi-modal data, e.g., image processor."
        },
        "block_size": {
          "anyOf": [
            {
              "type": "integer"
            },
            {
              "type": "null"
            }
          ],
          "title": "Block Size",
          "description": "Size of a cache block in number of tokens."
        },
        "gpu_memory_utilization": {
          "anyOf": [
            {
              "type": "number"
            },
            {
              "type": "null"
            }
          ],
          "title": "Gpu Memory Utilization",
          "description": "Fraction of GPU memory to use for the vLLM execution."
        },
        "swap_space": {
          "anyOf": [
            {
              "type": "number"
            },
            {
              "type": "null"
            }
          ],
          "title": "Swap Space",
          "description": "Size of the CPU swap space per GPU (in GiB)."
        },
        "cache_dtype": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "title": "Cache Dtype",
          "description": "Data type for kv cache storage."
        },
        "num_gpu_blocks_override": {
          "anyOf": [
            {
              "type": "integer"
            },
            {
              "type": "null"
            }
          ],
          "title": "Num Gpu Blocks Override",
          "description": "Number of GPU blocks to use. This overrides the profiled num_gpu_blocks if specified. Does nothing if None."
        },
        "enable_prefix_caching": {
          "anyOf": [
            {
              "type": "boolean"
            },
            {
              "type": "null"
            }
          ],
          "title": "Enable Prefix Caching",
          "description": "Enables automatic prefix caching."
        },
        "model": {
          "type": "string",
          "title": "Model",
          "description": "ID of the model to use."
        },
        "checkpoint_path": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "title": "Checkpoint Path",
          "description": "Path to the checkpoint to load the model from."
        },
        "num_shards": {
          "anyOf": [
            {
              "type": "integer",
              "minimum": 1.0
            },
            {
              "type": "null"
            }
          ],
          "title": "Num Shards",
          "description": "\nSuggested number of shards to distribute the model. When not specified, will infer the number of shards based on model config.\nSystem may decide to use a different number than the given value.\n",
          "default": 1
        },
        "max_context_length": {
          "anyOf": [
            {
              "type": "integer",
              "minimum": 1.0
            },
            {
              "type": "null"
            }
          ],
          "title": "Max Context Length",
          "description": "Maximum context length to use for the model. Defaults to the max allowed by the model. Deprecated in favor of max_model_len."
        },
        "response_role": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "title": "Response Role",
          "description": "Role of the response in the conversation. Only supported in chat completions."
        },
        "labels": {
          "additionalProperties": {
            "type": "string"
          },
          "type": "object",
          "title": "Labels",
          "description": "Labels to attach to the batch inference job.",
          "default": {}
        }
      },
      "type": "object",
      "required": [
        "model"
      ],
      "title": "CreateBatchCompletionsV1ModelConfig"
    },
    "CreateBatchCompletionsV1RequestContent": {
      "properties": {
        "prompts": {
          "items": {
            "type": "string"
          },
          "type": "array",
          "title": "Prompts"
        },
        "max_new_tokens": {
          "type": "integer",
          "title": "Max New Tokens"
        },
        "temperature": {
          "type": "number",
          "maximum": 1.0,
          "minimum": 0.0,
          "title": "Temperature"
        },
        "stop_sequences": {
          "anyOf": [
            {
              "items": {
                "type": "string"
              },
              "type": "array"
            },
            {
              "type": "null"
            }
          ],
          "title": "Stop Sequences"
        },
        "return_token_log_probs": {
          "anyOf": [
            {
              "type": "boolean"
            },
            {
              "type": "null"
            }
          ],
          "title": "Return Token Log Probs",
          "default": false
        },
        "presence_penalty": {
          "anyOf": [
            {
              "type": "number",
              "maximum": 2.0,
              "minimum": 0.0
            },
            {
              "type": "null"
            }
          ],
          "title": "Presence Penalty"
        },
        "frequency_penalty": {
          "anyOf": [
            {
              "type": "number",
              "maximum": 2.0,
              "minimum": 0.0
            },
            {
              "type": "null"
            }
          ],
          "title": "Frequency Penalty"
        },
        "top_k": {
          "anyOf": [
            {
              "type": "integer",
              "minimum": -1.0
            },
            {
              "type": "null"
            }
          ],
          "title": "Top K"
        },
        "top_p": {
          "anyOf": [
            {
              "type": "number",
              "maximum": 1.0,
              "exclusiveMinimum": 0.0
            },
            {
              "type": "null"
            }
          ],
          "title": "Top P"
        },
        "skip_special_tokens": {
          "anyOf": [
            {
              "type": "boolean"
            },
            {
              "type": "null"
            }
          ],
          "title": "Skip Special Tokens",
          "default": true
        }
      },
      "type": "object",
      "required": [
        "prompts",
        "max_new_tokens",
        "temperature"
      ],
      "title": "CreateBatchCompletionsV1RequestContent"
    },
    "GpuType": {
      "type": "string",
      "enum": [
        "nvidia-tesla-t4",
        "nvidia-ampere-a10",
        "nvidia-ampere-a100",
        "nvidia-ampere-a100e",
        "nvidia-hopper-h100",
        "nvidia-hopper-h100-1g20gb",
        "nvidia-hopper-h100-3g40gb"
      ],
      "title": "GpuType",
      "description": "Lists allowed GPU types for Launch."
    },
    "ToolConfig": {
      "properties": {
        "name": {
          "type": "string",
          "title": "Name"
        },
        "max_iterations": {
          "anyOf": [
            {
              "type": "integer"
            },
            {
              "type": "null"
            }
          ],
          "title": "Max Iterations",
          "default": 10
        },
        "execution_timeout_seconds": {
          "anyOf": [
            {
              "type": "integer"
            },
            {
              "type": "null"
            }
          ],
          "title": "Execution Timeout Seconds",
          "default": 60
        },
        "should_retry_on_error": {
          "anyOf": [
            {
              "type": "boolean"
            },
            {
              "type": "null"
            }
          ],
          "title": "Should Retry On Error",
          "default": true
        }
      },
      "type": "object",
      "required": [
        "name"
      ],
      "title": "ToolConfig",
      "description": "Configuration for tool use.\nNOTE: this config is highly experimental and signature will change significantly in future iterations."
    }
  }
}