Create Deployment
curl --request POST \
--url https://{host}/api/model-deployments \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"name": "<string>",
"model_source": {
"kind": "artifact",
"run_id": "<string>"
},
"config": {
"engine": "vllm",
"gpus": 1,
"dtype": "auto",
"quantization": "none",
"max_model_len": 2097216,
"gpu_memory_utilization": 0.9,
"max_concurrency": 32768,
"trust_remote_code": false,
"served_model_name": "model",
"tensor_parallel_size": 32,
"pipeline_parallel_size": 32,
"data_parallel_size": 32,
"extra_args": [
"<string>"
],
"env": {},
"vllm": {
"cpu_offload_gib": 0,
"kv_cache_dtype": "auto",
"enforce_eager": false,
"max_num_batched_tokens": 524320,
"enable_prefix_caching": false,
"enable_chunked_prefill": true,
"tool_call_parser": "<string>",
"reasoning_parser": "<string>",
"chat_template": "<string>"
},
"sglang": {
"mem_fraction_static": 0.88,
"chunked_prefill_size": 524352,
"schedule_policy": "lpm",
"max_running_requests": 32768,
"disable_radix_cache": false,
"tool_call_parser": "<string>",
"reasoning_parser": "<string>",
"chat_template": "<string>"
}
},
"fleet_id": "",
"serve_mode": "proxied"
}
'import requests
url = "https://{host}/api/model-deployments"
payload = {
"name": "<string>",
"model_source": {
"kind": "artifact",
"run_id": "<string>"
},
"config": {
"engine": "vllm",
"gpus": 1,
"dtype": "auto",
"quantization": "none",
"max_model_len": 2097216,
"gpu_memory_utilization": 0.9,
"max_concurrency": 32768,
"trust_remote_code": False,
"served_model_name": "model",
"tensor_parallel_size": 32,
"pipeline_parallel_size": 32,
"data_parallel_size": 32,
"extra_args": ["<string>"],
"env": {},
"vllm": {
"cpu_offload_gib": 0,
"kv_cache_dtype": "auto",
"enforce_eager": False,
"max_num_batched_tokens": 524320,
"enable_prefix_caching": False,
"enable_chunked_prefill": True,
"tool_call_parser": "<string>",
"reasoning_parser": "<string>",
"chat_template": "<string>"
},
"sglang": {
"mem_fraction_static": 0.88,
"chunked_prefill_size": 524352,
"schedule_policy": "lpm",
"max_running_requests": 32768,
"disable_radix_cache": False,
"tool_call_parser": "<string>",
"reasoning_parser": "<string>",
"chat_template": "<string>"
}
},
"fleet_id": "",
"serve_mode": "proxied"
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
name: '<string>',
model_source: {kind: 'artifact', run_id: '<string>'},
config: {
engine: 'vllm',
gpus: 1,
dtype: 'auto',
quantization: 'none',
max_model_len: 2097216,
gpu_memory_utilization: 0.9,
max_concurrency: 32768,
trust_remote_code: false,
served_model_name: 'model',
tensor_parallel_size: 32,
pipeline_parallel_size: 32,
data_parallel_size: 32,
extra_args: ['<string>'],
env: {},
vllm: {
cpu_offload_gib: 0,
kv_cache_dtype: 'auto',
enforce_eager: false,
max_num_batched_tokens: 524320,
enable_prefix_caching: false,
enable_chunked_prefill: true,
tool_call_parser: '<string>',
reasoning_parser: '<string>',
chat_template: '<string>'
},
sglang: {
mem_fraction_static: 0.88,
chunked_prefill_size: 524352,
schedule_policy: 'lpm',
max_running_requests: 32768,
disable_radix_cache: false,
tool_call_parser: '<string>',
reasoning_parser: '<string>',
chat_template: '<string>'
}
},
fleet_id: '',
serve_mode: 'proxied'
})
};
fetch('https://{host}/api/model-deployments', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://{host}/api/model-deployments",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'name' => '<string>',
'model_source' => [
'kind' => 'artifact',
'run_id' => '<string>'
],
'config' => [
'engine' => 'vllm',
'gpus' => 1,
'dtype' => 'auto',
'quantization' => 'none',
'max_model_len' => 2097216,
'gpu_memory_utilization' => 0.9,
'max_concurrency' => 32768,
'trust_remote_code' => false,
'served_model_name' => 'model',
'tensor_parallel_size' => 32,
'pipeline_parallel_size' => 32,
'data_parallel_size' => 32,
'extra_args' => [
'<string>'
],
'env' => [
],
'vllm' => [
'cpu_offload_gib' => 0,
'kv_cache_dtype' => 'auto',
'enforce_eager' => false,
'max_num_batched_tokens' => 524320,
'enable_prefix_caching' => false,
'enable_chunked_prefill' => true,
'tool_call_parser' => '<string>',
'reasoning_parser' => '<string>',
'chat_template' => '<string>'
],
'sglang' => [
'mem_fraction_static' => 0.88,
'chunked_prefill_size' => 524352,
'schedule_policy' => 'lpm',
'max_running_requests' => 32768,
'disable_radix_cache' => false,
'tool_call_parser' => '<string>',
'reasoning_parser' => '<string>',
'chat_template' => '<string>'
]
],
'fleet_id' => '',
'serve_mode' => 'proxied'
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://{host}/api/model-deployments"
payload := strings.NewReader("{\n \"name\": \"<string>\",\n \"model_source\": {\n \"kind\": \"artifact\",\n \"run_id\": \"<string>\"\n },\n \"config\": {\n \"engine\": \"vllm\",\n \"gpus\": 1,\n \"dtype\": \"auto\",\n \"quantization\": \"none\",\n \"max_model_len\": 2097216,\n \"gpu_memory_utilization\": 0.9,\n \"max_concurrency\": 32768,\n \"trust_remote_code\": false,\n \"served_model_name\": \"model\",\n \"tensor_parallel_size\": 32,\n \"pipeline_parallel_size\": 32,\n \"data_parallel_size\": 32,\n \"extra_args\": [\n \"<string>\"\n ],\n \"env\": {},\n \"vllm\": {\n \"cpu_offload_gib\": 0,\n \"kv_cache_dtype\": \"auto\",\n \"enforce_eager\": false,\n \"max_num_batched_tokens\": 524320,\n \"enable_prefix_caching\": false,\n \"enable_chunked_prefill\": true,\n \"tool_call_parser\": \"<string>\",\n \"reasoning_parser\": \"<string>\",\n \"chat_template\": \"<string>\"\n },\n \"sglang\": {\n \"mem_fraction_static\": 0.88,\n \"chunked_prefill_size\": 524352,\n \"schedule_policy\": \"lpm\",\n \"max_running_requests\": 32768,\n \"disable_radix_cache\": false,\n \"tool_call_parser\": \"<string>\",\n \"reasoning_parser\": \"<string>\",\n \"chat_template\": \"<string>\"\n }\n },\n \"fleet_id\": \"\",\n \"serve_mode\": \"proxied\"\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://{host}/api/model-deployments")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"name\": \"<string>\",\n \"model_source\": {\n \"kind\": \"artifact\",\n \"run_id\": \"<string>\"\n },\n \"config\": {\n \"engine\": \"vllm\",\n \"gpus\": 1,\n \"dtype\": \"auto\",\n \"quantization\": \"none\",\n \"max_model_len\": 2097216,\n \"gpu_memory_utilization\": 0.9,\n \"max_concurrency\": 32768,\n \"trust_remote_code\": false,\n \"served_model_name\": \"model\",\n \"tensor_parallel_size\": 32,\n \"pipeline_parallel_size\": 32,\n \"data_parallel_size\": 32,\n \"extra_args\": [\n \"<string>\"\n ],\n \"env\": {},\n \"vllm\": {\n \"cpu_offload_gib\": 0,\n \"kv_cache_dtype\": \"auto\",\n \"enforce_eager\": false,\n \"max_num_batched_tokens\": 524320,\n \"enable_prefix_caching\": false,\n \"enable_chunked_prefill\": true,\n \"tool_call_parser\": \"<string>\",\n \"reasoning_parser\": \"<string>\",\n \"chat_template\": \"<string>\"\n },\n \"sglang\": {\n \"mem_fraction_static\": 0.88,\n \"chunked_prefill_size\": 524352,\n \"schedule_policy\": \"lpm\",\n \"max_running_requests\": 32768,\n \"disable_radix_cache\": false,\n \"tool_call_parser\": \"<string>\",\n \"reasoning_parser\": \"<string>\",\n \"chat_template\": \"<string>\"\n }\n },\n \"fleet_id\": \"\",\n \"serve_mode\": \"proxied\"\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://{host}/api/model-deployments")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"name\": \"<string>\",\n \"model_source\": {\n \"kind\": \"artifact\",\n \"run_id\": \"<string>\"\n },\n \"config\": {\n \"engine\": \"vllm\",\n \"gpus\": 1,\n \"dtype\": \"auto\",\n \"quantization\": \"none\",\n \"max_model_len\": 2097216,\n \"gpu_memory_utilization\": 0.9,\n \"max_concurrency\": 32768,\n \"trust_remote_code\": false,\n \"served_model_name\": \"model\",\n \"tensor_parallel_size\": 32,\n \"pipeline_parallel_size\": 32,\n \"data_parallel_size\": 32,\n \"extra_args\": [\n \"<string>\"\n ],\n \"env\": {},\n \"vllm\": {\n \"cpu_offload_gib\": 0,\n \"kv_cache_dtype\": \"auto\",\n \"enforce_eager\": false,\n \"max_num_batched_tokens\": 524320,\n \"enable_prefix_caching\": false,\n \"enable_chunked_prefill\": true,\n \"tool_call_parser\": \"<string>\",\n \"reasoning_parser\": \"<string>\",\n \"chat_template\": \"<string>\"\n },\n \"sglang\": {\n \"mem_fraction_static\": 0.88,\n \"chunked_prefill_size\": 524352,\n \"schedule_policy\": \"lpm\",\n \"max_running_requests\": 32768,\n \"disable_radix_cache\": false,\n \"tool_call_parser\": \"<string>\",\n \"reasoning_parser\": \"<string>\",\n \"chat_template\": \"<string>\"\n }\n },\n \"fleet_id\": \"\",\n \"serve_mode\": \"proxied\"\n}"
response = http.request(request)
puts response.read_body{
"deployment": {
"id": "<string>",
"name": "<string>",
"owner_user_id": 123,
"owner_username": "<string>",
"desired_state": "<string>",
"status": "<string>",
"endpoint_path": "<string>",
"created_at": "2023-11-07T05:31:56Z",
"updated_at": "2023-11-07T05:31:56Z",
"fleet_id": "",
"serve_mode": "proxied",
"current_revision_id": "<string>",
"candidate_revision_id": "<string>",
"last_error": "<string>",
"current_revision": {
"id": "<string>",
"deployment_id": "<string>",
"sequence": 123,
"engine": "<string>",
"model_source": {},
"config": {},
"status": "<string>",
"created_by": "<string>",
"created_at": "2023-11-07T05:31:56Z",
"runtime_ref": "<string>",
"runtime_meta": {},
"internal_endpoint": "<string>",
"error": "<string>",
"ready_at": "2023-11-07T05:31:56Z",
"retired_at": "2023-11-07T05:31:56Z"
},
"candidate_revision": {
"id": "<string>",
"deployment_id": "<string>",
"sequence": 123,
"engine": "<string>",
"model_source": {},
"config": {},
"status": "<string>",
"created_by": "<string>",
"created_at": "2023-11-07T05:31:56Z",
"runtime_ref": "<string>",
"runtime_meta": {},
"internal_endpoint": "<string>",
"error": "<string>",
"ready_at": "2023-11-07T05:31:56Z",
"retired_at": "2023-11-07T05:31:56Z"
},
"revisions": [
{
"id": "<string>",
"deployment_id": "<string>",
"sequence": 123,
"engine": "<string>",
"model_source": {},
"config": {},
"status": "<string>",
"created_by": "<string>",
"created_at": "2023-11-07T05:31:56Z",
"runtime_ref": "<string>",
"runtime_meta": {},
"internal_endpoint": "<string>",
"error": "<string>",
"ready_at": "2023-11-07T05:31:56Z",
"retired_at": "2023-11-07T05:31:56Z"
}
],
"tokens": [
{
"id": "<string>",
"deployment_id": "<string>",
"name": "<string>",
"prefix": "<string>",
"created_by": "<string>",
"created_at": "2023-11-07T05:31:56Z",
"last_used_at": "2023-11-07T05:31:56Z",
"revoked_at": "2023-11-07T05:31:56Z"
}
],
"reflow_sample_rate": 0,
"reflow_retention_days": 30,
"reflow_capture_system_prompt": true,
"reflow_dropped_total": 0,
"reflow_trimmed_total": 0
},
"initial_token": {
"id": "<string>",
"deployment_id": "<string>",
"name": "<string>",
"prefix": "<string>",
"created_by": "<string>",
"created_at": "2023-11-07T05:31:56Z",
"token": "<string>",
"last_used_at": "2023-11-07T05:31:56Z",
"revoked_at": "2023-11-07T05:31:56Z"
}
}{
"detail": [
{
"loc": [
"<string>"
],
"msg": "<string>",
"type": "<string>",
"input": "<unknown>",
"ctx": {}
}
]
}Create Deployment
POST
/
api
/
model-deployments
Create Deployment
curl --request POST \
--url https://{host}/api/model-deployments \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"name": "<string>",
"model_source": {
"kind": "artifact",
"run_id": "<string>"
},
"config": {
"engine": "vllm",
"gpus": 1,
"dtype": "auto",
"quantization": "none",
"max_model_len": 2097216,
"gpu_memory_utilization": 0.9,
"max_concurrency": 32768,
"trust_remote_code": false,
"served_model_name": "model",
"tensor_parallel_size": 32,
"pipeline_parallel_size": 32,
"data_parallel_size": 32,
"extra_args": [
"<string>"
],
"env": {},
"vllm": {
"cpu_offload_gib": 0,
"kv_cache_dtype": "auto",
"enforce_eager": false,
"max_num_batched_tokens": 524320,
"enable_prefix_caching": false,
"enable_chunked_prefill": true,
"tool_call_parser": "<string>",
"reasoning_parser": "<string>",
"chat_template": "<string>"
},
"sglang": {
"mem_fraction_static": 0.88,
"chunked_prefill_size": 524352,
"schedule_policy": "lpm",
"max_running_requests": 32768,
"disable_radix_cache": false,
"tool_call_parser": "<string>",
"reasoning_parser": "<string>",
"chat_template": "<string>"
}
},
"fleet_id": "",
"serve_mode": "proxied"
}
'import requests
url = "https://{host}/api/model-deployments"
payload = {
"name": "<string>",
"model_source": {
"kind": "artifact",
"run_id": "<string>"
},
"config": {
"engine": "vllm",
"gpus": 1,
"dtype": "auto",
"quantization": "none",
"max_model_len": 2097216,
"gpu_memory_utilization": 0.9,
"max_concurrency": 32768,
"trust_remote_code": False,
"served_model_name": "model",
"tensor_parallel_size": 32,
"pipeline_parallel_size": 32,
"data_parallel_size": 32,
"extra_args": ["<string>"],
"env": {},
"vllm": {
"cpu_offload_gib": 0,
"kv_cache_dtype": "auto",
"enforce_eager": False,
"max_num_batched_tokens": 524320,
"enable_prefix_caching": False,
"enable_chunked_prefill": True,
"tool_call_parser": "<string>",
"reasoning_parser": "<string>",
"chat_template": "<string>"
},
"sglang": {
"mem_fraction_static": 0.88,
"chunked_prefill_size": 524352,
"schedule_policy": "lpm",
"max_running_requests": 32768,
"disable_radix_cache": False,
"tool_call_parser": "<string>",
"reasoning_parser": "<string>",
"chat_template": "<string>"
}
},
"fleet_id": "",
"serve_mode": "proxied"
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
name: '<string>',
model_source: {kind: 'artifact', run_id: '<string>'},
config: {
engine: 'vllm',
gpus: 1,
dtype: 'auto',
quantization: 'none',
max_model_len: 2097216,
gpu_memory_utilization: 0.9,
max_concurrency: 32768,
trust_remote_code: false,
served_model_name: 'model',
tensor_parallel_size: 32,
pipeline_parallel_size: 32,
data_parallel_size: 32,
extra_args: ['<string>'],
env: {},
vllm: {
cpu_offload_gib: 0,
kv_cache_dtype: 'auto',
enforce_eager: false,
max_num_batched_tokens: 524320,
enable_prefix_caching: false,
enable_chunked_prefill: true,
tool_call_parser: '<string>',
reasoning_parser: '<string>',
chat_template: '<string>'
},
sglang: {
mem_fraction_static: 0.88,
chunked_prefill_size: 524352,
schedule_policy: 'lpm',
max_running_requests: 32768,
disable_radix_cache: false,
tool_call_parser: '<string>',
reasoning_parser: '<string>',
chat_template: '<string>'
}
},
fleet_id: '',
serve_mode: 'proxied'
})
};
fetch('https://{host}/api/model-deployments', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://{host}/api/model-deployments",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'name' => '<string>',
'model_source' => [
'kind' => 'artifact',
'run_id' => '<string>'
],
'config' => [
'engine' => 'vllm',
'gpus' => 1,
'dtype' => 'auto',
'quantization' => 'none',
'max_model_len' => 2097216,
'gpu_memory_utilization' => 0.9,
'max_concurrency' => 32768,
'trust_remote_code' => false,
'served_model_name' => 'model',
'tensor_parallel_size' => 32,
'pipeline_parallel_size' => 32,
'data_parallel_size' => 32,
'extra_args' => [
'<string>'
],
'env' => [
],
'vllm' => [
'cpu_offload_gib' => 0,
'kv_cache_dtype' => 'auto',
'enforce_eager' => false,
'max_num_batched_tokens' => 524320,
'enable_prefix_caching' => false,
'enable_chunked_prefill' => true,
'tool_call_parser' => '<string>',
'reasoning_parser' => '<string>',
'chat_template' => '<string>'
],
'sglang' => [
'mem_fraction_static' => 0.88,
'chunked_prefill_size' => 524352,
'schedule_policy' => 'lpm',
'max_running_requests' => 32768,
'disable_radix_cache' => false,
'tool_call_parser' => '<string>',
'reasoning_parser' => '<string>',
'chat_template' => '<string>'
]
],
'fleet_id' => '',
'serve_mode' => 'proxied'
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://{host}/api/model-deployments"
payload := strings.NewReader("{\n \"name\": \"<string>\",\n \"model_source\": {\n \"kind\": \"artifact\",\n \"run_id\": \"<string>\"\n },\n \"config\": {\n \"engine\": \"vllm\",\n \"gpus\": 1,\n \"dtype\": \"auto\",\n \"quantization\": \"none\",\n \"max_model_len\": 2097216,\n \"gpu_memory_utilization\": 0.9,\n \"max_concurrency\": 32768,\n \"trust_remote_code\": false,\n \"served_model_name\": \"model\",\n \"tensor_parallel_size\": 32,\n \"pipeline_parallel_size\": 32,\n \"data_parallel_size\": 32,\n \"extra_args\": [\n \"<string>\"\n ],\n \"env\": {},\n \"vllm\": {\n \"cpu_offload_gib\": 0,\n \"kv_cache_dtype\": \"auto\",\n \"enforce_eager\": false,\n \"max_num_batched_tokens\": 524320,\n \"enable_prefix_caching\": false,\n \"enable_chunked_prefill\": true,\n \"tool_call_parser\": \"<string>\",\n \"reasoning_parser\": \"<string>\",\n \"chat_template\": \"<string>\"\n },\n \"sglang\": {\n \"mem_fraction_static\": 0.88,\n \"chunked_prefill_size\": 524352,\n \"schedule_policy\": \"lpm\",\n \"max_running_requests\": 32768,\n \"disable_radix_cache\": false,\n \"tool_call_parser\": \"<string>\",\n \"reasoning_parser\": \"<string>\",\n \"chat_template\": \"<string>\"\n }\n },\n \"fleet_id\": \"\",\n \"serve_mode\": \"proxied\"\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://{host}/api/model-deployments")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"name\": \"<string>\",\n \"model_source\": {\n \"kind\": \"artifact\",\n \"run_id\": \"<string>\"\n },\n \"config\": {\n \"engine\": \"vllm\",\n \"gpus\": 1,\n \"dtype\": \"auto\",\n \"quantization\": \"none\",\n \"max_model_len\": 2097216,\n \"gpu_memory_utilization\": 0.9,\n \"max_concurrency\": 32768,\n \"trust_remote_code\": false,\n \"served_model_name\": \"model\",\n \"tensor_parallel_size\": 32,\n \"pipeline_parallel_size\": 32,\n \"data_parallel_size\": 32,\n \"extra_args\": [\n \"<string>\"\n ],\n \"env\": {},\n \"vllm\": {\n \"cpu_offload_gib\": 0,\n \"kv_cache_dtype\": \"auto\",\n \"enforce_eager\": false,\n \"max_num_batched_tokens\": 524320,\n \"enable_prefix_caching\": false,\n \"enable_chunked_prefill\": true,\n \"tool_call_parser\": \"<string>\",\n \"reasoning_parser\": \"<string>\",\n \"chat_template\": \"<string>\"\n },\n \"sglang\": {\n \"mem_fraction_static\": 0.88,\n \"chunked_prefill_size\": 524352,\n \"schedule_policy\": \"lpm\",\n \"max_running_requests\": 32768,\n \"disable_radix_cache\": false,\n \"tool_call_parser\": \"<string>\",\n \"reasoning_parser\": \"<string>\",\n \"chat_template\": \"<string>\"\n }\n },\n \"fleet_id\": \"\",\n \"serve_mode\": \"proxied\"\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://{host}/api/model-deployments")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"name\": \"<string>\",\n \"model_source\": {\n \"kind\": \"artifact\",\n \"run_id\": \"<string>\"\n },\n \"config\": {\n \"engine\": \"vllm\",\n \"gpus\": 1,\n \"dtype\": \"auto\",\n \"quantization\": \"none\",\n \"max_model_len\": 2097216,\n \"gpu_memory_utilization\": 0.9,\n \"max_concurrency\": 32768,\n \"trust_remote_code\": false,\n \"served_model_name\": \"model\",\n \"tensor_parallel_size\": 32,\n \"pipeline_parallel_size\": 32,\n \"data_parallel_size\": 32,\n \"extra_args\": [\n \"<string>\"\n ],\n \"env\": {},\n \"vllm\": {\n \"cpu_offload_gib\": 0,\n \"kv_cache_dtype\": \"auto\",\n \"enforce_eager\": false,\n \"max_num_batched_tokens\": 524320,\n \"enable_prefix_caching\": false,\n \"enable_chunked_prefill\": true,\n \"tool_call_parser\": \"<string>\",\n \"reasoning_parser\": \"<string>\",\n \"chat_template\": \"<string>\"\n },\n \"sglang\": {\n \"mem_fraction_static\": 0.88,\n \"chunked_prefill_size\": 524352,\n \"schedule_policy\": \"lpm\",\n \"max_running_requests\": 32768,\n \"disable_radix_cache\": false,\n \"tool_call_parser\": \"<string>\",\n \"reasoning_parser\": \"<string>\",\n \"chat_template\": \"<string>\"\n }\n },\n \"fleet_id\": \"\",\n \"serve_mode\": \"proxied\"\n}"
response = http.request(request)
puts response.read_body{
"deployment": {
"id": "<string>",
"name": "<string>",
"owner_user_id": 123,
"owner_username": "<string>",
"desired_state": "<string>",
"status": "<string>",
"endpoint_path": "<string>",
"created_at": "2023-11-07T05:31:56Z",
"updated_at": "2023-11-07T05:31:56Z",
"fleet_id": "",
"serve_mode": "proxied",
"current_revision_id": "<string>",
"candidate_revision_id": "<string>",
"last_error": "<string>",
"current_revision": {
"id": "<string>",
"deployment_id": "<string>",
"sequence": 123,
"engine": "<string>",
"model_source": {},
"config": {},
"status": "<string>",
"created_by": "<string>",
"created_at": "2023-11-07T05:31:56Z",
"runtime_ref": "<string>",
"runtime_meta": {},
"internal_endpoint": "<string>",
"error": "<string>",
"ready_at": "2023-11-07T05:31:56Z",
"retired_at": "2023-11-07T05:31:56Z"
},
"candidate_revision": {
"id": "<string>",
"deployment_id": "<string>",
"sequence": 123,
"engine": "<string>",
"model_source": {},
"config": {},
"status": "<string>",
"created_by": "<string>",
"created_at": "2023-11-07T05:31:56Z",
"runtime_ref": "<string>",
"runtime_meta": {},
"internal_endpoint": "<string>",
"error": "<string>",
"ready_at": "2023-11-07T05:31:56Z",
"retired_at": "2023-11-07T05:31:56Z"
},
"revisions": [
{
"id": "<string>",
"deployment_id": "<string>",
"sequence": 123,
"engine": "<string>",
"model_source": {},
"config": {},
"status": "<string>",
"created_by": "<string>",
"created_at": "2023-11-07T05:31:56Z",
"runtime_ref": "<string>",
"runtime_meta": {},
"internal_endpoint": "<string>",
"error": "<string>",
"ready_at": "2023-11-07T05:31:56Z",
"retired_at": "2023-11-07T05:31:56Z"
}
],
"tokens": [
{
"id": "<string>",
"deployment_id": "<string>",
"name": "<string>",
"prefix": "<string>",
"created_by": "<string>",
"created_at": "2023-11-07T05:31:56Z",
"last_used_at": "2023-11-07T05:31:56Z",
"revoked_at": "2023-11-07T05:31:56Z"
}
],
"reflow_sample_rate": 0,
"reflow_retention_days": 30,
"reflow_capture_system_prompt": true,
"reflow_dropped_total": 0,
"reflow_trimmed_total": 0
},
"initial_token": {
"id": "<string>",
"deployment_id": "<string>",
"name": "<string>",
"prefix": "<string>",
"created_by": "<string>",
"created_at": "2023-11-07T05:31:56Z",
"token": "<string>",
"last_used_at": "2023-11-07T05:31:56Z",
"revoked_at": "2023-11-07T05:31:56Z"
}
}{
"detail": [
{
"loc": [
"<string>"
],
"msg": "<string>",
"type": "<string>",
"input": "<unknown>",
"ctx": {}
}
]
}授权
Bearer authentication header of the form Bearer <token>, where <token> is your auth token.
请求体
application/json
Required string length:
2 - 128Pattern:
^[A-Za-z0-9][A-Za-z0-9._-]*$- ArtifactModelSource
- HuggingFaceModelSource
- SharedPathModelSource
- RegistryModelSource
Show child attributes
Show child attributes
Show child attributes
Show child attributes
可用选项:
proxied, direct