With Portkey, you can perform batch inference operations with Vertex AI models. This is the most efficient way to:Documentation Index
Fetch the complete documentation index at: https://portkey-docs-feat-support-overview-page.mintlify.app/llms.txt
Use this file to discover all available pages before exploring further.
- Process large volumes of data with Vertex AI models
- Test your data with different foundation models
- Perform A/B testing with different foundation models
Upload a file for batch inference
- Python
- NodeJS
- OpenAI Python
- OpenAI NodeJS
- REST API
from portkey_ai import Portkey
# Initialize the Portkey client
portkey = Portkey(
api_key="PORTKEY_API_KEY", # Replace with your Portkey API key
virtual_key="VERTEX_VIRTUAL_KEY", # Add your Vertex virtual key
vertex_storage_bucket_name="your_bucket_name", # Specify the GCS bucket name
provider_file_name="your_file_name.jsonl", # Specify the file name in GCS
provider_model="gemini-1.5-flash-001" # Specify the model to use
)
# Upload a file for batch inference
file = portkey.files.create(
file=open("dataset.jsonl", "rb"),
purpose="batch"
)
print(file)
import { Portkey } from "portkey-ai";
import * as fs from 'fs';
// Initialize the Portkey client
const portkey = Portkey({
apiKey: "PORTKEY_API_KEY", // Replace with your Portkey API key
virtualKey: "VERTEX_VIRTUAL_KEY", // Add your Vertex virtual key
vertexStorageBucketName: "your_bucket_name", // Specify the GCS bucket name
providerFileName: "your_file_name.jsonl", // Specify the file name in GCS
providerModel: "gemini-1.5-flash-001" // Specify the model to use
});
(async () => {
// Upload a file for batch inference
const file = await portkey.files.create({
file: fs.createReadStream("dataset.jsonl"),
purpose: "batch"
});
console.log(file);
})();
from openai import OpenAI
from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders
openai = OpenAI(
api_key='OPENAI_API_KEY',
base_url=PORTKEY_GATEWAY_URL,
default_headers=createHeaders(
virtual_key="VERTEX_VIRTUAL_KEY",
api_key="PORTKEY_API_KEY",
vertex_storage_bucket_name="your_bucket_name",
provider_file_name="your_file_name.jsonl",
provider_model="gemini-1.5-flash-001"
)
)
# Upload a file for batch inference
file = openai.files.create(
file=open("dataset.jsonl", "rb"),
purpose="batch"
)
print(file)
import OpenAI from 'openai';
import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai';
import * as fs from 'fs';
const openai = new OpenAI({
apiKey: 'OPENAI_API_KEY',
baseURL: PORTKEY_GATEWAY_URL,
defaultHeaders: createHeaders({
virtualKey: "VERTEX_VIRTUAL_KEY",
apiKey: "PORTKEY_API_KEY",
vertexStorageBucketName: "your_bucket_name",
providerFileName: "your_file_name.jsonl",
providerModel: "gemini-1.5-flash-001"
})
});
(async () => {
// Upload a file for batch inference
const file = await openai.files.create({
file: fs.createReadStream("dataset.jsonl"),
purpose: "batch"
});
console.log(file);
})();
curl -X POST --header 'x-portkey-api-key: <portkey_api_key>' \
--header 'x-portkey-virtual-key: <vertex_virtual_key>' \
--header 'x-portkey-vertex-storage-bucket-name: <bucket_name>' \
--header 'x-portkey-provider-file-name: <file_name>.jsonl' \
--header 'x-portkey-provider-model: <model_name>' \
--form 'purpose="batch"' \
--form 'file=@dataset.jsonl' \
'https://api.portkey.ai/v1/files'
Create a batch job
- Python
- NodeJS
- OpenAI Python
- OpenAI NodeJS
- REST API
from portkey_ai import Portkey
# Initialize the Portkey client
portkey = Portkey(
api_key="PORTKEY_API_KEY", # Replace with your Portkey API key
virtual_key="VERTEX_VIRTUAL_KEY" # Add your Vertex virtual key
)
# Create a batch inference job
batch_job = portkey.batches.create(
input_file_id="<file_id>", # File ID from the upload step
endpoint="/v1/chat/completions", # API endpoint to use
completion_window="24h", # Time window for completion
model="gemini-1.5-flash-001"
)
print(batch_job)
import { Portkey } from "portkey-ai";
// Initialize the Portkey client
const portkey = Portkey({
apiKey: "PORTKEY_API_KEY", // Replace with your Portkey API key
virtualKey: "VERTEX_VIRTUAL_KEY" // Add your Vertex virtual key
});
(async () => {
// Create a batch inference job
const batchJob = await portkey.batches.create({
input_file_id: "<file_id>", // File ID from the upload step
endpoint: "/v1/chat/completions", // API endpoint to use
completion_window: "24h", // Time window for completion
model:"gemini-1.5-flash-001"
});
console.log(batchJob);
})();
from openai import OpenAI
from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders
openai = OpenAI(
api_key='OPENAI_API_KEY',
base_url=PORTKEY_GATEWAY_URL,
default_headers=createHeaders(
virtual_key="VERTEX_VIRTUAL_KEY",
api_key="PORTKEY_API_KEY"
)
)
# Create a batch inference job
batch_job = openai.batches.create(
input_file_id="<file_id>", # File ID from the upload step
endpoint="/v1/chat/completions", # API endpoint to use
completion_window="24h", # Time window for completion
model="gemini-1.5-flash-001"
)
print(batch_job)
import OpenAI from 'openai';
import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai';
const openai = new OpenAI({
apiKey: 'OPENAI_API_KEY',
baseURL: PORTKEY_GATEWAY_URL,
defaultHeaders: createHeaders({
virtualKey: "VERTEX_VIRTUAL_KEY",
apiKey: "PORTKEY_API_KEY"
})
});
(async () => {
// Create a batch inference job
const batchJob = await openai.batches.create({
input_file_id: "<file_id>", // File ID from the upload step
endpoint: "/v1/chat/completions", // API endpoint to use
completion_window: "24h", // Time window for completion
model:"gemini-1.5-flash-001"
});
console.log(batchJob);
})();
curl -X POST --header 'Content-Type: application/json' \
--header 'x-portkey-api-key: <portkey_api_key>' \
--header 'x-portkey-virtual-key: <vertex_virtual_key>' \
--data \
$'{"input_file_id": "<file_id>", "endpoint": "/v1/chat/completions", "completion_window": "24h", "model":"gemini-1.5-flash-001"}' \
'https://api.portkey.ai/v1/batches'
List batch jobs
- Python
- NodeJS
- OpenAI Python
- OpenAI NodeJS
- REST API
from portkey_ai import Portkey
# Initialize the Portkey client
portkey = Portkey(
api_key="PORTKEY_API_KEY", # Replace with your Portkey API key
virtual_key="VERTEX_VIRTUAL_KEY" # Add your Vertex virtual key
)
# List all batch jobs
jobs = portkey.batches.list(
limit=10 # Optional: Number of jobs to retrieve (default: 20)
)
print(jobs)
import { Portkey } from "portkey-ai";
// Initialize the Portkey client
const portkey = Portkey({
apiKey: "PORTKEY_API_KEY", // Replace with your Portkey API key
virtualKey: "VERTEX_VIRTUAL_KEY" // Add your Vertex virtual key
});
(async () => {
// List all batch jobs
const jobs = await portkey.batches.list({
limit: 10 // Optional: Number of jobs to retrieve (default: 20)
});
console.log(jobs);
})();
from openai import OpenAI
from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders
openai = OpenAI(
api_key='OPENAI_API_KEY',
base_url=PORTKEY_GATEWAY_URL,
default_headers=createHeaders(
virtual_key="VERTEX_VIRTUAL_KEY",
api_key="PORTKEY_API_KEY"
)
)
# List all batch jobs
jobs = openai.batches.list(
limit=10 # Optional: Number of jobs to retrieve (default: 20)
)
print(jobs)
import OpenAI from 'openai';
import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai';
const openai = new OpenAI({
apiKey: 'OPENAI_API_KEY',
baseURL: PORTKEY_GATEWAY_URL,
defaultHeaders: createHeaders({
virtualKey: "VERTEX_VIRTUAL_KEY",
apiKey: "PORTKEY_API_KEY"
})
});
(async () => {
// List all batch jobs
const jobs = await openai.batches.list({
limit: 10 // Optional:
});
console.log(jobs);
})();
curl -X GET --header 'x-portkey-api-key: <portkey_api_key>' \
--header 'x-portkey-virtual-key: <vertex_virtual_key>' \
'https://api.portkey.ai/v1/batches'
Get a batch job
- Python
- NodeJS
- OpenAI Python
- OpenAI NodeJS
- REST API
from portkey_ai import Portkey
# Initialize the Portkey client
portkey = Portkey(
api_key="PORTKEY_API_KEY", # Replace with your Portkey API key
virtual_key="VERTEX_VIRTUAL_KEY" # Add your Vertex virtual key
)
# Retrieve a specific batch job
job = portkey.batches.retrieve(
"job_id" # The ID of the batch job to retrieve
)
print(job)
import { Portkey } from "portkey-ai";
// Initialize the Portkey client
const portkey = Portkey({
apiKey: "PORTKEY_API_KEY", // Replace with your Portkey API key
virtualKey: "VERTEX_VIRTUAL_KEY" // Add your Vertex virtual key
});
(async () => {
// Retrieve a specific batch job
const job = await portkey.batches.retrieve(
"job_id" // The ID of the batch job to retrieve
);
console.log(job);
})();
from openai import OpenAI
from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders
openai = OpenAI(
api_key='OPENAI_API_KEY',
base_url=PORTKEY_GATEWAY_URL,
default_headers=createHeaders(
virtual_key="VERTEX_VIRTUAL_KEY",
api_key="PORTKEY_API_KEY"
)
)
# Retrieve a specific batch job
job = openai.batches.retrieve(
"job_id" # The ID of the batch job to retrieve
)
print(job)
import OpenAI from 'openai';
import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai';
const openai = new OpenAI({
apiKey: 'OPENAI_API_KEY',
baseURL: PORTKEY_GATEWAY_URL,
defaultHeaders: createHeaders({
virtualKey: "VERTEX_VIRTUAL_KEY",
apiKey: "PORTKEY_API_KEY"
})
});
(async () => {
// Retrieve a specific batch job
const job = await openai.batches.retrieve(
"job_id" // The ID of the batch job to retrieve
);
console.log(job);
})();
curl -X GET --header 'x-portkey-api-key: <portkey_api_key>' \
--header 'x-portkey-virtual-key: <vertex_virtual_key>' \
'https://api.portkey.ai/v1/batches/<job_id>'
Get batch job output
- REST API
curl -X GET --header 'x-portkey-api-key: <portkey_api_key>' \
--header 'x-portkey-virtual-key: <vertex_virtual_key>' \
'https://api.portkey.ai/v1/batches/<job_id>/output'