Skip to main content
POST
/
pipelines
Create New Pipeline
curl --request POST \
  --url https://api.streamkap.com/pipelines \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "name": "<string>",
  "destination": {
    "connector": "<string>",
    "name": "<string>",
    "id": "<string>"
  },
  "source": {
    "connector": "<string>",
    "name": "<string>",
    "id": "<string>",
    "topics": [
      "<string>"
    ]
  },
  "tags": [
    "<string>"
  ],
  "snapshot_new_tables": false,
  "transforms": [
    {
      "topic_id": "<string>"
    }
  ],
  "topic_auto_discovery_transforms": [],
  "created_from": "api"
}
'
import requests

url = "https://api.streamkap.com/pipelines"

payload = {
    "name": "<string>",
    "destination": {
        "connector": "<string>",
        "name": "<string>",
        "id": "<string>"
    },
    "source": {
        "connector": "<string>",
        "name": "<string>",
        "id": "<string>",
        "topics": ["<string>"]
    },
    "tags": ["<string>"],
    "snapshot_new_tables": False,
    "transforms": [{ "topic_id": "<string>" }],
    "topic_auto_discovery_transforms": [],
    "created_from": "api"
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)
const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    name: '<string>',
    destination: {connector: '<string>', name: '<string>', id: '<string>'},
    source: {connector: '<string>', name: '<string>', id: '<string>', topics: ['<string>']},
    tags: ['<string>'],
    snapshot_new_tables: false,
    transforms: [{topic_id: '<string>'}],
    topic_auto_discovery_transforms: [],
    created_from: 'api'
  })
};

fetch('https://api.streamkap.com/pipelines', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));
<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.streamkap.com/pipelines",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'name' => '<string>',
    'destination' => [
        'connector' => '<string>',
        'name' => '<string>',
        'id' => '<string>'
    ],
    'source' => [
        'connector' => '<string>',
        'name' => '<string>',
        'id' => '<string>',
        'topics' => [
                '<string>'
        ]
    ],
    'tags' => [
        '<string>'
    ],
    'snapshot_new_tables' => false,
    'transforms' => [
        [
                'topic_id' => '<string>'
        ]
    ],
    'topic_auto_discovery_transforms' => [
        
    ],
    'created_from' => 'api'
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}
package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.streamkap.com/pipelines"

	payload := strings.NewReader("{\n  \"name\": \"<string>\",\n  \"destination\": {\n    \"connector\": \"<string>\",\n    \"name\": \"<string>\",\n    \"id\": \"<string>\"\n  },\n  \"source\": {\n    \"connector\": \"<string>\",\n    \"name\": \"<string>\",\n    \"id\": \"<string>\",\n    \"topics\": [\n      \"<string>\"\n    ]\n  },\n  \"tags\": [\n    \"<string>\"\n  ],\n  \"snapshot_new_tables\": false,\n  \"transforms\": [\n    {\n      \"topic_id\": \"<string>\"\n    }\n  ],\n  \"topic_auto_discovery_transforms\": [],\n  \"created_from\": \"api\"\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}
HttpResponse<String> response = Unirest.post("https://api.streamkap.com/pipelines")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"name\": \"<string>\",\n  \"destination\": {\n    \"connector\": \"<string>\",\n    \"name\": \"<string>\",\n    \"id\": \"<string>\"\n  },\n  \"source\": {\n    \"connector\": \"<string>\",\n    \"name\": \"<string>\",\n    \"id\": \"<string>\",\n    \"topics\": [\n      \"<string>\"\n    ]\n  },\n  \"tags\": [\n    \"<string>\"\n  ],\n  \"snapshot_new_tables\": false,\n  \"transforms\": [\n    {\n      \"topic_id\": \"<string>\"\n    }\n  ],\n  \"topic_auto_discovery_transforms\": [],\n  \"created_from\": \"api\"\n}")
  .asString();
require 'uri'
require 'net/http'

url = URI("https://api.streamkap.com/pipelines")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"name\": \"<string>\",\n  \"destination\": {\n    \"connector\": \"<string>\",\n    \"name\": \"<string>\",\n    \"id\": \"<string>\"\n  },\n  \"source\": {\n    \"connector\": \"<string>\",\n    \"name\": \"<string>\",\n    \"id\": \"<string>\",\n    \"topics\": [\n      \"<string>\"\n    ]\n  },\n  \"tags\": [\n    \"<string>\"\n  ],\n  \"snapshot_new_tables\": false,\n  \"transforms\": [\n    {\n      \"topic_id\": \"<string>\"\n    }\n  ],\n  \"topic_auto_discovery_transforms\": [],\n  \"created_from\": \"api\"\n}"

response = http.request(request)
puts response.read_body
{
  "name": "<string>",
  "sub_id": "<string>",
  "tenant_id": "<string>",
  "id": "<string>",
  "snapshot_new_tables": true,
  "topic_ids": [
    "<string>"
  ],
  "topics": [
    "<string>"
  ],
  "source": {
    "name": "<string>",
    "connector": "<string>",
    "id": "<string>",
    "connector_display_name": "<string>",
    "created_timestamp": "<string>",
    "sub_id": "<string>",
    "tenant_id": "<string>",
    "service_id": "<string>",
    "config": {
      "database.hostname.user.defined": "<string>",
      "database.port.user.defined": "5432",
      "database.user": "<string>",
      "database.password": "<string>",
      "database.dbname": "<string>",
      "signal.data.collection.schema.or.database": "<string>",
      "slot.name": "streamkap_pgoutput_slot",
      "publication.name": "streamkap_pub",
      "schema.include.list": "<string>",
      "table.include.list.user.defined": "<string>",
      "ssh.host": "<string>",
      "ssh.port": "22",
      "ssh.user": "streamkap",
      "ssh.public.key.user.displayed": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJVbRrHqNN3SyIYnNF46SMd+ArZz+QheAtjnQzKiBWrc",
      "transforms.ValueToKey.fields.include.list": "<string>",
      "transforms.ValueToKey.replace.null.with.default": true,
      "preserve.null.values": false,
      "transforms.OversizedRecords.fields.include.list": "<string>",
      "transforms.OversizedRecords.fields.exclude.list": "<string>",
      "transforms.OversizedRecords.max.field.size.bytes": 1048576,
      "transforms.OversizedRecords.oversized.field.behavior": "TRUNCATE",
      "transforms.OversizedRecords.truncation.suffix": "",
      "transforms.OversizedRecords.max.record.size.bytes": -1,
      "transforms.OversizedRecords.semantic.types.exclude": "io.debezium.data.Json,io.debezium.data.Xml",
      "transforms.OversizedRecords.replace.null.with.default": true,
      "InsertTopicName.enabled": false,
      "snapshot.read.only.user.defined": "Yes",
      "column.include.list.toggled": true,
      "column.include.list.user.defined": "<string>",
      "column.exclude.list.user.defined": "<string>",
      "heartbeat.enabled": true,
      "heartbeat.data.collection.schema.or.database": "<string>",
      "post.processors": "reselector",
      "database.sslmode": "require",
      "include.source.db.name.in.table.name.user.defined": false,
      "binary.handling.mode": "bytes",
      "transforms.InsertStaticKey1.static.field": "<string>",
      "transforms.InsertStaticKey1.static.value": "<string>",
      "transforms.InsertStaticValue1.static.field": "<string>",
      "transforms.InsertStaticValue1.static.value": "<string>",
      "transforms.InsertStaticKey2.static.field": "<string>",
      "transforms.InsertStaticKey2.static.value": "<string>",
      "transforms.InsertStaticValue2.static.field": "<string>",
      "transforms.InsertStaticValue2.static.value": "<string>",
      "predicates.IsTopicToEnrich.pattern": "$^",
      "ssh.enabled": false
    },
    "topic_ids": [
      "<string>"
    ],
    "topic_map": {},
    "topics": [
      "<string>"
    ],
    "tasks": [
      123
    ],
    "connector_status": "<string>",
    "desired_state": "<string>",
    "kc_cluster_id": "<string>",
    "task_statuses": {},
    "tags": [
      "<string>"
    ]
  },
  "destination": {
    "name": "<string>",
    "connector": "<string>",
    "id": "<string>",
    "connector_display_name": "<string>",
    "created_timestamp": "<string>",
    "sub_id": "<string>",
    "tenant_id": "<string>",
    "config": {
      "bigquery.json": {},
      "table.name.prefix": "<string>",
      "consumer.override.max.poll.records": 10000,
      "tasks.max": 5,
      "preserve.null.values": false,
      "quote.identifiers": true,
      "transforms.ToIntJ.fields.include.list": "<string>",
      "transforms.ToFloatJ.fields.include.list": "<string>",
      "transforms.ToDecimalJ.fields.include.list": "<string>",
      "transforms.ToDecimalJ.truncate.to.max.precision": false,
      "transforms.ToStringJ.fields.include.list": "<string>",
      "transforms.ToJsonJ.fields.include.list": "<string>",
      "transforms.ToJsonJ.convert.all.complex.types.user.defined": false,
      "transforms.ToJsonbJ.fields.include.list": "<string>",
      "transforms.ToJsonbJ.convert.all.complex.types.user.defined": false,
      "transforms.ToJsonbJ.convert.all.json": false,
      "transforms.StringReplace.fields.include.list": "<string>",
      "transforms.StringReplace.regex.patterns": "<string>",
      "transforms.StringReplace.replacement.values": "<string>",
      "transforms.OversizedRecords.fields.include.list": "<string>",
      "transforms.OversizedRecords.fields.exclude.list": "<string>",
      "transforms.OversizedRecords.max.field.size.bytes": 1048576,
      "transforms.OversizedRecords.oversized.field.behavior": "TRUNCATE",
      "transforms.OversizedRecords.truncation.suffix": "",
      "transforms.OversizedRecords.max.record.size.bytes": -1,
      "transforms.OversizedRecords.semantic.types.exclude": "io.debezium.data.Json,io.debezium.data.Xml",
      "transforms.OversizedRecords.replace.null.with.default": true,
      "transforms.AddStringSuffix.fields.include.list": "<string>",
      "transforms.changeTopicName.match.regex.user.defined": "<string>",
      "transforms.RenameFields.renames.user.defined": "<string>",
      "transforms.DropFields.fields.include.list": "<string>",
      "transforms.MarkColumnsAsRequired.fields.include.all": false,
      "transforms.MarkColumnsAsRequired.null.sentinel.mode": "NONE",
      "transforms.MarkColumnsAsOptional.fields.include.list": "<string>",
      "transforms.CopyField.copy.field.mapping": "<string>",
      "transforms.HeaderToFieldCustom.header.mappings": "<string>",
      "bigquery.region": "us-central1",
      "custom.bigquery.cluster.field": "<string>",
      "custom.bigquery.partition.field": "<string>",
      "bigquery.time.based.partition": false
    },
    "topic_ids": [
      "<string>"
    ],
    "topic_map": {},
    "topics": [
      "<string>"
    ],
    "tasks": [
      123
    ],
    "connector_status": "<string>",
    "desired_state": "<string>",
    "kc_cluster_id": "<string>",
    "task_statuses": {},
    "tags": [
      "<string>"
    ]
  },
  "transforms": [
    "<unknown>"
  ],
  "tags": [],
  "periodic_audit": {
    "topics": [
      "<string>"
    ],
    "timestamp_column": "<string>",
    "interval_minutes": 123,
    "fix_deletes_only": false
  },
  "topic_auto_discovery_transforms": []
}
{
  "detail": [
    {
      "loc": [
        "<string>"
      ],
      "msg": "<string>",
      "type": "<string>",
      "input": "<unknown>",
      "ctx": {}
    }
  ]
}

Authorizations

​
Authorization
string
header
required

Bearer authentication header of the form Bearer <token>, where <token> is your auth token.

Query Parameters

​
secret_returned
boolean
default:false

Whether to include secret values in the response

​
wait
boolean
default:true

When False, defer destination config update to reconciler instead of calling KC directly

Body

application/json
​
name
string
required
​
destination
PipelineDestinationConnector · object
required
​
source
PipelineSourceConnector · object
required
​
tags
string[]
required
​
snapshot_new_tables
boolean
default:false
​
transforms
PipelineTransformConnector · object[] | null
​
topic_auto_discovery_transforms
TopicAutoDiscoveryTransform · object[]
​
created_from
enum<string> | null
default:api

Origin of an entity in the system.

Tracks how entities (sources, destinations, pipelines) were created, enabling audit trails and origin-specific behavior.

Available options:
terraform,
web,
api

Response

Successful Response

​
name
string | null
​
sub_id
string | null
​
tenant_id
string | null
​
id
string | null
​
snapshot_new_tables
boolean | null
​
topic_ids
(string | null)[] | null
​
topics
(string | null)[] | null
​
source
SourceConnector · object | null

Source connector configuration and status.

​
destination
DestinationConnector · object | null

Destination connector configuration and status.

​
transforms
any[] | null
​
tags
string[] | null
​
periodic_audit
UpdatePipelineAuditReq · object | null

Update pipeline audit request.

​
topic_auto_discovery_transforms
TopicAutoDiscoveryTransform · object[]