Create New Transform

POST/transforms

Authorization

AuthorizationBearer token · headerrequired

Query parameters

secret_returnedboolean

Whether to include secret values in the response

default: false

Request body

requiredapplication/json

transformTransformsLibraryEnumrequired

Allowed:fan_outsql_joinenrichenrich_asyncmap_filtertoast_handlingun_nestingrolluptopic_router

configTransformConfigrequired

Show properties

transforms.namestringrequired

transforms.languagestring | any

Show properties

Any of:

string

string

any

any

transforms.input.topic.patternstringrequired

transforms.output.topic.patternstringrequired

transforms.input.serialization.formatTransformSerializationFormatEnumrequired

Allowed:AnyJsonAvro

transforms.output.serialization.formatTransformSerializationFormatEnumrequired

Allowed:AnyJsonAvro

transforms.input.job.parallelisminteger

default: 5

transforms.topic.ttlstring | any

Show properties

Any of:

string

string

any

any

kc.cluster.idstring | any

Show properties

Any of:

string

string

any

any

created_fromEntityOriginEnum | any

default: "api"

Show properties

Any of:

EntityOriginEnum

string

any

any

tagsstring[] | any

List of tag IDs to assign to this transform

Show properties

Any of:

string[]

Array of string

string

any

any

Responses

200

Successful Response

idstringrequired

namestringrequired

sub_idstringrequired

tenant_idstringrequired

service_idstring | any

Show properties

Any of:

string

string

any

any

start_timestring | string<date-time> | any

Show properties

Any of:

string

string

string<date-time>

string<date-time>

any

any

job_namestring | any

Show properties

Any of:

string

string

any

any

topic_idsstring[] | string | any

Show properties

Any of:

string[]

Array of string

string

string

string

any

any

transform_typeTransformsLibraryEnumrequired

Allowed:fan_outsql_joinenrichenrich_asyncmap_filtertoast_handlingun_nestingrolluptopic_router

configTransformConfig | TransformsEnrichAsyncConfig | any

Show properties

Any of:

TransformConfig

transforms.namestringrequired

transforms.languagestring | any

Show properties

Any of:

string

string

any

any

transforms.input.topic.patternstringrequired

transforms.output.topic.patternstringrequired

transforms.input.serialization.formatTransformSerializationFormatEnumrequired

Allowed:AnyJsonAvro

transforms.output.serialization.formatTransformSerializationFormatEnumrequired

Allowed:AnyJsonAvro

transforms.input.job.parallelisminteger

default: 5

transforms.topic.ttlstring | any

Show properties

Any of:

string

string

any

any

kc.cluster.idstring | any

Show properties

Any of:

string

string

any

any

TransformsEnrichAsyncConfig

transforms.namestringrequired

transforms.languagestring | any

Show properties

Any of:

string

string

any

any

transforms.input.topic.patternstringrequired

transforms.output.topic.patternstringrequired

transforms.input.serialization.formatTransformSerializationFormatEnumrequired

Allowed:AnyJsonAvro

transforms.output.serialization.formatTransformSerializationFormatEnumrequired

Allowed:AnyJsonAvro

transforms.input.job.parallelisminteger

default: 5

transforms.topic.ttlstring | any

Show properties

Any of:

string

string

any

any

kc.cluster.idstring | any

Show properties

Any of:

string

string

any

any

transforms.async.timeout.msinteger

default: 1000

transforms.async.capacityinteger

default: 10

any

any

Show properties

Any of:

TransformsMapFilterImplementation

languageTransformLanguageEnumrequired

Allowed:PYTHONJAVASCRIPTSQL

value_transformstringrequired

key_transformstring | any

Show properties

Any of:

string

string

any

any

topic_transformstring | any

Show properties

Any of:

string

string

any

any

common_transformstring | any

Show properties

Any of:

string

string

any

any

value_schema_transformstring | any

Show properties

Any of:

string

string

any

any

key_schema_transformstring | any

Show properties

Any of:

string

string

any

any

unit_testTransformsUnitTest | any

Show properties

Any of:

TransformsUnitTest

test_namestringrequired

input__jsonTransformsImplementationDetailsTestRecord[]required

Show properties

Array of TransformsImplementationDetailsTestRecord

topicstringrequired

valueobjectrequired

keyobject | any

Show properties

Any of:

object

object

any

any

timestampstring<date-time> | any

Show properties

Any of:

string<date-time>

string<date-time>

any

any

output__jsonTransformsImplementationDetailsTestRecord[]required

Show properties

Array of TransformsImplementationDetailsTestRecord

topicstringrequired

valueobjectrequired

keyobject | any

Show properties

Any of:

object

object

any

any

timestampstring<date-time> | any

Show properties

Any of:

string<date-time>

string<date-time>

any

any

result__jsonTransformsImplementationDetailsTestRecord[] | any

Show properties

Any of:

TransformsImplementationDetailsTestRecord[]

Array of TransformsImplementationDetailsTestRecord

topicstringrequired

valueobjectrequired

keyobject | any

Show properties

Any of:

object

object

any

any

timestampstring<date-time> | any

Show properties

Any of:

string<date-time>

string<date-time>

any

any

any

any

is_successboolean | any

Show properties

Any of:

boolean

boolean

any

any

html_diffstring | any

Show properties

Any of:

string

string

any

any

text_diffstring | any

Show properties

Any of:

string

string

any

any

any

any

TransformsEnrichImplementation

mainTableTransformInputTablerequired

Show properties

namestringrequired

topicMatcherRegexstringrequired

createTableSQLstring | any

Show properties

Any of:

string

string

any

any

lookupTableTransformInputTablerequired

Show properties

namestringrequired

topicMatcherRegexstringrequired

createTableSQLstring | any

Show properties

Any of:

string

string

any

any

enrichSQLstringrequired

TransformsEnrichAsyncImplementation

languageTransformLanguageEnumrequired

Allowed:PYTHONJAVASCRIPTSQL

value_transformstringrequired

key_transformstring | any

Show properties

Any of:

string

string

any

any

topic_transformstring | any

Show properties

Any of:

string

string

any

any

common_transformstring | any

Show properties

Any of:

string

string

any

any

value_schema_transformstring | any

Show properties

Any of:

string

string

any

any

key_schema_transformstring | any

Show properties

Any of:

string

string

any

any

unit_testTransformsUnitTest | any

Show properties

Any of:

TransformsUnitTest

test_namestringrequired

input__jsonTransformsImplementationDetailsTestRecord[]required

Show properties

Array of TransformsImplementationDetailsTestRecord

topicstringrequired

valueobjectrequired

keyobject | any

Show properties

Any of:

object

object

any

any

timestampstring<date-time> | any

Show properties

Any of:

string<date-time>

string<date-time>

any

any

output__jsonTransformsImplementationDetailsTestRecord[]required

Show properties

Array of TransformsImplementationDetailsTestRecord

topicstringrequired

valueobjectrequired

keyobject | any

Show properties

Any of:

object

object

any

any

timestampstring<date-time> | any

Show properties

Any of:

string<date-time>

string<date-time>

any

any

result__jsonTransformsImplementationDetailsTestRecord[] | any

Show properties

Any of:

TransformsImplementationDetailsTestRecord[]

Array of TransformsImplementationDetailsTestRecord

topicstringrequired

valueobjectrequired

keyobject | any

Show properties

Any of:

object

object

any

any

timestampstring<date-time> | any

Show properties

Any of:

string<date-time>

string<date-time>

any

any

any

any

is_successboolean | any

Show properties

Any of:

boolean

boolean

any

any

html_diffstring | any

Show properties

Any of:

string

string

any

any

text_diffstring | any

Show properties

Any of:

string

string

any

any

any

any

async_timeout_msinteger

default: 2000

async_capacityinteger

default: 10

TransformsJoinImplementation

inputTablesTransformInputTable[]

Show properties

Array of TransformInputTable

namestringrequired

topicMatcherRegexstringrequired

createTableSQLstring | any

Show properties

Any of:

string

string

any

any

joinSQLstringrequired

keyFieldsstring[]

stateTtlMsstring | any

Show properties

Any of:

string

string

any

any

TransformsRollupImplementation

inputTablesTransformInputTable[]

Show properties

Array of TransformInputTable

namestringrequired

topicMatcherRegexstringrequired

createTableSQLstring | any

Show properties

Any of:

string

string

any

any

rollupSQLstringrequired

keyFieldsstring[]

sourceIdleTimeoutMsinteger

default: 30000

stateTTLMsinteger

default: 86400000

TransformsTopicRouterImplementation

topic_prefixstring

Deprecated - kept for backwards compat with existing DB records

default: ""

kc_cluster_idstring | any

KC cluster to deploy the connector to

Show properties

Any of:

string

string

any

any

any

any

metricsTransformMetric | any

Show properties

Any of:

TransformMetric

tasksinteger | any

Show properties

Any of:

integer

integer

any

any

latencyinteger | any

Show properties

Any of:

integer

integer

any

any

statusstring | any

Show properties

Any of:

string

string

any

any

durationinteger | any

Show properties

Any of:

integer

integer

any

any

start_timeinteger | string | string<date-time> | any

Show properties

Any of:

integer

integer

string

string

string<date-time>

string<date-time>

any

any

end_timestring | string<date-time> | any

Show properties

Any of:

string

string

string<date-time>

string<date-time>

any

any

checkpointsTransformCheckpointsInfo | any

Show properties

Any of:

TransformCheckpointsInfo

job_idstringrequired

countsobjectrequired

summaryobjectrequired

latestTransformCheckpointHistory | any

Show properties

Any of:

TransformCheckpointHistory

idintegerrequired

statusstringrequired

is_savepointbooleanrequired

trigger_timestampintegerrequired

latest_ack_timestampintegerrequired

state_sizeintegerrequired

end_to_end_durationintegerrequired

alignment_bufferedintegerrequired

num_subtasksintegerrequired

num_acknowledged_subtasksintegerrequired

any

any

historyTransformCheckpointHistory[]

default: []

Show properties

Array of TransformCheckpointHistory

idintegerrequired

statusstringrequired

is_savepointbooleanrequired

trigger_timestampintegerrequired

latest_ack_timestampintegerrequired

state_sizeintegerrequired

end_to_end_durationintegerrequired

alignment_bufferedintegerrequired

num_subtasksintegerrequired

num_acknowledged_subtasksintegerrequired

operator_breakdownTransformOperatorState[]

default: []

Show properties

Array of TransformOperatorState

operator_idstringrequired

operator_namestringrequired

state_sizeintegerrequired

checkpointed_sizeintegerrequired

end_to_end_durationintegerrequired

parallelismintegerrequired

any

any

any

any

live_versionstring | any

Show properties

Any of:

string

string

any

any

preview_versionstring | any

Show properties

Any of:

string

string

any

any

versioninteger

default: 1

updated_timestampstring<date-time> | any

Show properties

Any of:

string<date-time>

string<date-time>

any

any

implementation_updated_timestampstring<date-time> | any

Show properties

Any of:

string<date-time>

string<date-time>

any

any

descriptionstring | any

Show properties

Any of:

string

string

any

any

tagsstring[] | any

List of tag IDs assigned to this transform

Show properties

Any of:

string[]

Array of string

string

any

any

history_created_timestampstring<date-time> | any

Show properties

Any of:

string<date-time>

string<date-time>

any

any

history_updated_timestampstring<date-time> | any

Show properties

Any of:

string<date-time>

string<date-time>

any

any

422

Validation Error

detailValidationError[]

Show properties

Array of ValidationError

locstring | integer[]required

Show properties

Array of string | integer

Any of:

string

string

integer

integer

msgstringrequired

typestringrequired

inputany

ctxobject

Request

curl -X POST "https://api.streamkap.com/transforms" \
  -H "Authorization: Bearer YOUR_TOKEN" \
  -H "Content-Type: application/json" \
  -d '{
  "transform": "fan_out",
  "config": {
    "transforms.name": "string",
    "transforms.language": "string",
    "transforms.input.topic.pattern": "string",
    "transforms.output.topic.pattern": "string",
    "transforms.input.serialization.format": "Any",
    "transforms.output.serialization.format": "Any",
    "transforms.input.job.parallelism": 5,
    "transforms.topic.ttl": "string",
    "kc.cluster.id": "string"
  },
  "created_from": "api",
  "tags": [
    "string"
  ]
}'

const response = await fetch("https://api.streamkap.com/transforms", {
  method: "POST",
  headers: {
    "Authorization": "Bearer YOUR_TOKEN",
    "Content-Type": "application/json"
  },
  body: "{\n  \"transform\": \"fan_out\",\n  \"config\": {\n    \"transforms.name\": \"string\",\n    \"transforms.language\": \"string\",\n    \"transforms.input.topic.pattern\": \"string\",\n    \"transforms.output.topic.pattern\": \"string\",\n    \"transforms.input.serialization.format\": \"Any\",\n    \"transforms.output.serialization.format\": \"Any\",\n    \"transforms.input.job.parallelism\": 5,\n    \"transforms.topic.ttl\": \"string\",\n    \"kc.cluster.id\": \"string\"\n  },\n  \"created_from\": \"api\",\n  \"tags\": [\n    \"string\"\n  ]\n}"
});

import requests

response = requests.post(
    "https://api.streamkap.com/transforms",
    headers={
        "Authorization": "Bearer YOUR_TOKEN",
        "Content-Type": "application/json"
    },
    data="{\n  \"transform\": \"fan_out\",\n  \"config\": {\n    \"transforms.name\": \"string\",\n    \"transforms.language\": \"string\",\n    \"transforms.input.topic.pattern\": \"string\",\n    \"transforms.output.topic.pattern\": \"string\",\n    \"transforms.input.serialization.format\": \"Any\",\n    \"transforms.output.serialization.format\": \"Any\",\n    \"transforms.input.job.parallelism\": 5,\n    \"transforms.topic.ttl\": \"string\",\n    \"kc.cluster.id\": \"string\"\n  },\n  \"created_from\": \"api\",\n  \"tags\": [\n    \"string\"\n  ]\n}",
)

Response

{
  "id": "string",
  "name": "string",
  "sub_id": "string",
  "tenant_id": "string",
  "service_id": "string",
  "start_time": "string",
  "job_name": "string",
  "topic_ids": [
    "string"
  ],
  "transform_type": "fan_out",
  "config": {
    "transforms.name": "string",
    "transforms.language": "string",
    "transforms.input.topic.pattern": "string",
    "transforms.output.topic.pattern": "string",
    "transforms.input.serialization.format": "Any",
    "transforms.output.serialization.format": "Any",
    "transforms.input.job.parallelism": 5,
    "transforms.topic.ttl": "string",
    "kc.cluster.id": "string"
  },
  "implementation": {
    "language": "PYTHON",
    "value_transform": "string",
    "key_transform": "string",
    "topic_transform": "string",
    "common_transform": "string",
    "value_schema_transform": "string",
    "key_schema_transform": "string",
    "unit_test": {
      "test_name": "string",
      "input__json": [
        {
          "topic": "string",
          "value": {},
          "key": {},
          "timestamp": "2019-08-24T14:15:22Z"
        }
      ],
      "output__json": [
        {
          "topic": "string",
          "value": {},
          "key": {},
          "timestamp": "2019-08-24T14:15:22Z"
        }
      ],
      "result__json": [
        {
          "topic": "string",
          "value": {},
          "key": {},
          "timestamp": "2019-08-24T14:15:22Z"
        }
      ],
      "is_success": true,
      "html_diff": "string",
      "text_diff": "string"
    }
  },
  "metrics": {
    "tasks": 0,
    "latency": 0,
    "status": "string",
    "duration": 0,
    "start_time": 0,
    "end_time": "string",
    "checkpoints": {
      "job_id": "string",
      "counts": {},
      "summary": {},
      "latest": {
        "id": 0,
        "status": "string",
        "is_savepoint": true,
        "trigger_timestamp": 0,
        "latest_ack_timestamp": 0,
        "state_size": 0,
        "end_to_end_duration": 0,
        "alignment_buffered": 0,
        "num_subtasks": 0,
        "num_acknowledged_subtasks": 0
      },
      "history": [],
      "operator_breakdown": []
    }
  },
  "live_version": "string",
  "preview_version": "string",
  "version": 1,
  "updated_timestamp": "2019-08-24T14:15:22Z",
  "implementation_updated_timestamp": "2019-08-24T14:15:22Z",
  "description": "string",
  "tags": [
    "string"
  ],
  "history_created_timestamp": "2019-08-24T14:15:22Z",
  "history_updated_timestamp": "2019-08-24T14:15:22Z"
}

{
  "detail": [
    {
      "loc": [
        "string"
      ],
      "msg": "string",
      "type": "string",
      "input": null,
      "ctx": {}
    }
  ]
}