Pipelines

resource cloudflare_pipeline

required Expand Collapse

account_id: String

Specifies the public ID of the account.

Specifies the name of the Pipeline.

sql: String

Specifies SQL for the Pipeline processing flow.

computed Expand Collapse

id: String

Indicates a unique identifier for this pipeline.

created_at: String

failure_reason: String

Indicates the reason for the failure of the Pipeline.

modified_at: String

status: String

Indicates the current status of the Pipeline.

tables: List[Attributes]

List of streams and sinks used by this pipeline.

id: String

Unique identifier for the connection (stream or sink).

latest: Int64

Latest available version of the connection.

Name of the connection.

type: String

Type of the connection.

version: Int64

Current version of the connection used by this pipeline.

cloudflare_pipeline

resource "cloudflare_pipeline" "example_pipeline" {
  account_id = "0123105f4ecef8ad9ca31a8372d0c353"
  name = "my_pipeline"
  sql = "insert into sink select * from source;"
}

data cloudflare_pipeline

required Expand Collapse

pipeline_id: String

Specifies the public ID of the pipeline.

account_id: String

Specifies the public ID of the account.

computed Expand Collapse

id: String

Specifies the public ID of the pipeline.

created_at: String

failure_reason: String

Indicates the reason for the failure of the Pipeline.

modified_at: String

Indicates the name of the Pipeline.

sql: String

Specifies SQL for the Pipeline processing flow.

status: String

Indicates the current status of the Pipeline.

tables: List[Attributes]

List of streams and sinks used by this pipeline.

id: String

Unique identifier for the connection (stream or sink).

latest: Int64

Latest available version of the connection.

Name of the connection.

type: String

Type of the connection.

version: Int64

Current version of the connection used by this pipeline.

cloudflare_pipeline

data "cloudflare_pipeline" "example_pipeline" {
  account_id = "0123105f4ecef8ad9ca31a8372d0c353"
  pipeline_id = "043e105f4ecef8ad9ca31a8372d0c353"
}

PipelinesSinks

resource cloudflare_pipeline_sink

required Expand Collapse

account_id: String

Specifies the public ID of the account.

Defines the name of the Sink.

type: String

Specifies the type of sink.

optional Expand Collapse

config?: Attributes

Defines the configuration of the R2 Sink.

account_id: String

Cloudflare Account ID for the bucket

bucket: String

R2 Bucket to write to

credentials?: Attributes

access_key_id: String

Cloudflare Account ID for the bucket

secret_access_key: String

Cloudflare Account ID for the bucket

file_naming?: Attributes

Controls filename prefix/suffix and strategy.

prefix?: String

The prefix to use in file name. i.e prefix-.parquet

strategy?: String

Filename generation strategy.

suffix?: String

This will overwrite the default file suffix. i.e .parquet, use with caution

jurisdiction?: String

Jurisdiction this bucket is hosted in

partitioning?: Attributes

Data-layout partitioning for sinks.

time_pattern?: String

The pattern of the date string

path?: String

Subpath within the bucket to write to

rolling_policy?: Attributes

Rolling policy for file sinks (when & why to close a file and open a new one).

file_size_bytes?: Int64

Files will be rolled after reaching this number of bytes

inactivity_seconds?: Int64

Number of seconds of inactivity to wait before rolling over to a new file

interval_seconds?: Int64

Number of seconds to wait before rolling over to a new file

token?: String

Authentication token

table_name?: String

Table name

namespace?: String

Table namespace

format?: Attributes

type: String

decimal_encoding?: String

timestamp_format?: String

unstructured?: Bool

compression?: String

row_group_bytes?: Int64

schema?: Attributes

fields?: List[Attributes]

type: String

metadata_key?: String

name?: String

required?: Bool

sql_name?: String

unit?: String

format?: Attributes

type: String

decimal_encoding?: String

timestamp_format?: String

unstructured?: Bool

compression?: String

row_group_bytes?: Int64

inferred?: Bool

computed Expand Collapse

id: String

Indicates a unique identifier for this sink.

created_at: Time

modified_at: Time

cloudflare_pipeline_sink

resource "cloudflare_pipeline_sink" "example_pipeline_sink" {
  account_id = "0123105f4ecef8ad9ca31a8372d0c353"
  name = "my_sink"
  type = "r2"
  config = {
    account_id = "account_id"
    bucket = "bucket"
    credentials = {
      access_key_id = "access_key_id"
      secret_access_key = "secret_access_key"
    }
    file_naming = {
      prefix = "prefix"
      strategy = "serial"
      suffix = "suffix"
    }
    jurisdiction = "jurisdiction"
    partitioning = {
      time_pattern = "year=%Y/month=%m/day=%d/hour=%H"
    }
    path = "path"
    rolling_policy = {
      file_size_bytes = 0
      inactivity_seconds = 1
      interval_seconds = 1
    }
  }
  format = {
    type = "json"
    decimal_encoding = "number"
    timestamp_format = "rfc3339"
    unstructured = true
  }
  schema = {
    fields = [{
      type = "int32"
      metadata_key = "metadata_key"
      name = "name"
      required = true
      sql_name = "sql_name"
    }]
    format = {
      type = "json"
      decimal_encoding = "number"
      timestamp_format = "rfc3339"
      unstructured = true
    }
    inferred = true
  }
}

data cloudflare_pipeline_sink

required Expand Collapse

account_id: String

Specifies the public ID of the account.

optional Expand Collapse

sink_id?: String

Specifies the publid ID of the sink.

filter?: Attributes

pipeline_id?: String

computed Expand Collapse

id: String

Specifies the publid ID of the sink.

created_at: Time

modified_at: Time

Defines the name of the Sink.

type: String

Specifies the type of sink.

config: Attributes

Defines the configuration of the R2 Sink.

account_id: String

Cloudflare Account ID for the bucket

bucket: String

R2 Bucket to write to

file_naming: Attributes

Controls filename prefix/suffix and strategy.

prefix: String

The prefix to use in file name. i.e prefix-.parquet

strategy: String

Filename generation strategy.

suffix: String

This will overwrite the default file suffix. i.e .parquet, use with caution

jurisdiction: String

Jurisdiction this bucket is hosted in

partitioning: Attributes

Data-layout partitioning for sinks.

time_pattern: String

The pattern of the date string

path: String

Subpath within the bucket to write to

rolling_policy: Attributes

Rolling policy for file sinks (when & why to close a file and open a new one).

file_size_bytes: Int64

Files will be rolled after reaching this number of bytes

inactivity_seconds: Int64

Number of seconds of inactivity to wait before rolling over to a new file

interval_seconds: Int64

Number of seconds to wait before rolling over to a new file

table_name: String

Table name

namespace: String

Table namespace

format: Attributes

type: String

decimal_encoding: String

timestamp_format: String

unstructured: Bool

compression: String

row_group_bytes: Int64

schema: Attributes

fields: List[Attributes]

type: String

metadata_key: String

required: Bool

sql_name: String

unit: String

format: Attributes

type: String

decimal_encoding: String

timestamp_format: String

unstructured: Bool

compression: String

row_group_bytes: Int64

inferred: Bool

cloudflare_pipeline_sink

data "cloudflare_pipeline_sink" "example_pipeline_sink" {
  account_id = "0123105f4ecef8ad9ca31a8372d0c353"
  sink_id = "0223105f4ecef8ad9ca31a8372d0c353"
}

data cloudflare_pipeline_sinks

required Expand Collapse

account_id: String

Specifies the public ID of the account.

optional Expand Collapse

pipeline_id?: String

max_items?: Int64

Max items to fetch, default: 1000

computed Expand Collapse

result: List[Attributes]

The items returned by the data source

id: String

Indicates a unique identifier for this sink.

created_at: Time

modified_at: Time

Defines the name of the Sink.

type: String

Specifies the type of sink.

config: Attributes

Defines the configuration of the R2 Sink.

account_id: String

Cloudflare Account ID for the bucket

bucket: String

R2 Bucket to write to

file_naming: Attributes

Controls filename prefix/suffix and strategy.

prefix: String

The prefix to use in file name. i.e prefix-.parquet

strategy: String

Filename generation strategy.

suffix: String

This will overwrite the default file suffix. i.e .parquet, use with caution

jurisdiction: String

Jurisdiction this bucket is hosted in

partitioning: Attributes

Data-layout partitioning for sinks.

time_pattern: String

The pattern of the date string

path: String

Subpath within the bucket to write to

rolling_policy: Attributes

Rolling policy for file sinks (when & why to close a file and open a new one).

file_size_bytes: Int64

Files will be rolled after reaching this number of bytes

inactivity_seconds: Int64

Number of seconds of inactivity to wait before rolling over to a new file

interval_seconds: Int64

Number of seconds to wait before rolling over to a new file

table_name: String

Table name

namespace: String

Table namespace

format: Attributes

type: String

decimal_encoding: String

timestamp_format: String

unstructured: Bool

compression: String

row_group_bytes: Int64

schema: Attributes

fields: List[Attributes]

type: String

metadata_key: String

required: Bool

sql_name: String

unit: String

format: Attributes

type: String

decimal_encoding: String

timestamp_format: String

unstructured: Bool

compression: String

row_group_bytes: Int64

inferred: Bool

cloudflare_pipeline_sinks

data "cloudflare_pipeline_sinks" "example_pipeline_sinks" {
  account_id = "0123105f4ecef8ad9ca31a8372d0c353"
  pipeline_id = "pipeline_id"
}

PipelinesStreams

resource cloudflare_pipeline_stream

required Expand Collapse

account_id: String

Specifies the public ID of the account.

Specifies the name of the Stream.

optional Expand Collapse

format?: Attributes

type: String

decimal_encoding?: String

timestamp_format?: String

unstructured?: Bool

compression?: String

row_group_bytes?: Int64

schema?: Attributes

fields?: List[Attributes]

type: String

metadata_key?: String

name?: String

required?: Bool

sql_name?: String

unit?: String

format?: Attributes

type: String

decimal_encoding?: String

timestamp_format?: String

unstructured?: Bool

compression?: String

row_group_bytes?: Int64

inferred?: Bool

http?: Attributes

authentication: Bool

Indicates that authentication is required for the HTTP endpoint.

enabled: Bool

Indicates that the HTTP endpoint is enabled.

cors?: Attributes

Specifies the CORS options for the HTTP endpoint.

origins?: List[String]

worker_binding?: Attributes

enabled: Bool

Indicates that the worker binding is enabled.

computed Expand Collapse

id: String

Indicates a unique identifier for this stream.

created_at: Time

endpoint: String

Indicates the endpoint URL of this stream.

modified_at: Time

version: Int64

Indicates the current version of this stream.

cloudflare_pipeline_stream

resource "cloudflare_pipeline_stream" "example_pipeline_stream" {
  account_id = "0123105f4ecef8ad9ca31a8372d0c353"
  name = "my_stream"
  format = {
    type = "json"
    decimal_encoding = "number"
    timestamp_format = "rfc3339"
    unstructured = true
  }
  http = {
    authentication = false
    enabled = true
    cors = {
      origins = ["string"]
    }
  }
  schema = {
    fields = [{
      type = "int32"
      metadata_key = "metadata_key"
      name = "name"
      required = true
      sql_name = "sql_name"
    }]
    format = {
      type = "json"
      decimal_encoding = "number"
      timestamp_format = "rfc3339"
      unstructured = true
    }
    inferred = true
  }
  worker_binding = {
    enabled = true
  }
}

data cloudflare_pipeline_stream

required Expand Collapse

account_id: String

Specifies the public ID of the account.

optional Expand Collapse

stream_id?: String

Specifies the public ID of the stream.

filter?: Attributes

pipeline_id?: String

Specifies the public ID of the pipeline.

computed Expand Collapse

id: String

Specifies the public ID of the stream.

created_at: Time

endpoint: String

Indicates the endpoint URL of this stream.

modified_at: Time

Indicates the name of the Stream.

version: Int64

Indicates the current version of this stream.

format: Attributes

type: String

decimal_encoding: String

timestamp_format: String

unstructured: Bool

compression: String

row_group_bytes: Int64

http: Attributes

authentication: Bool

Indicates that authentication is required for the HTTP endpoint.

enabled: Bool

Indicates that the HTTP endpoint is enabled.

cors: Attributes

Specifies the CORS options for the HTTP endpoint.

origins: List[String]

schema: Attributes

fields: List[Attributes]

type: String

metadata_key: String

required: Bool

sql_name: String

unit: String

format: Attributes

type: String

decimal_encoding: String

timestamp_format: String

unstructured: Bool

compression: String

row_group_bytes: Int64

inferred: Bool

worker_binding: Attributes

enabled: Bool

Indicates that the worker binding is enabled.

cloudflare_pipeline_stream

data "cloudflare_pipeline_stream" "example_pipeline_stream" {
  account_id = "0123105f4ecef8ad9ca31a8372d0c353"
  stream_id = "033e105f4ecef8ad9ca31a8372d0c353"
}

data cloudflare_pipeline_streams

required Expand Collapse

account_id: String

Specifies the public ID of the account.

optional Expand Collapse

pipeline_id?: String

Specifies the public ID of the pipeline.

max_items?: Int64

Max items to fetch, default: 1000

computed Expand Collapse

result: List[Attributes]

The items returned by the data source

id: String

Indicates a unique identifier for this stream.

created_at: Time

http: Attributes

authentication: Bool

Indicates that authentication is required for the HTTP endpoint.

enabled: Bool

Indicates that the HTTP endpoint is enabled.

cors: Attributes

Specifies the CORS options for the HTTP endpoint.

origins: List[String]

modified_at: Time

Indicates the name of the Stream.

version: Int64

Indicates the current version of this stream.

worker_binding: Attributes

enabled: Bool

Indicates that the worker binding is enabled.

endpoint: String

Indicates the endpoint URL of this stream.

format: Attributes

type: String

decimal_encoding: String

timestamp_format: String

unstructured: Bool

compression: String

row_group_bytes: Int64

schema: Attributes

fields: List[Attributes]

type: String

metadata_key: String

required: Bool

sql_name: String

unit: String

format: Attributes

type: String

decimal_encoding: String

timestamp_format: String

unstructured: Bool

compression: String

row_group_bytes: Int64

inferred: Bool

cloudflare_pipeline_streams

data "cloudflare_pipeline_streams" "example_pipeline_streams" {
  account_id = "0123105f4ecef8ad9ca31a8372d0c353"
  pipeline_id = "043e105f4ecef8ad9ca31a8372d0c353"
}