Class: Aws::Glue::Types::KafkaStreamingSourceOptions
- Inherits:
-
Struct
- Object
- Struct
- Aws::Glue::Types::KafkaStreamingSourceOptions
- Defined in:
- gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb
Overview
Additional options for streaming.
Constant Summary collapse
- SENSITIVE =
[]
Instance Attribute Summary collapse
-
#add_record_timestamp ⇒ String
When this option is set to 'true', the data output will contain an additional column named "__src_timestamp" that indicates the time when the corresponding record received by the topic.
-
#assign ⇒ String
The specific
TopicPartitions
to consume. -
#bootstrap_servers ⇒ String
A list of bootstrap server URLs, for example, as
b-1.vpc-test-2.o4q88o.c6.kafka.us-east-1.amazonaws.com:9094
. -
#classification ⇒ String
An optional classification.
-
#connection_name ⇒ String
The name of the connection.
-
#delimiter ⇒ String
Specifies the delimiter character.
-
#emit_consumer_lag_metrics ⇒ String
When this option is set to 'true', for each batch, it will emit the metrics for the duration between the oldest record received by the topic and the time it arrives in Glue to CloudWatch.
-
#ending_offsets ⇒ String
The end point when a batch query is ended.
-
#include_headers ⇒ Boolean
Whether to include the Kafka headers.
-
#max_offsets_per_trigger ⇒ Integer
The rate limit on the maximum number of offsets that are processed per trigger interval.
-
#min_partitions ⇒ Integer
The desired minimum number of partitions to read from Kafka.
-
#num_retries ⇒ Integer
The number of times to retry before failing to fetch Kafka offsets.
-
#poll_timeout_ms ⇒ Integer
The timeout in milliseconds to poll data from Kafka in Spark job executors.
-
#retry_interval_ms ⇒ Integer
The time in milliseconds to wait before retrying to fetch Kafka offsets.
-
#security_protocol ⇒ String
The protocol used to communicate with brokers.
-
#starting_offsets ⇒ String
The starting position in the Kafka topic to read data from.
-
#starting_timestamp ⇒ Time
The timestamp of the record in the Kafka topic to start reading data from.
-
#subscribe_pattern ⇒ String
A Java regex string that identifies the topic list to subscribe to.
-
#topic_name ⇒ String
The topic name as specified in Apache Kafka.
Instance Attribute Details
#add_record_timestamp ⇒ String
When this option is set to 'true', the data output will contain an additional column named "__src_timestamp" that indicates the time when the corresponding record received by the topic. The default value is 'false'. This option is supported in Glue version 4.0 or later.
14187 14188 14189 14190 14191 14192 14193 14194 14195 14196 14197 14198 14199 14200 14201 14202 14203 14204 14205 14206 14207 14208 14209 |
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 14187 class KafkaStreamingSourceOptions < Struct.new( :bootstrap_servers, :security_protocol, :connection_name, :topic_name, :assign, :subscribe_pattern, :classification, :delimiter, :starting_offsets, :ending_offsets, :poll_timeout_ms, :num_retries, :retry_interval_ms, :max_offsets_per_trigger, :min_partitions, :include_headers, :add_record_timestamp, :emit_consumer_lag_metrics, :starting_timestamp) SENSITIVE = [] include Aws::Structure end |
#assign ⇒ String
The specific TopicPartitions
to consume. You must specify at least
one of "topicName"
, "assign"
or "subscribePattern"
.
14187 14188 14189 14190 14191 14192 14193 14194 14195 14196 14197 14198 14199 14200 14201 14202 14203 14204 14205 14206 14207 14208 14209 |
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 14187 class KafkaStreamingSourceOptions < Struct.new( :bootstrap_servers, :security_protocol, :connection_name, :topic_name, :assign, :subscribe_pattern, :classification, :delimiter, :starting_offsets, :ending_offsets, :poll_timeout_ms, :num_retries, :retry_interval_ms, :max_offsets_per_trigger, :min_partitions, :include_headers, :add_record_timestamp, :emit_consumer_lag_metrics, :starting_timestamp) SENSITIVE = [] include Aws::Structure end |
#bootstrap_servers ⇒ String
A list of bootstrap server URLs, for example, as
b-1.vpc-test-2.o4q88o.c6.kafka.us-east-1.amazonaws.com:9094
. This
option must be specified in the API call or defined in the table
metadata in the Data Catalog.
14187 14188 14189 14190 14191 14192 14193 14194 14195 14196 14197 14198 14199 14200 14201 14202 14203 14204 14205 14206 14207 14208 14209 |
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 14187 class KafkaStreamingSourceOptions < Struct.new( :bootstrap_servers, :security_protocol, :connection_name, :topic_name, :assign, :subscribe_pattern, :classification, :delimiter, :starting_offsets, :ending_offsets, :poll_timeout_ms, :num_retries, :retry_interval_ms, :max_offsets_per_trigger, :min_partitions, :include_headers, :add_record_timestamp, :emit_consumer_lag_metrics, :starting_timestamp) SENSITIVE = [] include Aws::Structure end |
#classification ⇒ String
An optional classification.
14187 14188 14189 14190 14191 14192 14193 14194 14195 14196 14197 14198 14199 14200 14201 14202 14203 14204 14205 14206 14207 14208 14209 |
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 14187 class KafkaStreamingSourceOptions < Struct.new( :bootstrap_servers, :security_protocol, :connection_name, :topic_name, :assign, :subscribe_pattern, :classification, :delimiter, :starting_offsets, :ending_offsets, :poll_timeout_ms, :num_retries, :retry_interval_ms, :max_offsets_per_trigger, :min_partitions, :include_headers, :add_record_timestamp, :emit_consumer_lag_metrics, :starting_timestamp) SENSITIVE = [] include Aws::Structure end |
#connection_name ⇒ String
The name of the connection.
14187 14188 14189 14190 14191 14192 14193 14194 14195 14196 14197 14198 14199 14200 14201 14202 14203 14204 14205 14206 14207 14208 14209 |
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 14187 class KafkaStreamingSourceOptions < Struct.new( :bootstrap_servers, :security_protocol, :connection_name, :topic_name, :assign, :subscribe_pattern, :classification, :delimiter, :starting_offsets, :ending_offsets, :poll_timeout_ms, :num_retries, :retry_interval_ms, :max_offsets_per_trigger, :min_partitions, :include_headers, :add_record_timestamp, :emit_consumer_lag_metrics, :starting_timestamp) SENSITIVE = [] include Aws::Structure end |
#delimiter ⇒ String
Specifies the delimiter character.
14187 14188 14189 14190 14191 14192 14193 14194 14195 14196 14197 14198 14199 14200 14201 14202 14203 14204 14205 14206 14207 14208 14209 |
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 14187 class KafkaStreamingSourceOptions < Struct.new( :bootstrap_servers, :security_protocol, :connection_name, :topic_name, :assign, :subscribe_pattern, :classification, :delimiter, :starting_offsets, :ending_offsets, :poll_timeout_ms, :num_retries, :retry_interval_ms, :max_offsets_per_trigger, :min_partitions, :include_headers, :add_record_timestamp, :emit_consumer_lag_metrics, :starting_timestamp) SENSITIVE = [] include Aws::Structure end |
#emit_consumer_lag_metrics ⇒ String
When this option is set to 'true', for each batch, it will emit the metrics for the duration between the oldest record received by the topic and the time it arrives in Glue to CloudWatch. The metric's name is "glue.driver.streaming.maxConsumerLagInMs". The default value is 'false'. This option is supported in Glue version 4.0 or later.
14187 14188 14189 14190 14191 14192 14193 14194 14195 14196 14197 14198 14199 14200 14201 14202 14203 14204 14205 14206 14207 14208 14209 |
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 14187 class KafkaStreamingSourceOptions < Struct.new( :bootstrap_servers, :security_protocol, :connection_name, :topic_name, :assign, :subscribe_pattern, :classification, :delimiter, :starting_offsets, :ending_offsets, :poll_timeout_ms, :num_retries, :retry_interval_ms, :max_offsets_per_trigger, :min_partitions, :include_headers, :add_record_timestamp, :emit_consumer_lag_metrics, :starting_timestamp) SENSITIVE = [] include Aws::Structure end |
#ending_offsets ⇒ String
The end point when a batch query is ended. Possible values are
either "latest"
or a JSON string that specifies an ending offset
for each TopicPartition
.
14187 14188 14189 14190 14191 14192 14193 14194 14195 14196 14197 14198 14199 14200 14201 14202 14203 14204 14205 14206 14207 14208 14209 |
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 14187 class KafkaStreamingSourceOptions < Struct.new( :bootstrap_servers, :security_protocol, :connection_name, :topic_name, :assign, :subscribe_pattern, :classification, :delimiter, :starting_offsets, :ending_offsets, :poll_timeout_ms, :num_retries, :retry_interval_ms, :max_offsets_per_trigger, :min_partitions, :include_headers, :add_record_timestamp, :emit_consumer_lag_metrics, :starting_timestamp) SENSITIVE = [] include Aws::Structure end |
#include_headers ⇒ Boolean
Whether to include the Kafka headers. When the option is set to
"true", the data output will contain an additional column named
"glue_streaming_kafka_headers" with type Array[Struct(key:
String, value: String)]
. The default value is "false". This
option is available in Glue version 3.0 or later only.
14187 14188 14189 14190 14191 14192 14193 14194 14195 14196 14197 14198 14199 14200 14201 14202 14203 14204 14205 14206 14207 14208 14209 |
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 14187 class KafkaStreamingSourceOptions < Struct.new( :bootstrap_servers, :security_protocol, :connection_name, :topic_name, :assign, :subscribe_pattern, :classification, :delimiter, :starting_offsets, :ending_offsets, :poll_timeout_ms, :num_retries, :retry_interval_ms, :max_offsets_per_trigger, :min_partitions, :include_headers, :add_record_timestamp, :emit_consumer_lag_metrics, :starting_timestamp) SENSITIVE = [] include Aws::Structure end |
#max_offsets_per_trigger ⇒ Integer
The rate limit on the maximum number of offsets that are processed
per trigger interval. The specified total number of offsets is
proportionally split across topicPartitions
of different volumes.
The default value is null, which means that the consumer reads all
offsets until the known latest offset.
14187 14188 14189 14190 14191 14192 14193 14194 14195 14196 14197 14198 14199 14200 14201 14202 14203 14204 14205 14206 14207 14208 14209 |
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 14187 class KafkaStreamingSourceOptions < Struct.new( :bootstrap_servers, :security_protocol, :connection_name, :topic_name, :assign, :subscribe_pattern, :classification, :delimiter, :starting_offsets, :ending_offsets, :poll_timeout_ms, :num_retries, :retry_interval_ms, :max_offsets_per_trigger, :min_partitions, :include_headers, :add_record_timestamp, :emit_consumer_lag_metrics, :starting_timestamp) SENSITIVE = [] include Aws::Structure end |
#min_partitions ⇒ Integer
The desired minimum number of partitions to read from Kafka. The default value is null, which means that the number of spark partitions is equal to the number of Kafka partitions.
14187 14188 14189 14190 14191 14192 14193 14194 14195 14196 14197 14198 14199 14200 14201 14202 14203 14204 14205 14206 14207 14208 14209 |
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 14187 class KafkaStreamingSourceOptions < Struct.new( :bootstrap_servers, :security_protocol, :connection_name, :topic_name, :assign, :subscribe_pattern, :classification, :delimiter, :starting_offsets, :ending_offsets, :poll_timeout_ms, :num_retries, :retry_interval_ms, :max_offsets_per_trigger, :min_partitions, :include_headers, :add_record_timestamp, :emit_consumer_lag_metrics, :starting_timestamp) SENSITIVE = [] include Aws::Structure end |
#num_retries ⇒ Integer
The number of times to retry before failing to fetch Kafka offsets.
The default value is 3
.
14187 14188 14189 14190 14191 14192 14193 14194 14195 14196 14197 14198 14199 14200 14201 14202 14203 14204 14205 14206 14207 14208 14209 |
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 14187 class KafkaStreamingSourceOptions < Struct.new( :bootstrap_servers, :security_protocol, :connection_name, :topic_name, :assign, :subscribe_pattern, :classification, :delimiter, :starting_offsets, :ending_offsets, :poll_timeout_ms, :num_retries, :retry_interval_ms, :max_offsets_per_trigger, :min_partitions, :include_headers, :add_record_timestamp, :emit_consumer_lag_metrics, :starting_timestamp) SENSITIVE = [] include Aws::Structure end |
#poll_timeout_ms ⇒ Integer
The timeout in milliseconds to poll data from Kafka in Spark job
executors. The default value is 512
.
14187 14188 14189 14190 14191 14192 14193 14194 14195 14196 14197 14198 14199 14200 14201 14202 14203 14204 14205 14206 14207 14208 14209 |
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 14187 class KafkaStreamingSourceOptions < Struct.new( :bootstrap_servers, :security_protocol, :connection_name, :topic_name, :assign, :subscribe_pattern, :classification, :delimiter, :starting_offsets, :ending_offsets, :poll_timeout_ms, :num_retries, :retry_interval_ms, :max_offsets_per_trigger, :min_partitions, :include_headers, :add_record_timestamp, :emit_consumer_lag_metrics, :starting_timestamp) SENSITIVE = [] include Aws::Structure end |
#retry_interval_ms ⇒ Integer
The time in milliseconds to wait before retrying to fetch Kafka
offsets. The default value is 10
.
14187 14188 14189 14190 14191 14192 14193 14194 14195 14196 14197 14198 14199 14200 14201 14202 14203 14204 14205 14206 14207 14208 14209 |
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 14187 class KafkaStreamingSourceOptions < Struct.new( :bootstrap_servers, :security_protocol, :connection_name, :topic_name, :assign, :subscribe_pattern, :classification, :delimiter, :starting_offsets, :ending_offsets, :poll_timeout_ms, :num_retries, :retry_interval_ms, :max_offsets_per_trigger, :min_partitions, :include_headers, :add_record_timestamp, :emit_consumer_lag_metrics, :starting_timestamp) SENSITIVE = [] include Aws::Structure end |
#security_protocol ⇒ String
The protocol used to communicate with brokers. The possible values
are "SSL"
or "PLAINTEXT"
.
14187 14188 14189 14190 14191 14192 14193 14194 14195 14196 14197 14198 14199 14200 14201 14202 14203 14204 14205 14206 14207 14208 14209 |
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 14187 class KafkaStreamingSourceOptions < Struct.new( :bootstrap_servers, :security_protocol, :connection_name, :topic_name, :assign, :subscribe_pattern, :classification, :delimiter, :starting_offsets, :ending_offsets, :poll_timeout_ms, :num_retries, :retry_interval_ms, :max_offsets_per_trigger, :min_partitions, :include_headers, :add_record_timestamp, :emit_consumer_lag_metrics, :starting_timestamp) SENSITIVE = [] include Aws::Structure end |
#starting_offsets ⇒ String
The starting position in the Kafka topic to read data from. The
possible values are "earliest"
or "latest"
. The default value is
"latest"
.
14187 14188 14189 14190 14191 14192 14193 14194 14195 14196 14197 14198 14199 14200 14201 14202 14203 14204 14205 14206 14207 14208 14209 |
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 14187 class KafkaStreamingSourceOptions < Struct.new( :bootstrap_servers, :security_protocol, :connection_name, :topic_name, :assign, :subscribe_pattern, :classification, :delimiter, :starting_offsets, :ending_offsets, :poll_timeout_ms, :num_retries, :retry_interval_ms, :max_offsets_per_trigger, :min_partitions, :include_headers, :add_record_timestamp, :emit_consumer_lag_metrics, :starting_timestamp) SENSITIVE = [] include Aws::Structure end |
#starting_timestamp ⇒ Time
The timestamp of the record in the Kafka topic to start reading data
from. The possible values are a timestamp string in UTC format of
the pattern yyyy-mm-ddTHH:MM:SSZ
(where Z represents a UTC
timezone offset with a +/-. For example:
"2023-04-04T08:00:00+08:00").
Only one of StartingTimestamp
or StartingOffsets
must be set.
14187 14188 14189 14190 14191 14192 14193 14194 14195 14196 14197 14198 14199 14200 14201 14202 14203 14204 14205 14206 14207 14208 14209 |
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 14187 class KafkaStreamingSourceOptions < Struct.new( :bootstrap_servers, :security_protocol, :connection_name, :topic_name, :assign, :subscribe_pattern, :classification, :delimiter, :starting_offsets, :ending_offsets, :poll_timeout_ms, :num_retries, :retry_interval_ms, :max_offsets_per_trigger, :min_partitions, :include_headers, :add_record_timestamp, :emit_consumer_lag_metrics, :starting_timestamp) SENSITIVE = [] include Aws::Structure end |
#subscribe_pattern ⇒ String
A Java regex string that identifies the topic list to subscribe to.
You must specify at least one of "topicName"
, "assign"
or
"subscribePattern"
.
14187 14188 14189 14190 14191 14192 14193 14194 14195 14196 14197 14198 14199 14200 14201 14202 14203 14204 14205 14206 14207 14208 14209 |
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 14187 class KafkaStreamingSourceOptions < Struct.new( :bootstrap_servers, :security_protocol, :connection_name, :topic_name, :assign, :subscribe_pattern, :classification, :delimiter, :starting_offsets, :ending_offsets, :poll_timeout_ms, :num_retries, :retry_interval_ms, :max_offsets_per_trigger, :min_partitions, :include_headers, :add_record_timestamp, :emit_consumer_lag_metrics, :starting_timestamp) SENSITIVE = [] include Aws::Structure end |
#topic_name ⇒ String
The topic name as specified in Apache Kafka. You must specify at
least one of "topicName"
, "assign"
or "subscribePattern"
.
14187 14188 14189 14190 14191 14192 14193 14194 14195 14196 14197 14198 14199 14200 14201 14202 14203 14204 14205 14206 14207 14208 14209 |
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 14187 class KafkaStreamingSourceOptions < Struct.new( :bootstrap_servers, :security_protocol, :connection_name, :topic_name, :assign, :subscribe_pattern, :classification, :delimiter, :starting_offsets, :ending_offsets, :poll_timeout_ms, :num_retries, :retry_interval_ms, :max_offsets_per_trigger, :min_partitions, :include_headers, :add_record_timestamp, :emit_consumer_lag_metrics, :starting_timestamp) SENSITIVE = [] include Aws::Structure end |