Class: Aws::Glue::Types::KafkaStreamingSourceOptions

Inherits:
Struct
  • Object
show all
Defined in:
gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb

Overview

Additional options for streaming.

Constant Summary collapse

SENSITIVE =
[]

Instance Attribute Summary collapse

Instance Attribute Details

#assignString

The specific TopicPartitions to consume. You must specify at least one of "topicName", "assign" or "subscribePattern".

Returns:

  • (String)


12306
12307
12308
12309
12310
12311
12312
12313
12314
12315
12316
12317
12318
12319
12320
12321
12322
12323
12324
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 12306

class KafkaStreamingSourceOptions < Struct.new(
  :bootstrap_servers,
  :security_protocol,
  :connection_name,
  :topic_name,
  :assign,
  :subscribe_pattern,
  :classification,
  :delimiter,
  :starting_offsets,
  :ending_offsets,
  :poll_timeout_ms,
  :num_retries,
  :retry_interval_ms,
  :max_offsets_per_trigger,
  :min_partitions)
  SENSITIVE = []
  include Aws::Structure
end

#bootstrap_serversString

A list of bootstrap server URLs, for example, as b-1.vpc-test-2.o4q88o.c6.kafka.us-east-1.amazonaws.com:9094. This option must be specified in the API call or defined in the table metadata in the Data Catalog.

Returns:

  • (String)


12306
12307
12308
12309
12310
12311
12312
12313
12314
12315
12316
12317
12318
12319
12320
12321
12322
12323
12324
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 12306

class KafkaStreamingSourceOptions < Struct.new(
  :bootstrap_servers,
  :security_protocol,
  :connection_name,
  :topic_name,
  :assign,
  :subscribe_pattern,
  :classification,
  :delimiter,
  :starting_offsets,
  :ending_offsets,
  :poll_timeout_ms,
  :num_retries,
  :retry_interval_ms,
  :max_offsets_per_trigger,
  :min_partitions)
  SENSITIVE = []
  include Aws::Structure
end

#classificationString

An optional classification.

Returns:

  • (String)


12306
12307
12308
12309
12310
12311
12312
12313
12314
12315
12316
12317
12318
12319
12320
12321
12322
12323
12324
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 12306

class KafkaStreamingSourceOptions < Struct.new(
  :bootstrap_servers,
  :security_protocol,
  :connection_name,
  :topic_name,
  :assign,
  :subscribe_pattern,
  :classification,
  :delimiter,
  :starting_offsets,
  :ending_offsets,
  :poll_timeout_ms,
  :num_retries,
  :retry_interval_ms,
  :max_offsets_per_trigger,
  :min_partitions)
  SENSITIVE = []
  include Aws::Structure
end

#connection_nameString

The name of the connection.

Returns:

  • (String)


12306
12307
12308
12309
12310
12311
12312
12313
12314
12315
12316
12317
12318
12319
12320
12321
12322
12323
12324
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 12306

class KafkaStreamingSourceOptions < Struct.new(
  :bootstrap_servers,
  :security_protocol,
  :connection_name,
  :topic_name,
  :assign,
  :subscribe_pattern,
  :classification,
  :delimiter,
  :starting_offsets,
  :ending_offsets,
  :poll_timeout_ms,
  :num_retries,
  :retry_interval_ms,
  :max_offsets_per_trigger,
  :min_partitions)
  SENSITIVE = []
  include Aws::Structure
end

#delimiterString

Specifies the delimiter character.

Returns:

  • (String)


12306
12307
12308
12309
12310
12311
12312
12313
12314
12315
12316
12317
12318
12319
12320
12321
12322
12323
12324
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 12306

class KafkaStreamingSourceOptions < Struct.new(
  :bootstrap_servers,
  :security_protocol,
  :connection_name,
  :topic_name,
  :assign,
  :subscribe_pattern,
  :classification,
  :delimiter,
  :starting_offsets,
  :ending_offsets,
  :poll_timeout_ms,
  :num_retries,
  :retry_interval_ms,
  :max_offsets_per_trigger,
  :min_partitions)
  SENSITIVE = []
  include Aws::Structure
end

#ending_offsetsString

The end point when a batch query is ended. Possible values are either "latest" or a JSON string that specifies an ending offset for each TopicPartition.

Returns:

  • (String)


12306
12307
12308
12309
12310
12311
12312
12313
12314
12315
12316
12317
12318
12319
12320
12321
12322
12323
12324
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 12306

class KafkaStreamingSourceOptions < Struct.new(
  :bootstrap_servers,
  :security_protocol,
  :connection_name,
  :topic_name,
  :assign,
  :subscribe_pattern,
  :classification,
  :delimiter,
  :starting_offsets,
  :ending_offsets,
  :poll_timeout_ms,
  :num_retries,
  :retry_interval_ms,
  :max_offsets_per_trigger,
  :min_partitions)
  SENSITIVE = []
  include Aws::Structure
end

#max_offsets_per_triggerInteger

The rate limit on the maximum number of offsets that are processed per trigger interval. The specified total number of offsets is proportionally split across topicPartitions of different volumes. The default value is null, which means that the consumer reads all offsets until the known latest offset.

Returns:

  • (Integer)


12306
12307
12308
12309
12310
12311
12312
12313
12314
12315
12316
12317
12318
12319
12320
12321
12322
12323
12324
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 12306

class KafkaStreamingSourceOptions < Struct.new(
  :bootstrap_servers,
  :security_protocol,
  :connection_name,
  :topic_name,
  :assign,
  :subscribe_pattern,
  :classification,
  :delimiter,
  :starting_offsets,
  :ending_offsets,
  :poll_timeout_ms,
  :num_retries,
  :retry_interval_ms,
  :max_offsets_per_trigger,
  :min_partitions)
  SENSITIVE = []
  include Aws::Structure
end

#min_partitionsInteger

The desired minimum number of partitions to read from Kafka. The default value is null, which means that the number of spark partitions is equal to the number of Kafka partitions.

Returns:

  • (Integer)


12306
12307
12308
12309
12310
12311
12312
12313
12314
12315
12316
12317
12318
12319
12320
12321
12322
12323
12324
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 12306

class KafkaStreamingSourceOptions < Struct.new(
  :bootstrap_servers,
  :security_protocol,
  :connection_name,
  :topic_name,
  :assign,
  :subscribe_pattern,
  :classification,
  :delimiter,
  :starting_offsets,
  :ending_offsets,
  :poll_timeout_ms,
  :num_retries,
  :retry_interval_ms,
  :max_offsets_per_trigger,
  :min_partitions)
  SENSITIVE = []
  include Aws::Structure
end

#num_retriesInteger

The number of times to retry before failing to fetch Kafka offsets. The default value is 3.

Returns:

  • (Integer)


12306
12307
12308
12309
12310
12311
12312
12313
12314
12315
12316
12317
12318
12319
12320
12321
12322
12323
12324
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 12306

class KafkaStreamingSourceOptions < Struct.new(
  :bootstrap_servers,
  :security_protocol,
  :connection_name,
  :topic_name,
  :assign,
  :subscribe_pattern,
  :classification,
  :delimiter,
  :starting_offsets,
  :ending_offsets,
  :poll_timeout_ms,
  :num_retries,
  :retry_interval_ms,
  :max_offsets_per_trigger,
  :min_partitions)
  SENSITIVE = []
  include Aws::Structure
end

#poll_timeout_msInteger

The timeout in milliseconds to poll data from Kafka in Spark job executors. The default value is 512.

Returns:

  • (Integer)


12306
12307
12308
12309
12310
12311
12312
12313
12314
12315
12316
12317
12318
12319
12320
12321
12322
12323
12324
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 12306

class KafkaStreamingSourceOptions < Struct.new(
  :bootstrap_servers,
  :security_protocol,
  :connection_name,
  :topic_name,
  :assign,
  :subscribe_pattern,
  :classification,
  :delimiter,
  :starting_offsets,
  :ending_offsets,
  :poll_timeout_ms,
  :num_retries,
  :retry_interval_ms,
  :max_offsets_per_trigger,
  :min_partitions)
  SENSITIVE = []
  include Aws::Structure
end

#retry_interval_msInteger

The time in milliseconds to wait before retrying to fetch Kafka offsets. The default value is 10.

Returns:

  • (Integer)


12306
12307
12308
12309
12310
12311
12312
12313
12314
12315
12316
12317
12318
12319
12320
12321
12322
12323
12324
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 12306

class KafkaStreamingSourceOptions < Struct.new(
  :bootstrap_servers,
  :security_protocol,
  :connection_name,
  :topic_name,
  :assign,
  :subscribe_pattern,
  :classification,
  :delimiter,
  :starting_offsets,
  :ending_offsets,
  :poll_timeout_ms,
  :num_retries,
  :retry_interval_ms,
  :max_offsets_per_trigger,
  :min_partitions)
  SENSITIVE = []
  include Aws::Structure
end

#security_protocolString

The protocol used to communicate with brokers. The possible values are "SSL" or "PLAINTEXT".

Returns:

  • (String)


12306
12307
12308
12309
12310
12311
12312
12313
12314
12315
12316
12317
12318
12319
12320
12321
12322
12323
12324
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 12306

class KafkaStreamingSourceOptions < Struct.new(
  :bootstrap_servers,
  :security_protocol,
  :connection_name,
  :topic_name,
  :assign,
  :subscribe_pattern,
  :classification,
  :delimiter,
  :starting_offsets,
  :ending_offsets,
  :poll_timeout_ms,
  :num_retries,
  :retry_interval_ms,
  :max_offsets_per_trigger,
  :min_partitions)
  SENSITIVE = []
  include Aws::Structure
end

#starting_offsetsString

The starting position in the Kafka topic to read data from. The possible values are "earliest" or "latest". The default value is "latest".

Returns:

  • (String)


12306
12307
12308
12309
12310
12311
12312
12313
12314
12315
12316
12317
12318
12319
12320
12321
12322
12323
12324
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 12306

class KafkaStreamingSourceOptions < Struct.new(
  :bootstrap_servers,
  :security_protocol,
  :connection_name,
  :topic_name,
  :assign,
  :subscribe_pattern,
  :classification,
  :delimiter,
  :starting_offsets,
  :ending_offsets,
  :poll_timeout_ms,
  :num_retries,
  :retry_interval_ms,
  :max_offsets_per_trigger,
  :min_partitions)
  SENSITIVE = []
  include Aws::Structure
end

#subscribe_patternString

A Java regex string that identifies the topic list to subscribe to. You must specify at least one of "topicName", "assign" or "subscribePattern".

Returns:

  • (String)


12306
12307
12308
12309
12310
12311
12312
12313
12314
12315
12316
12317
12318
12319
12320
12321
12322
12323
12324
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 12306

class KafkaStreamingSourceOptions < Struct.new(
  :bootstrap_servers,
  :security_protocol,
  :connection_name,
  :topic_name,
  :assign,
  :subscribe_pattern,
  :classification,
  :delimiter,
  :starting_offsets,
  :ending_offsets,
  :poll_timeout_ms,
  :num_retries,
  :retry_interval_ms,
  :max_offsets_per_trigger,
  :min_partitions)
  SENSITIVE = []
  include Aws::Structure
end

#topic_nameString

The topic name as specified in Apache Kafka. You must specify at least one of "topicName", "assign" or "subscribePattern".

Returns:

  • (String)


12306
12307
12308
12309
12310
12311
12312
12313
12314
12315
12316
12317
12318
12319
12320
12321
12322
12323
12324
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 12306

class KafkaStreamingSourceOptions < Struct.new(
  :bootstrap_servers,
  :security_protocol,
  :connection_name,
  :topic_name,
  :assign,
  :subscribe_pattern,
  :classification,
  :delimiter,
  :starting_offsets,
  :ending_offsets,
  :poll_timeout_ms,
  :num_retries,
  :retry_interval_ms,
  :max_offsets_per_trigger,
  :min_partitions)
  SENSITIVE = []
  include Aws::Structure
end