Class: Aws::Glue::Types::S3CsvSource
- Inherits:
-
Struct
- Object
- Struct
- Aws::Glue::Types::S3CsvSource
- Defined in:
- gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb
Overview
Specifies a command-separated value (CSV) data store stored in Amazon S3.
Constant Summary collapse
- SENSITIVE =
[]
Instance Attribute Summary collapse
-
#additional_options ⇒ Types::S3DirectSourceAdditionalOptions
Specifies additional connection options.
-
#compression_type ⇒ String
Specifies how the data is compressed.
-
#escaper ⇒ String
Specifies a character to use for escaping.
-
#exclusions ⇒ Array<String>
A string containing a JSON list of Unix-style glob patterns to exclude.
-
#group_files ⇒ String
Grouping files is turned on by default when the input contains more than 50,000 files.
-
#group_size ⇒ String
The target group size in bytes.
-
#max_band ⇒ Integer
This option controls the duration in milliseconds after which the s3 listing is likely to be consistent.
-
#max_files_in_band ⇒ Integer
This option specifies the maximum number of files to save from the last maxBand seconds.
-
#multiline ⇒ Boolean
A Boolean value that specifies whether a single record can span multiple lines.
-
#name ⇒ String
The name of the data store.
-
#optimize_performance ⇒ Boolean
A Boolean value that specifies whether to use the advanced SIMD CSV reader along with Apache Arrow based columnar memory formats.
-
#output_schemas ⇒ Array<Types::GlueSchema>
Specifies the data schema for the S3 CSV source.
-
#paths ⇒ Array<String>
A list of the Amazon S3 paths to read from.
-
#quote_char ⇒ String
Specifies the character to use for quoting.
-
#recurse ⇒ Boolean
If set to true, recursively reads files in all subdirectories under the specified paths.
-
#separator ⇒ String
Specifies the delimiter character.
-
#skip_first ⇒ Boolean
A Boolean value that specifies whether to skip the first data line.
-
#with_header ⇒ Boolean
A Boolean value that specifies whether to treat the first line as a header.
-
#write_header ⇒ Boolean
A Boolean value that specifies whether to write the header to output.
Instance Attribute Details
#additional_options ⇒ Types::S3DirectSourceAdditionalOptions
Specifies additional connection options.
18572 18573 18574 18575 18576 18577 18578 18579 18580 18581 18582 18583 18584 18585 18586 18587 18588 18589 18590 18591 18592 18593 18594 |
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 18572 class S3CsvSource < Struct.new( :name, :paths, :compression_type, :exclusions, :group_size, :group_files, :recurse, :max_band, :max_files_in_band, :additional_options, :separator, :escaper, :quote_char, :multiline, :with_header, :write_header, :skip_first, :optimize_performance, :output_schemas) SENSITIVE = [] include Aws::Structure end |
#compression_type ⇒ String
Specifies how the data is compressed. This is generally not
necessary if the data has a standard file extension. Possible values
are "gzip"
and "bzip"
).
18572 18573 18574 18575 18576 18577 18578 18579 18580 18581 18582 18583 18584 18585 18586 18587 18588 18589 18590 18591 18592 18593 18594 |
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 18572 class S3CsvSource < Struct.new( :name, :paths, :compression_type, :exclusions, :group_size, :group_files, :recurse, :max_band, :max_files_in_band, :additional_options, :separator, :escaper, :quote_char, :multiline, :with_header, :write_header, :skip_first, :optimize_performance, :output_schemas) SENSITIVE = [] include Aws::Structure end |
#escaper ⇒ String
Specifies a character to use for escaping. This option is used only
when reading CSV files. The default value is none
. If enabled, the
character which immediately follows is used as-is, except for a
small set of well-known escapes (\n
, \r
, \t
, and \0
).
18572 18573 18574 18575 18576 18577 18578 18579 18580 18581 18582 18583 18584 18585 18586 18587 18588 18589 18590 18591 18592 18593 18594 |
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 18572 class S3CsvSource < Struct.new( :name, :paths, :compression_type, :exclusions, :group_size, :group_files, :recurse, :max_band, :max_files_in_band, :additional_options, :separator, :escaper, :quote_char, :multiline, :with_header, :write_header, :skip_first, :optimize_performance, :output_schemas) SENSITIVE = [] include Aws::Structure end |
#exclusions ⇒ Array<String>
A string containing a JSON list of Unix-style glob patterns to exclude. For example, "[\"**.pdf\"]" excludes all PDF files.
18572 18573 18574 18575 18576 18577 18578 18579 18580 18581 18582 18583 18584 18585 18586 18587 18588 18589 18590 18591 18592 18593 18594 |
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 18572 class S3CsvSource < Struct.new( :name, :paths, :compression_type, :exclusions, :group_size, :group_files, :recurse, :max_band, :max_files_in_band, :additional_options, :separator, :escaper, :quote_char, :multiline, :with_header, :write_header, :skip_first, :optimize_performance, :output_schemas) SENSITIVE = [] include Aws::Structure end |
#group_files ⇒ String
Grouping files is turned on by default when the input contains more
than 50,000 files. To turn on grouping with fewer than 50,000 files,
set this parameter to "inPartition". To disable grouping when
there are more than 50,000 files, set this parameter to "none"
.
18572 18573 18574 18575 18576 18577 18578 18579 18580 18581 18582 18583 18584 18585 18586 18587 18588 18589 18590 18591 18592 18593 18594 |
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 18572 class S3CsvSource < Struct.new( :name, :paths, :compression_type, :exclusions, :group_size, :group_files, :recurse, :max_band, :max_files_in_band, :additional_options, :separator, :escaper, :quote_char, :multiline, :with_header, :write_header, :skip_first, :optimize_performance, :output_schemas) SENSITIVE = [] include Aws::Structure end |
#group_size ⇒ String
The target group size in bytes. The default is computed based on the
input data size and the size of your cluster. When there are fewer
than 50,000 input files, "groupFiles"
must be set to
"inPartition"
for this to take effect.
18572 18573 18574 18575 18576 18577 18578 18579 18580 18581 18582 18583 18584 18585 18586 18587 18588 18589 18590 18591 18592 18593 18594 |
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 18572 class S3CsvSource < Struct.new( :name, :paths, :compression_type, :exclusions, :group_size, :group_files, :recurse, :max_band, :max_files_in_band, :additional_options, :separator, :escaper, :quote_char, :multiline, :with_header, :write_header, :skip_first, :optimize_performance, :output_schemas) SENSITIVE = [] include Aws::Structure end |
#max_band ⇒ Integer
This option controls the duration in milliseconds after which the s3 listing is likely to be consistent. Files with modification timestamps falling within the last maxBand milliseconds are tracked specially when using JobBookmarks to account for Amazon S3 eventual consistency. Most users don't need to set this option. The default is 900000 milliseconds, or 15 minutes.
18572 18573 18574 18575 18576 18577 18578 18579 18580 18581 18582 18583 18584 18585 18586 18587 18588 18589 18590 18591 18592 18593 18594 |
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 18572 class S3CsvSource < Struct.new( :name, :paths, :compression_type, :exclusions, :group_size, :group_files, :recurse, :max_band, :max_files_in_band, :additional_options, :separator, :escaper, :quote_char, :multiline, :with_header, :write_header, :skip_first, :optimize_performance, :output_schemas) SENSITIVE = [] include Aws::Structure end |
#max_files_in_band ⇒ Integer
This option specifies the maximum number of files to save from the last maxBand seconds. If this number is exceeded, extra files are skipped and only processed in the next job run.
18572 18573 18574 18575 18576 18577 18578 18579 18580 18581 18582 18583 18584 18585 18586 18587 18588 18589 18590 18591 18592 18593 18594 |
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 18572 class S3CsvSource < Struct.new( :name, :paths, :compression_type, :exclusions, :group_size, :group_files, :recurse, :max_band, :max_files_in_band, :additional_options, :separator, :escaper, :quote_char, :multiline, :with_header, :write_header, :skip_first, :optimize_performance, :output_schemas) SENSITIVE = [] include Aws::Structure end |
#multiline ⇒ Boolean
A Boolean value that specifies whether a single record can span
multiple lines. This can occur when a field contains a quoted
new-line character. You must set this option to True if any record
spans multiple lines. The default value is False
, which allows for
more aggressive file-splitting during parsing.
18572 18573 18574 18575 18576 18577 18578 18579 18580 18581 18582 18583 18584 18585 18586 18587 18588 18589 18590 18591 18592 18593 18594 |
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 18572 class S3CsvSource < Struct.new( :name, :paths, :compression_type, :exclusions, :group_size, :group_files, :recurse, :max_band, :max_files_in_band, :additional_options, :separator, :escaper, :quote_char, :multiline, :with_header, :write_header, :skip_first, :optimize_performance, :output_schemas) SENSITIVE = [] include Aws::Structure end |
#name ⇒ String
The name of the data store.
18572 18573 18574 18575 18576 18577 18578 18579 18580 18581 18582 18583 18584 18585 18586 18587 18588 18589 18590 18591 18592 18593 18594 |
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 18572 class S3CsvSource < Struct.new( :name, :paths, :compression_type, :exclusions, :group_size, :group_files, :recurse, :max_band, :max_files_in_band, :additional_options, :separator, :escaper, :quote_char, :multiline, :with_header, :write_header, :skip_first, :optimize_performance, :output_schemas) SENSITIVE = [] include Aws::Structure end |
#optimize_performance ⇒ Boolean
A Boolean value that specifies whether to use the advanced SIMD CSV reader along with Apache Arrow based columnar memory formats. Only available in Glue version 3.0.
18572 18573 18574 18575 18576 18577 18578 18579 18580 18581 18582 18583 18584 18585 18586 18587 18588 18589 18590 18591 18592 18593 18594 |
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 18572 class S3CsvSource < Struct.new( :name, :paths, :compression_type, :exclusions, :group_size, :group_files, :recurse, :max_band, :max_files_in_band, :additional_options, :separator, :escaper, :quote_char, :multiline, :with_header, :write_header, :skip_first, :optimize_performance, :output_schemas) SENSITIVE = [] include Aws::Structure end |
#output_schemas ⇒ Array<Types::GlueSchema>
Specifies the data schema for the S3 CSV source.
18572 18573 18574 18575 18576 18577 18578 18579 18580 18581 18582 18583 18584 18585 18586 18587 18588 18589 18590 18591 18592 18593 18594 |
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 18572 class S3CsvSource < Struct.new( :name, :paths, :compression_type, :exclusions, :group_size, :group_files, :recurse, :max_band, :max_files_in_band, :additional_options, :separator, :escaper, :quote_char, :multiline, :with_header, :write_header, :skip_first, :optimize_performance, :output_schemas) SENSITIVE = [] include Aws::Structure end |
#paths ⇒ Array<String>
A list of the Amazon S3 paths to read from.
18572 18573 18574 18575 18576 18577 18578 18579 18580 18581 18582 18583 18584 18585 18586 18587 18588 18589 18590 18591 18592 18593 18594 |
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 18572 class S3CsvSource < Struct.new( :name, :paths, :compression_type, :exclusions, :group_size, :group_files, :recurse, :max_band, :max_files_in_band, :additional_options, :separator, :escaper, :quote_char, :multiline, :with_header, :write_header, :skip_first, :optimize_performance, :output_schemas) SENSITIVE = [] include Aws::Structure end |
#quote_char ⇒ String
Specifies the character to use for quoting. The default is a double
quote: '"'
. Set this to -1
to turn off quoting entirely.
18572 18573 18574 18575 18576 18577 18578 18579 18580 18581 18582 18583 18584 18585 18586 18587 18588 18589 18590 18591 18592 18593 18594 |
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 18572 class S3CsvSource < Struct.new( :name, :paths, :compression_type, :exclusions, :group_size, :group_files, :recurse, :max_band, :max_files_in_band, :additional_options, :separator, :escaper, :quote_char, :multiline, :with_header, :write_header, :skip_first, :optimize_performance, :output_schemas) SENSITIVE = [] include Aws::Structure end |
#recurse ⇒ Boolean
If set to true, recursively reads files in all subdirectories under the specified paths.
18572 18573 18574 18575 18576 18577 18578 18579 18580 18581 18582 18583 18584 18585 18586 18587 18588 18589 18590 18591 18592 18593 18594 |
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 18572 class S3CsvSource < Struct.new( :name, :paths, :compression_type, :exclusions, :group_size, :group_files, :recurse, :max_band, :max_files_in_band, :additional_options, :separator, :escaper, :quote_char, :multiline, :with_header, :write_header, :skip_first, :optimize_performance, :output_schemas) SENSITIVE = [] include Aws::Structure end |
#separator ⇒ String
Specifies the delimiter character. The default is a comma: ",", but any other character can be specified.
18572 18573 18574 18575 18576 18577 18578 18579 18580 18581 18582 18583 18584 18585 18586 18587 18588 18589 18590 18591 18592 18593 18594 |
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 18572 class S3CsvSource < Struct.new( :name, :paths, :compression_type, :exclusions, :group_size, :group_files, :recurse, :max_band, :max_files_in_band, :additional_options, :separator, :escaper, :quote_char, :multiline, :with_header, :write_header, :skip_first, :optimize_performance, :output_schemas) SENSITIVE = [] include Aws::Structure end |
#skip_first ⇒ Boolean
A Boolean value that specifies whether to skip the first data line.
The default value is False
.
18572 18573 18574 18575 18576 18577 18578 18579 18580 18581 18582 18583 18584 18585 18586 18587 18588 18589 18590 18591 18592 18593 18594 |
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 18572 class S3CsvSource < Struct.new( :name, :paths, :compression_type, :exclusions, :group_size, :group_files, :recurse, :max_band, :max_files_in_band, :additional_options, :separator, :escaper, :quote_char, :multiline, :with_header, :write_header, :skip_first, :optimize_performance, :output_schemas) SENSITIVE = [] include Aws::Structure end |
#with_header ⇒ Boolean
A Boolean value that specifies whether to treat the first line as a
header. The default value is False
.
18572 18573 18574 18575 18576 18577 18578 18579 18580 18581 18582 18583 18584 18585 18586 18587 18588 18589 18590 18591 18592 18593 18594 |
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 18572 class S3CsvSource < Struct.new( :name, :paths, :compression_type, :exclusions, :group_size, :group_files, :recurse, :max_band, :max_files_in_band, :additional_options, :separator, :escaper, :quote_char, :multiline, :with_header, :write_header, :skip_first, :optimize_performance, :output_schemas) SENSITIVE = [] include Aws::Structure end |
#write_header ⇒ Boolean
A Boolean value that specifies whether to write the header to
output. The default value is True
.
18572 18573 18574 18575 18576 18577 18578 18579 18580 18581 18582 18583 18584 18585 18586 18587 18588 18589 18590 18591 18592 18593 18594 |
# File 'gems/aws-sdk-glue/lib/aws-sdk-glue/types.rb', line 18572 class S3CsvSource < Struct.new( :name, :paths, :compression_type, :exclusions, :group_size, :group_files, :recurse, :max_band, :max_files_in_band, :additional_options, :separator, :escaper, :quote_char, :multiline, :with_header, :write_header, :skip_first, :optimize_performance, :output_schemas) SENSITIVE = [] include Aws::Structure end |