SparkJobProps Struct
Common properties for different types of Spark jobs.
Example
// The code below shows an example of how to instantiate this type.
// The values are placeholders you should change.
import * as glue_alpha from '@aws-cdk/aws-glue-alpha';
import * as cdk from 'aws-cdk-lib';
import { aws_iam as iam } from 'aws-cdk-lib';
import { aws_logs as logs } from 'aws-cdk-lib';
import { aws_s3 as s3 } from 'aws-cdk-lib';
declare const bucket: s3.Bucket;
declare const code: glue_alpha.Code;
declare const connection: glue_alpha.Connection;
declare const logGroup: logs.LogGroup;
declare const role: iam.Role;
declare const securityConfiguration: glue_alpha.SecurityConfiguration;
const sparkJobProps: glue_alpha.SparkJobProps = {
role: role,
script: code,
// the properties below are optional
connections: [connection],
continuousLogging: {
enabled: false,
// the properties below are optional
conversionPattern: 'conversionPattern',
logGroup: logGroup,
logStreamPrefix: 'logStreamPrefix',
quiet: false,
},
defaultArguments: {
defaultArgumentsKey: 'defaultArguments',
},
description: 'description',
enableMetrics: false,
enableObservabilityMetrics: false,
glueVersion: glue_alpha.GlueVersion.V0_9,
jobName: 'jobName',
maxConcurrentRuns: 123,
maxRetries: 123,
securityConfiguration: securityConfiguration,
sparkUI: {
bucket: bucket,
prefix: 'prefix',
},
tags: {
tagsKey: 'tags',
},
timeout: cdk.Duration.minutes(30),
workerConfiguration: {
numberOfWorkers: 123,
workerType: glue_alpha.WorkerType.STANDARD,
},
};
Properties
connectionsIConnection[]connectionstyping.List[IConnection]connectionsList<IConnection>ConnectionsIConnection[]Connections*[]IConnection= [] - no connections are added to…#default
Default: [] - no connections are added to the job
continuousLoggingContinuousLoggingPropscontinuous_loggingContinuousLoggingPropscontinuousLoggingContinuousLoggingPropsContinuousLoggingContinuousLoggingPropsContinuousLoggingContinuousLoggingProps= continuous logging is enabled.#defaultArguments{ [key: string]: string }default_argumentstyping.Mapping[str, str]defaultArgumentsMap<String, String>DefaultArgumentsIDictionary<string, string>DefaultArguments*map[string]*string= no arguments#notes
This map is the escape hatch for Glue job arguments that this construct does not model. It MUST NOT be used to set arguments that already have a dedicated prop — configure those through the corresponding prop instead (continuousLogging, enableMetrics, enableObservabilityMetrics, sparkUI, className, extraJars, extraJarsFirst, extraPythonFiles, extraFiles). Passing a construct-managed argument (e.g. --enable-continuous-cloudwatch-log, --enable-metrics, --enable-spark-ui, --job-language) or a Glue-reserved argument (--debug, --mode, --JOB_NAME, --endpoint) here throws at synthesis time, so there is exactly one way to express each intent.
Also note that these are emitted verbatim into the CloudFormation template, so avoid placing secrets here in plaintext. Pass secrets to the job at runtime through AWS Secrets Manager instead. A synthesis-time warning is emitted when an argument key looks like a credential and holds a plaintext literal.
enableMetricsbooleanenable_metricsboolenableMetricsBooleanEnableMetricsboolEnableMetrics*bool= true#When enabled, adds '--enable-metrics' to job arguments.
enableObservabilityMetricsbooleanenable_observability_metricsboolenableObservabilityMetricsBooleanEnableObservabilityMetricsboolEnableObservabilityMetrics*bool= true#When enabled, adds '--enable-observability-metrics': 'true' to job arguments.
glueVersionGlueVersionglue_versionGlueVersionglueVersionGlueVersionGlueVersionGlueVersionGlueVersionGlueVersion= determined by the job type: 4.0 …#default
Default: determined by the job type: 4.0 for ETL and Streaming, 5.0 for Flex, 3.0 for Python Shell
jobNamestringjob_namestrjobNameStringJobNamestringJobName*string= a name is automatically generate…#default
Default: a name is automatically generated
maxConcurrentRunsnumbermax_concurrent_runsint | floatmaxConcurrentRunsNumberMaxConcurrentRunsdoubleMaxConcurrentRuns*float64= 1#An error is returned when this threshold is reached. The maximum value you can specify is controlled by a service limit.
Can be uploaded from the local directory structure using fromAsset or referenced via S3 location using fromBucket
securityConfigurationISecurityConfigurationsecurity_configurationISecurityConfigurationsecurityConfigurationISecurityConfigurationSecurityConfigurationISecurityConfigurationSecurityConfigurationISecurityConfiguration= no security configuration.#sparkUISparkUIPropsspark_uiSparkUIPropssparkUISparkUIPropsSparkUISparkUIPropsSparkUISparkUIProps= Spark UI debugging and monitorin…#default
Default: Spark UI debugging and monitoring is disabled.
tags{ [key: string]: string }tagstyping.Mapping[str, str]tagsMap<String, String>TagsIDictionary<string, string>Tags*map[string]*string= {} - no tags#timeoutDurationtimeoutDurationtimeoutDurationTimeoutDurationTimeoutDuration= 2880 (2 days for non-streaming)#Specified in minutes.
workerConfigurationWorkerConfigurationworker_configurationWorkerConfigurationworkerConfigurationWorkerConfigurationWorkerConfigurationWorkerConfigurationWorkerConfigurationWorkerConfiguration= the job runs with the G_1X worke…#default
Default: the job runs with the G_1X worker type and 10 workers.