AWS Data Pipeline
Developer Guide (API Version 2012-10-29)

Configure an Amazon EMR cluster in a private subnet

This example includes a configuration that launches the cluster into a private subnet in a VPC. For more information, see Launch Amazon EMR Clusters into a VPC in the Amazon EMR Management Guide. This configuration is optional. You can use it in any pipeline that uses an EmrCluster object.

To launch an Amazon EMR cluster in a private subnet, specify SubnetId, emrManagedMasterSecurityGroupId, emrManagedSlaveSecurityGroupId, and serviceAccessSecurityGroupId in your EmrCluster configuration.

{ "objects": [ { "output": { "ref": "S3BackupLocation" }, "input": { "ref": "DDBSourceTable" }, "maximumRetries": "2", "name": "TableBackupActivity", "step": "s3://dynamodb-emr-#{myDDBRegion}/emr-ddb-storage-handler/2.1.0/emr-ddb-2.1.0.jar,org.apache.hadoop.dynamodb.tools.DynamoDbExport,#{output.directoryPath},#{input.tableName},#{input.readThroughputPercent}", "id": "TableBackupActivity", "runsOn": { "ref": "EmrClusterForBackup" }, "type": "EmrActivity", "resizeClusterBeforeRunning": "false" }, { "readThroughputPercent": "#{myDDBReadThroughputRatio}", "name": "DDBSourceTable", "id": "DDBSourceTable", "type": "DynamoDBDataNode", "tableName": "#{myDDBTableName}" }, { "directoryPath": "#{myOutputS3Loc}/#{format(@scheduledStartTime, 'YYYY-MM-dd-HH-mm-ss')}", "name": "S3BackupLocation", "id": "S3BackupLocation", "type": "S3DataNode" }, { "name": "EmrClusterForBackup", "coreInstanceCount": "1", "taskInstanceCount": "1", "taskInstanceType": "m4.xlarge", "coreInstanceType": "m4.xlarge", "releaseLabel": "emr-4.7.0", "masterInstanceType": "m4.xlarge", "id": "EmrClusterForBackup", "subnetId": "#{mySubnetId}", "emrManagedMasterSecurityGroupId": "#{myMasterSecurityGroup}", "emrManagedSlaveSecurityGroupId": "#{mySlaveSecurityGroup}", "serviceAccessSecurityGroupId": "#{myServiceAccessSecurityGroup}", "region": "#{myDDBRegion}", "type": "EmrCluster", "keyPair": "user-key-pair" }, { "failureAndRerunMode": "CASCADE", "resourceRole": "DataPipelineDefaultResourceRole", "role": "DataPipelineDefaultRole", "pipelineLogUri": "#{myPipelineLogUri}", "scheduleType": "ONDEMAND", "name": "Default", "id": "Default" } ], "parameters": [ { "description": "Output S3 folder", "id": "myOutputS3Loc", "type": "AWS::S3::ObjectKey" }, { "description": "Source DynamoDB table name", "id": "myDDBTableName", "type": "String" }, { "default": "0.25", "watermark": "Enter value between 0.1-1.0", "description": "DynamoDB read throughput ratio", "id": "myDDBReadThroughputRatio", "type": "Double" }, { "default": "us-east-1", "watermark": "us-east-1", "description": "Region of the DynamoDB table", "id": "myDDBRegion", "type": "String" } ], "values": { "myDDBRegion": "us-east-1", "myDDBTableName": "ddb_table", "myDDBReadThroughputRatio": "0.25", "myOutputS3Loc": "s3://s3_path", "mySubnetId": "subnet_id", "myServiceAccessSecurityGroup": "service access security group", "mySlaveSecurityGroup": "slave security group", "myMasterSecurityGroup": "master security group", "myPipelineLogUri": "s3://s3_path" } }