@aws-cdk/aws-pipes-alpha

Amazon EventBridge Pipes Construct Library

Version 2.274.0-alpha.0

Separate package

@aws-cdk/aws-pipes-alpha is not part of aws-cdk-lib — install it separately:

npm install @aws-cdk/aws-pipes-alpha
pip install aws-cdk.aws-pipes-alpha

This module is experimental (alpha): its API may change in any release, without a major version bump.

---

The APIs of higher level constructs in this module are experimental and under active development. They are subject to non-backward compatible changes or removal in any future version. These are not subject to the Semantic Versioning model and breaking changes will be announced in the release notes. This means that while you may use them, you may need to update your source code when upgrading to a newer version of this package.

---

EventBridge Pipes let you create source to target connections between several AWS services. While transporting messages from a source to a target the messages can be filtered, transformed and enriched.

For more details see the service documentation.

Pipe

EventBridge Pipes is a fully managed service that enables point-to-point integrations between event producers and consumers. Pipes can be used to connect several AWS services to each other, or to connect AWS services to external services.

A pipe has a source and a target. The source events can be filtered and enriched before reaching the target.

Example - pipe usage

The following code examples use an example implementation of a source and target.

To define a pipe you need to create a new Pipe construct. The Pipe construct needs a source and a target.

declare const sourceQueue: sqs.Queue;
declare const targetQueue: sqs.Queue;

const pipe = new pipes.Pipe(this, 'Pipe', {
  source: new SqsSource(sourceQueue),
  target: new SqsTarget(targetQueue),
});

# source_queue: sqs.Queue
# target_queue: sqs.Queue


pipe = pipes.Pipe(self, "Pipe",
    source=SqsSource(source_queue),
    target=SqsTarget(target_queue)
)

Queue sourceQueue;
Queue targetQueue;


Pipe pipe = Pipe.Builder.create(this, "Pipe")
        .source(new SqsSource(sourceQueue))
        .target(new SqsTarget(targetQueue))
        .build();

Queue sourceQueue;
Queue targetQueue;


var pipe = new Pipe(this, "Pipe", new PipeProps {
    Source = new SqsSource(sourceQueue),
    Target = new SqsTarget(targetQueue)
});

var sourceQueue Queue
var targetQueue Queue


pipe := pipes.NewPipe(this, jsii.String("Pipe"), &PipeProps{
	Source: awscdkpipessourcesalpha.NewSqsSource(sourceQueue),
	Target: awscdkpipestargetsalpha.NewSqsTarget(targetQueue),
})

This minimal example creates a pipe with a SQS queue as source and a SQS queue as target. Messages from the source are put into the body of the target message.

Source

A source is a AWS Service that is polled. The following sources are possible:

• Amazon DynamoDB stream
• Amazon Kinesis stream
• Amazon MQ broker
• Amazon MSK stream
• Amazon SQS queue
• Apache Kafka stream

Currently, DynamoDB, Kinesis, and SQS are supported. If you are interested in support for additional sources, kindly let us know by opening a GitHub issue or raising a PR.

Example source

declare const sourceQueue: sqs.Queue;
const pipeSource = new SqsSource(sourceQueue);

# source_queue: sqs.Queue

pipe_source = SqsSource(source_queue)

Queue sourceQueue;

SqsSource pipeSource = new SqsSource(sourceQueue);

Queue sourceQueue;

var pipeSource = new SqsSource(sourceQueue);

var sourceQueue Queue

pipeSource := awscdkpipessourcesalpha.NewSqsSource(sourceQueue)

Filter

A filter can be used to filter the events from the source before they are forwarded to the enrichment or, if no enrichment is present, target step. Multiple filter expressions are possible. If one of the filter expressions matches, the event is forwarded to the enrichment or target step.

Example - filter usage

declare const sourceQueue: sqs.Queue;
declare const targetQueue: sqs.Queue;

const sourceFilter = new pipes.Filter(
  [
    pipes.FilterPattern.fromObject({
      body: {
        // only forward events with customerType B2B or B2C
        customerType: ['B2B', 'B2C']
      },
    })
  ]
)

const pipe = new pipes.Pipe(this, 'Pipe', {
  source: new SqsSource(sourceQueue),
  target: new SqsTarget(targetQueue),
  filter: sourceFilter,
});

# source_queue: sqs.Queue
# target_queue: sqs.Queue


source_filter = pipes.Filter([
    pipes.FilterPattern.from_object({
        "body": {
            # only forward events with customerType B2B or B2C
            "customer_type": ["B2B", "B2C"]
        }
    })
])

pipe = pipes.Pipe(self, "Pipe",
    source=SqsSource(source_queue),
    target=SqsTarget(target_queue),
    filter=source_filter
)

Queue sourceQueue;
Queue targetQueue;


Filter sourceFilter = new Filter(List.of(FilterPattern.fromObject(Map.of(
        "body", Map.of(
                // only forward events with customerType B2B or B2C
                "customerType", List.of("B2B", "B2C"))))));

Pipe pipe = Pipe.Builder.create(this, "Pipe")
        .source(new SqsSource(sourceQueue))
        .target(new SqsTarget(targetQueue))
        .filter(sourceFilter)
        .build();

Queue sourceQueue;
Queue targetQueue;


var sourceFilter = new Filter(new [] { FilterPattern.FromObject(new Dictionary<string, object> {
    { "body", new Dictionary<string, string[]> {
        // only forward events with customerType B2B or B2C
        { "customerType", new [] { "B2B", "B2C" } }
    } }
}) });

var pipe = new Pipe(this, "Pipe", new PipeProps {
    Source = new SqsSource(sourceQueue),
    Target = new SqsTarget(targetQueue),
    Filter = sourceFilter
});

var sourceQueue Queue
var targetQueue Queue


sourceFilter := pipes.NewFilter([]IFilterPattern{
	pipes.FilterPattern_FromObject(map[string]interface{}{
		"body": map[string][]*string{
			// only forward events with customerType B2B or B2C
			"customerType": []*string{
				jsii.String("B2B"),
				jsii.String("B2C"),
			},
		},
	}),
})

pipe := pipes.NewPipe(this, jsii.String("Pipe"), &PipeProps{
	Source: awscdkpipessourcesalpha.NewSqsSource(sourceQueue),
	Target: awscdkpipestargetsalpha.NewSqsTarget(targetQueue),
	Filter: sourceFilter,
})

This example shows a filter that only forwards events with the customerType B2B or B2C from the source messages. Messages that are not matching the filter are not forwarded to the enrichment or target step.

You can define multiple filter pattern which are combined with a logical OR.

Additional filter pattern and details can be found in the EventBridge pipes docs.

Input transformation

For enrichments and targets the input event can be transformed. The transformation is applied for each item of the batch. A transformation has access to the input event as well to some context information of the pipe itself like the name of the pipe. See docs for details.

Example - input transformation from object

The input transformation can be created from an object. The object can contain static values, dynamic values or pipe variables.

declare const sourceQueue: sqs.Queue;
declare const targetQueue: sqs.Queue;

const targetInputTransformation = pipes.InputTransformation.fromObject({
  staticField: 'static value',
  dynamicField: pipes.DynamicInput.fromEventPath('$.body.payload'),
  pipeVariable: pipes.DynamicInput.pipeName,
});

const pipe = new pipes.Pipe(this, 'Pipe', {
  pipeName: 'MyPipe',
  source: new SqsSource(sourceQueue),
  target: new SqsTarget(targetQueue, {
    inputTransformation: targetInputTransformation,
  }),
});

# source_queue: sqs.Queue
# target_queue: sqs.Queue


target_input_transformation = pipes.InputTransformation.from_object({
    "static_field": "static value",
    "dynamic_field": pipes.DynamicInput.from_event_path("$.body.payload"),
    "pipe_variable": pipes.DynamicInput.pipe_name
})

pipe = pipes.Pipe(self, "Pipe",
    pipe_name="MyPipe",
    source=SqsSource(source_queue),
    target=SqsTarget(target_queue,
        input_transformation=target_input_transformation
    )
)

Queue sourceQueue;
Queue targetQueue;


InputTransformation targetInputTransformation = InputTransformation.fromObject(Map.of(
        "staticField", "static value",
        "dynamicField", DynamicInput.fromEventPath("$.body.payload"),
        "pipeVariable", DynamicInput.getPipeName()));

Pipe pipe = Pipe.Builder.create(this, "Pipe")
        .pipeName("MyPipe")
        .source(new SqsSource(sourceQueue))
        .target(SqsTarget.Builder.create(targetQueue)
                .inputTransformation(targetInputTransformation)
                .build())
        .build();

Queue sourceQueue;
Queue targetQueue;


var targetInputTransformation = InputTransformation.FromObject(new Dictionary<string, object> {
    { "staticField", "static value" },
    { "dynamicField", DynamicInput.FromEventPath("$.body.payload") },
    { "pipeVariable", DynamicInput.PipeName }
});

var pipe = new Pipe(this, "Pipe", new PipeProps {
    PipeName = "MyPipe",
    Source = new SqsSource(sourceQueue),
    Target = new SqsTarget(targetQueue, new SqsTargetParameters {
        InputTransformation = targetInputTransformation
    })
});

var sourceQueue Queue
var targetQueue Queue


targetInputTransformation := pipes.inputTransformation_FromObject(map[string]interface{}{
	"staticField": jsii.String("static value"),
	"dynamicField": pipes.DynamicInput_fromEventPath(jsii.String("$.body.payload")),
	"pipeVariable": pipes.DynamicInput_pipeName(),
})

pipe := pipes.NewPipe(this, jsii.String("Pipe"), &PipeProps{
	PipeName: jsii.String("MyPipe"),
	Source: awscdkpipessourcesalpha.NewSqsSource(sourceQueue),
	Target: awscdkpipestargetsalpha.NewSqsTarget(targetQueue, &SqsTargetParameters{
		InputTransformation: targetInputTransformation,
	}),
})

This example shows a transformation that adds a static field, a dynamic field and a pipe variable to the input event. The dynamic field is extracted from the input event. The pipe variable is extracted from the pipe context.

So when the following batch of input events is processed by the pipe

[
  {
    ...
    "body": "{\"payload\": \"Test message.\"}",
    ...
  }
]

it is converted into the following payload:

[
  {
    ...
    "staticField": "static value",
    "dynamicField": "Test message.",
    "pipeVariable": "MyPipe",
    ...
  }
]

If the transformation is applied to a target it might be converted to a string representation. For example, the resulting SQS message body looks like this:

[
  {
    ...
    "body": "{\"staticField\": \"static value\", \"dynamicField\": \"Test message.\", \"pipeVariable\": \"MyPipe\"}",
    ...
  }
]

Example - input transformation from event path

In cases where you want to forward only a part of the event to the target you can use the transformation event path.

This only works for targets because the enrichment needs to have a valid json as input.

declare const sourceQueue: sqs.Queue;
declare const targetQueue: sqs.Queue;

const targetInputTransformation = pipes.InputTransformation.fromEventPath('$.body.payload')

const pipe = new pipes.Pipe(this, 'Pipe', {
  source: new SqsSource(sourceQueue),
  target: new SqsTarget(targetQueue, {
    inputTransformation: targetInputTransformation,
  }),
});

# source_queue: sqs.Queue
# target_queue: sqs.Queue


target_input_transformation = pipes.InputTransformation.from_event_path("$.body.payload")

pipe = pipes.Pipe(self, "Pipe",
    source=SqsSource(source_queue),
    target=SqsTarget(target_queue,
        input_transformation=target_input_transformation
    )
)

Queue sourceQueue;
Queue targetQueue;


InputTransformation targetInputTransformation = InputTransformation.fromEventPath("$.body.payload");

Pipe pipe = Pipe.Builder.create(this, "Pipe")
        .source(new SqsSource(sourceQueue))
        .target(SqsTarget.Builder.create(targetQueue)
                .inputTransformation(targetInputTransformation)
                .build())
        .build();

Queue sourceQueue;
Queue targetQueue;


var targetInputTransformation = InputTransformation.FromEventPath("$.body.payload");

var pipe = new Pipe(this, "Pipe", new PipeProps {
    Source = new SqsSource(sourceQueue),
    Target = new SqsTarget(targetQueue, new SqsTargetParameters {
        InputTransformation = targetInputTransformation
    })
});

var sourceQueue Queue
var targetQueue Queue


targetInputTransformation := pipes.inputTransformation_FromEventPath(jsii.String("$.body.payload"))

pipe := pipes.NewPipe(this, jsii.String("Pipe"), &PipeProps{
	Source: awscdkpipessourcesalpha.NewSqsSource(sourceQueue),
	Target: awscdkpipestargetsalpha.NewSqsTarget(targetQueue, &SqsTargetParameters{
		InputTransformation: targetInputTransformation,
	}),
})

This transformation extracts the body of the event.

So when the following batch of input events is processed by the pipe

 [
  {
    ...
    "body": "\"{\"payload\": \"Test message.\"}\"",
    ...
  }
]

it is converted into the following target payload:

[
  {
    ...
    "body": "Test message."
    ...
  }
]

The implicit payload parsing (e.g. SQS message body to JSON) only works if the input is the source payload. Implicit body parsing is not applied on enrichment results.

Example - input transformation from text

In cases where you want to forward a static text to the target or use your own formatted inputTemplate you can use the transformation from text.

declare const sourceQueue: sqs.Queue;
declare const targetQueue: sqs.Queue;

const targetInputTransformation = pipes.InputTransformation.fromText('My static text');

const pipe = new pipes.Pipe(this, 'Pipe', {
  source: new SqsSource(sourceQueue),
  target: new SqsTarget(targetQueue, {
    inputTransformation: targetInputTransformation,
  }),
});

# source_queue: sqs.Queue
# target_queue: sqs.Queue


target_input_transformation = pipes.InputTransformation.from_text("My static text")

pipe = pipes.Pipe(self, "Pipe",
    source=SqsSource(source_queue),
    target=SqsTarget(target_queue,
        input_transformation=target_input_transformation
    )
)

Queue sourceQueue;
Queue targetQueue;


InputTransformation targetInputTransformation = InputTransformation.fromText("My static text");

Pipe pipe = Pipe.Builder.create(this, "Pipe")
        .source(new SqsSource(sourceQueue))
        .target(SqsTarget.Builder.create(targetQueue)
                .inputTransformation(targetInputTransformation)
                .build())
        .build();

Queue sourceQueue;
Queue targetQueue;


var targetInputTransformation = InputTransformation.FromText("My static text");

var pipe = new Pipe(this, "Pipe", new PipeProps {
    Source = new SqsSource(sourceQueue),
    Target = new SqsTarget(targetQueue, new SqsTargetParameters {
        InputTransformation = targetInputTransformation
    })
});

var sourceQueue Queue
var targetQueue Queue


targetInputTransformation := pipes.inputTransformation_FromText(jsii.String("My static text"))

pipe := pipes.NewPipe(this, jsii.String("Pipe"), &PipeProps{
	Source: awscdkpipessourcesalpha.NewSqsSource(sourceQueue),
	Target: awscdkpipestargetsalpha.NewSqsTarget(targetQueue, &SqsTargetParameters{
		InputTransformation: targetInputTransformation,
	}),
})

This transformation forwards the static text to the target.

[
  {
    ...
    "body": "My static text"
    ...
  }
]

Enrichment

In the enrichment step the (un)filtered payloads from the source can be used to invoke one of the following services:

• API destination
• Amazon API Gateway
• Lambda function
• Step Functions state machine
  • only express workflow

Example enrichment implementation

Currently no implementation exist for any of the supported enrichments. The following example shows how an implementation can look like. The actual implementation is not part of this package and will be in a separate one.

class LambdaEnrichment implements pipes.IEnrichment {
  enrichmentArn: string;
  private inputTransformation: pipes.InputTransformation  | undefined;

  constructor(private readonly lambda: lambda.Function, props: {inputTransformation?: pipes.InputTransformation} = {}) {
    this.enrichmentArn = lambda.functionArn;
    this.inputTransformation = props?.inputTransformation
  }

  bind(pipe: pipes.IPipe): pipes.EnrichmentParametersConfig {
    return {
      enrichmentParameters: {
        inputTemplate: this.inputTransformation?.bind(pipe).inputTemplate,
      },
    };
  }

  grantInvoke(pipeRole: cdk.aws_iam.IRole): void {
    this.lambda.grantInvoke(pipeRole);
  }
}

@jsii.implements(pipes.IEnrichment)
class LambdaEnrichment:

    def __init__(self, lambda_, props=None):
        self.enrichment_arn = lambda_.function_arn
        self.input_transformation = props.input_transformation

    def bind(self, pipe):
        return pipes.EnrichmentParametersConfig(
            enrichment_parameters=cdk.aws_pipes.CfnPipe.PipeEnrichmentParametersProperty(
                input_template=self.input_transformation.bind(pipe).input_template
            )
        )

    def grant_invoke(self, pipe_role):
        self.lambda_.grant_invoke(pipe_role)

public class LambdaEnrichment implements IEnrichment {
    public String enrichmentArn;
    private InputTransformation inputTransformation;

    public LambdaEnrichment(Function lambda) {
        this(lambda, Map.of());
    }

    public LambdaEnrichment(Function lambda, Map<String, Object> props) {
        this.enrichmentArn = functionArn;
        this.inputTransformation = props.getInputTransformation();
    }

    public EnrichmentParametersConfig bind(IPipe pipe) {
        return EnrichmentParametersConfig.builder()
                .enrichmentParameters(PipeEnrichmentParametersProperty.builder()
                        .inputTemplate(this.inputTransformation.bind(pipe).getInputTemplate())
                        .build())
                .build();
    }

    public void grantInvoke(IRole pipeRole) {
        this.lambda.grantInvoke(pipeRole);
    }
}

class LambdaEnrichment : IEnrichment
{
    public string EnrichmentArn { get; set; }
    private InputTransformation? inputTransformation;

    public LambdaEnrichment(Function lambda, IDictionary<string, object> props=new Dictionary<string, object> { })
    {
        EnrichmentArn = FunctionArn;
        inputTransformation = props.inputTransformation;
    }

    public EnrichmentParametersConfig Bind(IPipe pipe)
    {
        return new EnrichmentParametersConfig {
            EnrichmentParameters = new PipeEnrichmentParametersProperty {
                InputTemplate = inputTransformation.Bind(pipe).InputTemplate
            }
        };
    }

    public void GrantInvoke(IRole pipeRole)
    {
        Lambda.GrantInvoke(pipeRole);
    }
}

type lambdaEnrichment struct {
	enrichmentArn *string
	inputTransformation inputTransformation
}

func newLambdaEnrichment(lambda Function, props map[string]interface{}) *lambdaEnrichment {
	if props == nil {
		props = map[string]interface{}{
		}
	}
	this := &lambdaEnrichment{}
	this.enrichmentArn = lambda.functionArn
	this.inputTransformation = props.inputTransformation
	return this
}

func (this *lambdaEnrichment) bind(pipe IPipe) EnrichmentParametersConfig {
	return &EnrichmentParametersConfig{
		EnrichmentParameters: &PipeEnrichmentParametersProperty{
			InputTemplate: this.inputTransformation.Bind(pipe).InputTemplate,
		},
	}
}

func (this *lambdaEnrichment) grantInvoke(pipeRole IRole) {
	this.*lambda.GrantInvoke(*pipeRole)
}

An enrichment implementation needs to provide the enrichmentArn, enrichmentParameters and grant the pipe role invoke access to the enrichment.

Example - enrichment usage

declare const sourceQueue: sqs.Queue;
declare const targetQueue: sqs.Queue;
declare const enrichmentLambda: lambda.Function;

const enrichmentInputTransformation = pipes.InputTransformation.fromObject({
  staticField: 'static value',
  dynamicField: pipes.DynamicInput.fromEventPath('$.body.payload'),
  pipeVariable: pipes.DynamicInput.pipeName,
});

const pipe = new pipes.Pipe(this, 'Pipe', {
  source: new SqsSource(sourceQueue),
  target: new SqsTarget(targetQueue),
  enrichment: new LambdaEnrichment(enrichmentLambda, {
    inputTransformation: enrichmentInputTransformation,
  }),
});

# source_queue: sqs.Queue
# target_queue: sqs.Queue
# enrichment_lambda: lambda.Function


enrichment_input_transformation = pipes.InputTransformation.from_object({
    "static_field": "static value",
    "dynamic_field": pipes.DynamicInput.from_event_path("$.body.payload"),
    "pipe_variable": pipes.DynamicInput.pipe_name
})

pipe = pipes.Pipe(self, "Pipe",
    source=SqsSource(source_queue),
    target=SqsTarget(target_queue),
    enrichment=LambdaEnrichment(enrichment_lambda, {
        "input_transformation": enrichment_input_transformation
    })
)

Queue sourceQueue;
Queue targetQueue;
Function enrichmentLambda;


InputTransformation enrichmentInputTransformation = InputTransformation.fromObject(Map.of(
        "staticField", "static value",
        "dynamicField", DynamicInput.fromEventPath("$.body.payload"),
        "pipeVariable", DynamicInput.getPipeName()));

Pipe pipe = Pipe.Builder.create(this, "Pipe")
        .source(new SqsSource(sourceQueue))
        .target(new SqsTarget(targetQueue))
        .enrichment(new LambdaEnrichment(enrichmentLambda, Map.of(
                "inputTransformation", enrichmentInputTransformation)))
        .build();

Queue sourceQueue;
Queue targetQueue;
Function enrichmentLambda;


var enrichmentInputTransformation = InputTransformation.FromObject(new Dictionary<string, object> {
    { "staticField", "static value" },
    { "dynamicField", DynamicInput.FromEventPath("$.body.payload") },
    { "pipeVariable", DynamicInput.PipeName }
});

var pipe = new Pipe(this, "Pipe", new PipeProps {
    Source = new SqsSource(sourceQueue),
    Target = new SqsTarget(targetQueue),
    Enrichment = new LambdaEnrichment(enrichmentLambda, new Dictionary<string, InputTransformation?> {
        { "inputTransformation", enrichmentInputTransformation }
    })
});

var sourceQueue Queue
var targetQueue Queue
var enrichmentLambda Function


enrichmentInputTransformation := pipes.inputTransformation_FromObject(map[string]interface{}{
	"staticField": jsii.String("static value"),
	"dynamicField": pipes.DynamicInput_fromEventPath(jsii.String("$.body.payload")),
	"pipeVariable": pipes.DynamicInput_pipeName(),
})

pipe := pipes.NewPipe(this, jsii.String("Pipe"), &PipeProps{
	Source: awscdkpipessourcesalpha.NewSqsSource(sourceQueue),
	Target: awscdkpipestargetsalpha.NewSqsTarget(targetQueue),
	Enrichment: NewLambdaEnrichment(enrichmentLambda, map[string]*inputTransformation{
		"inputTransformation": enrichmentInputTransformation,
	}),
})

This example adds a lambda function as enrichment to the pipe. The lambda function is invoked with the batch of messages from the source after applying the transformation. The lambda function can return a result which is forwarded to the target.

So the following batch of input events is processed by the pipe

[
  {
    ...
    "body": "{\"payload\": \"Test message.\"}",
    ...
  }
]

it is converted into the following payload which is sent to the lambda function.

[
  {
    ...
    "staticField": "static value",
    "dynamicField": "Test message.",
    "pipeVariable": "MyPipe",
    ...
  }
]

The lambda function can return a result which is forwarded to the target. For example a lambda function that returns a concatenation of the static field, dynamic field and pipe variable

export async function handler (event: any) {
  return event.staticField + "-" + event.dynamicField + "-" + event.pipeVariable;
};

def handler(event):
    return event.static_field + "-" + event.dynamic_field + "-" + event.pipe_variable

public Promise handler(Object event) {
    return event.getStaticField() + "-" + event.getDynamicField() + "-" + event.getPipeVariable();
}

public Promise Handler(object event)
{
    return event.StaticField + "-" + event.DynamicField + "-" + event.PipeVariable;
}

func Handler(event interface{}) promise {
	return jsii.String(*event.staticField + "-" + *event.dynamicField + "-" + *event.pipeVariable)
}

will produce the following target message in the target SQS queue.

[
  {
    ...
    "body": "static value-Test message.-MyPipe",
    ...
  }
]

Target

A Target is the end of the Pipe. After the payload from the source is pulled, filtered and enriched it is forwarded to the target. For now the following targets are supported:

• API destination
• API Gateway
• Batch job queue
• CloudWatch log group
• ECS task
• Event bus in the same account and Region
• Firehose delivery stream
• Inspector assessment template
• Kinesis stream
• Lambda function (SYNC or ASYNC)
• Redshift cluster data API queries
• SageMaker Pipeline
• SNS topic
• SQS queue
• Step Functions state machine
  • Express workflows (ASYNC)
  • Standard workflows (SYNC or ASYNC)

The target event can be transformed before it is forwarded to the target using the same input transformation as in the enrichment step.

Example target

declare const targetQueue: sqs.Queue;
const pipeTarget = new SqsTarget(targetQueue);

# target_queue: sqs.Queue

pipe_target = SqsTarget(target_queue)

Queue targetQueue;

SqsTarget pipeTarget = new SqsTarget(targetQueue);

Queue targetQueue;

var pipeTarget = new SqsTarget(targetQueue);

var targetQueue Queue

pipeTarget := awscdkpipestargetsalpha.NewSqsTarget(targetQueue)

Log destination

A pipe can produce log events that are forwarded to different log destinations. You can configure multiple destinations, but all the destination share the same log level and log data. For details check the official documentation.

The log level and data that is included in the log events is configured on the pipe class itself. The actual destination is defined independently, and there are three options:

1. CloudwatchLogsLogDestination
2. FirehoseLogDestination
3. S3LogDestination

Example log destination usage

declare const sourceQueue: sqs.Queue;
declare const targetQueue: sqs.Queue;
declare const logGroup: logs.LogGroup;

const cwlLogDestination = new pipes.CloudwatchLogsLogDestination(logGroup);

const pipe = new pipes.Pipe(this, 'Pipe', {
  source: new SqsSource(sourceQueue),
  target: new SqsTarget(targetQueue),
  logLevel: pipes.LogLevel.TRACE,
  logIncludeExecutionData: [pipes.IncludeExecutionData.ALL],
  logDestinations: [cwlLogDestination],
});

# source_queue: sqs.Queue
# target_queue: sqs.Queue
# log_group: logs.LogGroup


cwl_log_destination = pipes.CloudwatchLogsLogDestination(log_group)

pipe = pipes.Pipe(self, "Pipe",
    source=SqsSource(source_queue),
    target=SqsTarget(target_queue),
    log_level=pipes.LogLevel.TRACE,
    log_include_execution_data=[pipes.IncludeExecutionData.ALL],
    log_destinations=[cwl_log_destination]
)

Queue sourceQueue;
Queue targetQueue;
LogGroup logGroup;


CloudwatchLogsLogDestination cwlLogDestination = new CloudwatchLogsLogDestination(logGroup);

Pipe pipe = Pipe.Builder.create(this, "Pipe")
        .source(new SqsSource(sourceQueue))
        .target(new SqsTarget(targetQueue))
        .logLevel(LogLevel.TRACE)
        .logIncludeExecutionData(List.of(IncludeExecutionData.ALL))
        .logDestinations(List.of(cwlLogDestination))
        .build();

Queue sourceQueue;
Queue targetQueue;
LogGroup logGroup;


var cwlLogDestination = new CloudwatchLogsLogDestination(logGroup);

var pipe = new Pipe(this, "Pipe", new PipeProps {
    Source = new SqsSource(sourceQueue),
    Target = new SqsTarget(targetQueue),
    LogLevel = LogLevel.TRACE,
    LogIncludeExecutionData = new [] { IncludeExecutionData.ALL },
    LogDestinations = new [] { cwlLogDestination }
});

var sourceQueue Queue
var targetQueue Queue
var logGroup LogGroup


cwlLogDestination := pipes.NewCloudwatchLogsLogDestination(logGroup)

pipe := pipes.NewPipe(this, jsii.String("Pipe"), &PipeProps{
	Source: awscdkpipessourcesalpha.NewSqsSource(sourceQueue),
	Target: awscdkpipestargetsalpha.NewSqsTarget(targetQueue),
	LogLevel: pipes.LogLevel_TRACE,
	LogIncludeExecutionData: []aLL{
		pipes.IncludeExecutionData_*aLL,
	},
	LogDestinations: []ILogDestination{
		cwlLogDestination,
	},
})

This example uses a CloudWatch Logs log group to store the log emitted during a pipe execution. The log level is set to TRACE so all steps of the pipe are logged. Additionally all execution data is logged as well.

Encrypt pipe data with KMS

You can specify that EventBridge use a customer managed key to encrypt pipe data stored at rest, rather than use an AWS owned key as is the default. Details can be found in the documentation.

To do this, you need to specify the key in the kmsKey property of the pipe.

declare const sourceQueue: sqs.Queue;
declare const targetQueue: sqs.Queue;
declare const kmsKey: kms.Key;

const pipe = new pipes.Pipe(this, 'Pipe', {
  source: new SqsSource(sourceQueue),
  target: new SqsTarget(targetQueue),
  kmsKey,
  // pipeName is required when using a KMS key
  pipeName: 'MyPipe',
});

# source_queue: sqs.Queue
# target_queue: sqs.Queue
# kms_key: kms.Key


pipe = pipes.Pipe(self, "Pipe",
    source=SqsSource(source_queue),
    target=SqsTarget(target_queue),
    kms_key=kms_key,
    # pipeName is required when using a KMS key
    pipe_name="MyPipe"
)

Queue sourceQueue;
Queue targetQueue;
Key kmsKey;


Pipe pipe = Pipe.Builder.create(this, "Pipe")
        .source(new SqsSource(sourceQueue))
        .target(new SqsTarget(targetQueue))
        .kmsKey(kmsKey)
        // pipeName is required when using a KMS key
        .pipeName("MyPipe")
        .build();

Queue sourceQueue;
Queue targetQueue;
Key kmsKey;


var pipe = new Pipe(this, "Pipe", new PipeProps {
    Source = new SqsSource(sourceQueue),
    Target = new SqsTarget(targetQueue),
    KmsKey = kmsKey,
    // pipeName is required when using a KMS key
    PipeName = "MyPipe"
});

var sourceQueue Queue
var targetQueue Queue
var kmsKey Key


pipe := pipes.NewPipe(this, jsii.String("Pipe"), &PipeProps{
	Source: awscdkpipessourcesalpha.NewSqsSource(sourceQueue),
	Target: awscdkpipestargetsalpha.NewSqsTarget(targetQueue),
	KmsKey: KmsKey,
	// pipeName is required when using a KMS key
	PipeName: jsii.String("MyPipe"),
})

Types

Name	Kind	Summary
Pipe	Construct (L2)	Amazon EventBridge Pipes connects sources to targets.
CloudwatchLogsLogDestination	Class	CloudWatch Logs log group for delivery of pipe logs.
DesiredState	Enum	The state the pipe should be in.
DynamicInput	Class	Dynamic variables that can be used in the input transformation.
EnrichmentParametersConfig	Struct	The parameters required to set up enrichment on your pipe.
Filter	Class	The collection of event patterns used to filter events.
FilterPattern	Class	Generate a filter pattern from an input.
FirehoseLogDestination	Class	Firehose stream for delivery of pipe logs.
IEnrichment	Interface	Enrichment step to enhance the data from the source before sending it to the target.
IFilter	Interface	The collection of event patterns used to filter events.
IFilterPattern	Interface	Filter events using an event pattern.
IInputTransformation	Interface	Transform or replace the input event payload.
ILogDestination	Interface	Log destination base class.
IncludeExecutionData	Enum	Log data configuration for a pipe.
InputTransformation	Class	Transform or replace the input event payload.
InputTransformationConfig	Struct	The inputTemplate that is used to transform the input event payload with unquoted variables.
IPipe	Interface	Interface representing a created or an imported Pipe.
ISource	Interface	Source interface.
ITarget	Interface	Target configuration.
LogDestinationConfig	Struct	Log destination configuration.
LogDestinationParameters	Struct	Log destination configuration parameters.
LogLevel	Enum	Log configuration for a pipe.
PipeProps	Struct	Properties for a pipe.
PipeVariable	Enum	Reserved pipe variables.
S3LogDestination	Class	S3 bucket for delivery of pipe logs.
S3LogDestinationProps	Struct	Properties for S3LogDestination.
S3OutputFormat	Enum	Log format for S3LogDestination logging configuration.
SourceConfig	Struct	Source properties.
SourceParameters	Struct	Source properties.
SourceWithDeadLetterTarget	Class	Sources that support a dead-letter target.
TargetConfig	Struct	Target config properties.
TargetParameter	Class	Define dynamic target parameters.