@aws-cdk/aws-sagemaker-alpha

Amazon SageMaker Construct Library

Version 2.273.0-alpha.0

Separate package

@aws-cdk/aws-sagemaker-alpha is not part of aws-cdk-lib — install it separately:

npm install @aws-cdk/aws-sagemaker-alpha
pip install aws-cdk.aws-sagemaker-alpha

This module is experimental (alpha): its API may change in any release, without a major version bump.

---

The APIs of higher level constructs in this module are experimental and under active development. They are subject to non-backward compatible changes or removal in any future version. These are not subject to the Semantic Versioning model and breaking changes will be announced in the release notes. This means that while you may use them, you may need to update your source code when upgrading to a newer version of this package.

---

Amazon SageMaker provides every developer and data scientist with the ability to build, train, and deploy machine learning models quickly. Amazon SageMaker is a fully-managed service that covers the entire machine learning workflow to label and prepare your data, choose an algorithm, train the model, tune and optimize it for deployment, make predictions, and take action. Your models get to production faster with much less effort and lower cost.

Model

To create a machine learning model with Amazon Sagemaker, use the Model construct. This construct includes properties that can be configured to define model components, including the model inference code as a Docker image and an optional set of separate model data artifacts. See the AWS documentation to learn more about SageMaker models.

Single Container Model

In the event that a single container is sufficient for your inference use-case, you can define a single-container model:

import * as sagemaker from '@aws-cdk/aws-sagemaker-alpha';
import * as path from 'path';

const image = sagemaker.ContainerImage.fromAsset(path.join('path', 'to', 'Dockerfile', 'directory'));
const modelData = sagemaker.ModelData.fromAsset(path.join('path', 'to', 'artifact', 'file.tar.gz'));

const model = new sagemaker.Model(this, 'PrimaryContainerModel', {
  containers: [
    {
      image: image,
      modelData: modelData,
    }
  ]
});

import aws_cdk.aws_sagemaker_alpha as sagemaker
import path as path


image = sagemaker.ContainerImage.from_asset(path.join("path", "to", "Dockerfile", "directory"))
model_data = sagemaker.ModelData.from_asset(path.join("path", "to", "artifact", "file.tar.gz"))

model = sagemaker.Model(self, "PrimaryContainerModel",
    containers=[sagemaker.ContainerDefinition(
        image=image,
        model_data=model_data
    )
    ]
)

import software.amazon.awscdk.services.sagemaker.alpha.*;
import path.*;


ContainerImage image = ContainerImage.fromAsset(join("path", "to", "Dockerfile", "directory"));
ModelData modelData = ModelData.fromAsset(join("path", "to", "artifact", "file.tar.gz"));

Model model = Model.Builder.create(this, "PrimaryContainerModel")
        .containers(List.of(ContainerDefinition.builder()
                .image(image)
                .modelData(modelData)
                .build()))
        .build();

using Amazon.CDK.AWS.Sagemaker.Alpha;
using Path;


var image = ContainerImage.FromAsset(Join("path", "to", "Dockerfile", "directory"));
var modelData = ModelData.FromAsset(Join("path", "to", "artifact", "file.tar.gz"));

var model = new Model(this, "PrimaryContainerModel", new ModelProps {
    Containers = new [] { new ContainerDefinition {
        Image = image,
        ModelData = modelData
    } }
});

import "github.com/aws/aws-cdk-go/awscdksagemakeralpha"
import "github.com/aws-samples/dummy/path"


image := sagemaker.ContainerImage_FromAsset(path.join(jsii.String("path"), jsii.String("to"), jsii.String("Dockerfile"), jsii.String("directory")))
modelData := sagemaker.ModelData_FromAsset(path.join(jsii.String("path"), jsii.String("to"), jsii.String("artifact"), jsii.String("file.tar.gz")))

model := sagemaker.NewModel(this, jsii.String("PrimaryContainerModel"), &ModelProps{
	Containers: []ContainerDefinition{
		&ContainerDefinition{
			Image: image,
			ModelData: modelData,
		},
	},
})

Inference Pipeline Model

An inference pipeline is an Amazon SageMaker model that is composed of a linear sequence of multiple containers that process requests for inferences on data. See the AWS documentation to learn more about SageMaker inference pipelines. To define an inference pipeline, you can provide additional containers for your model:

import * as sagemaker from '@aws-cdk/aws-sagemaker-alpha';

declare const image1: sagemaker.ContainerImage;
declare const modelData1: sagemaker.ModelData;
declare const image2: sagemaker.ContainerImage;
declare const modelData2: sagemaker.ModelData;
declare const image3: sagemaker.ContainerImage;
declare const modelData3: sagemaker.ModelData;

const model = new sagemaker.Model(this, 'InferencePipelineModel', {
  containers: [
    { image: image1, modelData: modelData1 },
    { image: image2, modelData: modelData2 },
    { image: image3, modelData: modelData3 }
  ],
});

import aws_cdk.aws_sagemaker_alpha as sagemaker

# image1: sagemaker.ContainerImage
# model_data1: sagemaker.ModelData
# image2: sagemaker.ContainerImage
# model_data2: sagemaker.ModelData
# image3: sagemaker.ContainerImage
# model_data3: sagemaker.ModelData


model = sagemaker.Model(self, "InferencePipelineModel",
    containers=[sagemaker.ContainerDefinition(image=image1, model_data=model_data1), sagemaker.ContainerDefinition(image=image2, model_data=model_data2), sagemaker.ContainerDefinition(image=image3, model_data=model_data3)
    ]
)

import software.amazon.awscdk.services.sagemaker.alpha.*;

ContainerImage image1;
ModelData modelData1;
ContainerImage image2;
ModelData modelData2;
ContainerImage image3;
ModelData modelData3;


Model model = Model.Builder.create(this, "InferencePipelineModel")
        .containers(List.of(ContainerDefinition.builder().image(image1).modelData(modelData1).build(), ContainerDefinition.builder().image(image2).modelData(modelData2).build(), ContainerDefinition.builder().image(image3).modelData(modelData3).build()))
        .build();

using Amazon.CDK.AWS.Sagemaker.Alpha;

ContainerImage image1;
ModelData modelData1;
ContainerImage image2;
ModelData modelData2;
ContainerImage image3;
ModelData modelData3;


var model = new Model(this, "InferencePipelineModel", new ModelProps {
    Containers = new [] { new ContainerDefinition { Image = image1, ModelData = modelData1 }, new ContainerDefinition { Image = image2, ModelData = modelData2 }, new ContainerDefinition { Image = image3, ModelData = modelData3 } }
});

import sagemaker "github.com/aws/aws-cdk-go/awscdksagemakeralpha"

var image1 ContainerImage
var modelData1 ModelData
var image2 ContainerImage
var modelData2 ModelData
var image3 ContainerImage
var modelData3 ModelData


model := sagemaker.NewModel(this, jsii.String("InferencePipelineModel"), &ModelProps{
	Containers: []ContainerDefinition{
		&ContainerDefinition{
			Image: image1,
			ModelData: modelData1,
		},
		&ContainerDefinition{
			Image: image2,
			ModelData: modelData2,
		},
		&ContainerDefinition{
			Image: image3,
			ModelData: modelData3,
		},
	},
})

Model Properties

Network Isolation

If you enable network isolation, the containers can't make any outbound network calls, even to other AWS services such as Amazon S3. Additionally, no AWS credentials are made available to the container runtime environment.

To enable network isolation, set the networkIsolation property to true:

import * as sagemaker from '@aws-cdk/aws-sagemaker-alpha';

declare const image: sagemaker.ContainerImage;
declare const modelData: sagemaker.ModelData;

const model = new sagemaker.Model(this, 'ContainerModel', {
  containers: [
    {
      image,
      modelData,
    }
  ],
  networkIsolation: true,
});

import aws_cdk.aws_sagemaker_alpha as sagemaker

# image: sagemaker.ContainerImage
# model_data: sagemaker.ModelData


model = sagemaker.Model(self, "ContainerModel",
    containers=[sagemaker.ContainerDefinition(
        image=image,
        model_data=model_data
    )
    ],
    network_isolation=True
)

import software.amazon.awscdk.services.sagemaker.alpha.*;

ContainerImage image;
ModelData modelData;


Model model = Model.Builder.create(this, "ContainerModel")
        .containers(List.of(ContainerDefinition.builder()
                .image(image)
                .modelData(modelData)
                .build()))
        .networkIsolation(true)
        .build();

using Amazon.CDK.AWS.Sagemaker.Alpha;

ContainerImage image;
ModelData modelData;


var model = new Model(this, "ContainerModel", new ModelProps {
    Containers = new [] { new ContainerDefinition {
        Image = image,
        ModelData = modelData
    } },
    NetworkIsolation = true
});

import sagemaker "github.com/aws/aws-cdk-go/awscdksagemakeralpha"

var image ContainerImage
var modelData ModelData


model := sagemaker.NewModel(this, jsii.String("ContainerModel"), &ModelProps{
	Containers: []ContainerDefinition{
		&ContainerDefinition{
			Image: *Image,
			ModelData: *ModelData,
		},
	},
	NetworkIsolation: jsii.Boolean(true),
})

Container Images

Inference code can be stored in the Amazon EC2 Container Registry (Amazon ECR), which is specified via ContainerDefinition's image property which accepts a class that extends the ContainerImage abstract base class.

Asset Image

Reference a local directory containing a Dockerfile:

import * as sagemaker from '@aws-cdk/aws-sagemaker-alpha';
import * as path from 'path';

const image = sagemaker.ContainerImage.fromAsset(path.join('path', 'to', 'Dockerfile', 'directory'));

import aws_cdk.aws_sagemaker_alpha as sagemaker
import path as path


image = sagemaker.ContainerImage.from_asset(path.join("path", "to", "Dockerfile", "directory"))

import software.amazon.awscdk.services.sagemaker.alpha.*;
import path.*;


ContainerImage image = ContainerImage.fromAsset(join("path", "to", "Dockerfile", "directory"));

using Amazon.CDK.AWS.Sagemaker.Alpha;
using Path;


var image = ContainerImage.FromAsset(Join("path", "to", "Dockerfile", "directory"));

import sagemaker "github.com/aws/aws-cdk-go/awscdksagemakeralpha"
import path "github.com/aws-samples/dummy/path"


image := sagemaker.ContainerImage_FromAsset(path.join(jsii.String("path"), jsii.String("to"), jsii.String("Dockerfile"), jsii.String("directory")))

ECR Image

Reference an image available within ECR:

import * as ecr from 'aws-cdk-lib/aws-ecr';
import * as sagemaker from '@aws-cdk/aws-sagemaker-alpha';

const repository = ecr.Repository.fromRepositoryName(this, 'Repository', 'repo');
const image = sagemaker.ContainerImage.fromEcrRepository(repository, 'tag');

import aws_cdk.aws_ecr as ecr
import aws_cdk.aws_sagemaker_alpha as sagemaker


repository = ecr.Repository.from_repository_name(self, "Repository", "repo")
image = sagemaker.ContainerImage.from_ecr_repository(repository, "tag")

import software.amazon.awscdk.services.ecr.*;
import software.amazon.awscdk.services.sagemaker.alpha.*;


IRepository repository = Repository.fromRepositoryName(this, "Repository", "repo");
ContainerImage image = ContainerImage.fromEcrRepository(repository, "tag");

using Amazon.CDK.AWS.ECR;
using Amazon.CDK.AWS.Sagemaker.Alpha;


var repository = Repository.FromRepositoryName(this, "Repository", "repo");
var image = ContainerImage.FromEcrRepository(repository, "tag");

import ecr "github.com/aws/aws-cdk-go/awscdk"
import sagemaker "github.com/aws/aws-cdk-go/awscdksagemakeralpha"


repository := ecr.Repository_FromRepositoryName(this, jsii.String("Repository"), jsii.String("repo"))
image := sagemaker.ContainerImage_FromEcrRepository(repository, jsii.String("tag"))

DLC Image

Reference a deep learning container image:

import * as sagemaker from '@aws-cdk/aws-sagemaker-alpha';

const repositoryName = 'huggingface-pytorch-training';
const tag = '1.13.1-transformers4.26.0-gpu-py39-cu117-ubuntu20.04';

const image = sagemaker.ContainerImage.fromDlc(repositoryName, tag);

import aws_cdk.aws_sagemaker_alpha as sagemaker


repository_name = "huggingface-pytorch-training"
tag = "1.13.1-transformers4.26.0-gpu-py39-cu117-ubuntu20.04"

image = sagemaker.ContainerImage.from_dlc(repository_name, tag)

import software.amazon.awscdk.services.sagemaker.alpha.*;


String repositoryName = "huggingface-pytorch-training";
String tag = "1.13.1-transformers4.26.0-gpu-py39-cu117-ubuntu20.04";

ContainerImage image = ContainerImage.fromDlc(repositoryName, tag);

using Amazon.CDK.AWS.Sagemaker.Alpha;


var repositoryName = "huggingface-pytorch-training";
var tag = "1.13.1-transformers4.26.0-gpu-py39-cu117-ubuntu20.04";

var image = ContainerImage.FromDlc(repositoryName, tag);

import sagemaker "github.com/aws/aws-cdk-go/awscdksagemakeralpha"


repositoryName := "huggingface-pytorch-training"
tag := "1.13.1-transformers4.26.0-gpu-py39-cu117-ubuntu20.04"

image := sagemaker.ContainerImage_FromDlc(repositoryName, tag)

Model Artifacts

If you choose to decouple your model artifacts from your inference code (as is natural given different rates of change between inference code and model artifacts), the artifacts can be specified via the modelData property which accepts a class that extends the ModelData abstract base class. The default is to have no model artifacts associated with a model.

Asset Model Data

Reference local model data:

import * as sagemaker from '@aws-cdk/aws-sagemaker-alpha';
import * as path from 'path';

const modelData = sagemaker.ModelData.fromAsset(path.join('path', 'to', 'artifact', 'file.tar.gz'));

import aws_cdk.aws_sagemaker_alpha as sagemaker
import path as path


model_data = sagemaker.ModelData.from_asset(path.join("path", "to", "artifact", "file.tar.gz"))

import software.amazon.awscdk.services.sagemaker.alpha.*;
import path.*;


ModelData modelData = ModelData.fromAsset(join("path", "to", "artifact", "file.tar.gz"));

using Amazon.CDK.AWS.Sagemaker.Alpha;
using Path;


var modelData = ModelData.FromAsset(Join("path", "to", "artifact", "file.tar.gz"));

import sagemaker "github.com/aws/aws-cdk-go/awscdksagemakeralpha"
import path "github.com/aws-samples/dummy/path"


modelData := sagemaker.ModelData_FromAsset(path.join(jsii.String("path"), jsii.String("to"), jsii.String("artifact"), jsii.String("file.tar.gz")))

S3 Model Data

Reference an S3 bucket and object key as the artifacts for a model:

import * as s3 from 'aws-cdk-lib/aws-s3';
import * as sagemaker from '@aws-cdk/aws-sagemaker-alpha';

const bucket = new s3.Bucket(this, 'MyBucket');
const modelData = sagemaker.ModelData.fromBucket(bucket, 'path/to/artifact/file.tar.gz');

import aws_cdk.aws_s3 as s3
import aws_cdk.aws_sagemaker_alpha as sagemaker


bucket = s3.Bucket(self, "MyBucket")
model_data = sagemaker.ModelData.from_bucket(bucket, "path/to/artifact/file.tar.gz")

import software.amazon.awscdk.services.s3.*;
import software.amazon.awscdk.services.sagemaker.alpha.*;


Bucket bucket = new Bucket(this, "MyBucket");
ModelData modelData = ModelData.fromBucket(bucket, "path/to/artifact/file.tar.gz");

using Amazon.CDK.AWS.S3;
using Amazon.CDK.AWS.Sagemaker.Alpha;


var bucket = new Bucket(this, "MyBucket");
var modelData = ModelData.FromBucket(bucket, "path/to/artifact/file.tar.gz");

import s3 "github.com/aws/aws-cdk-go/awscdk"
import sagemaker "github.com/aws/aws-cdk-go/awscdksagemakeralpha"


bucket := s3.NewBucket(this, jsii.String("MyBucket"))
modelData := sagemaker.ModelData_FromBucket(bucket, jsii.String("path/to/artifact/file.tar.gz"))

Model Hosting

Amazon SageMaker provides model hosting services for model deployment. Amazon SageMaker provides an HTTPS endpoint where your machine learning model is available to provide inferences.

Endpoint Configuration

By using the EndpointConfig construct, you can define a set of endpoint configuration which can be used to provision one or more endpoints. In this configuration, you identify one or more models to deploy and the resources that you want Amazon SageMaker to provision. You define one or more production variants, each of which identifies a model. Each production variant also describes the resources that you want Amazon SageMaker to provision. If you are hosting multiple models, you also assign a variant weight to specify how much traffic you want to allocate to each model. For example, suppose that you want to host two models, A and B, and you assign traffic weight 2 for model A and 1 for model B. Amazon SageMaker distributes two-thirds of the traffic to Model A, and one-third to model B:

import * as sagemaker from '@aws-cdk/aws-sagemaker-alpha';

declare const modelA: sagemaker.Model;
declare const modelB: sagemaker.Model;

const endpointConfig = new sagemaker.EndpointConfig(this, 'EndpointConfig', {
  instanceProductionVariants: [
    {
      model: modelA,
      variantName: 'modelA',
      initialVariantWeight: 2.0,
    },
    {
      model: modelB,
      variantName: 'variantB',
      initialVariantWeight: 1.0,
    },
  ]
});

import aws_cdk.aws_sagemaker_alpha as sagemaker

# model_a: sagemaker.Model
# model_b: sagemaker.Model


endpoint_config = sagemaker.EndpointConfig(self, "EndpointConfig",
    instance_production_variants=[sagemaker.InstanceProductionVariantProps(
        model=model_a,
        variant_name="modelA",
        initial_variant_weight=2
    ), sagemaker.InstanceProductionVariantProps(
        model=model_b,
        variant_name="variantB",
        initial_variant_weight=1
    )
    ]
)

import software.amazon.awscdk.services.sagemaker.alpha.*;

Model modelA;
Model modelB;


EndpointConfig endpointConfig = EndpointConfig.Builder.create(this, "EndpointConfig")
        .instanceProductionVariants(List.of(InstanceProductionVariantProps.builder()
                .model(modelA)
                .variantName("modelA")
                .initialVariantWeight(2)
                .build(), InstanceProductionVariantProps.builder()
                .model(modelB)
                .variantName("variantB")
                .initialVariantWeight(1)
                .build()))
        .build();

using Amazon.CDK.AWS.Sagemaker.Alpha;

Model modelA;
Model modelB;


var endpointConfig = new EndpointConfig(this, "EndpointConfig", new EndpointConfigProps {
    InstanceProductionVariants = new [] { new InstanceProductionVariantProps {
        Model = modelA,
        VariantName = "modelA",
        InitialVariantWeight = 2
    }, new InstanceProductionVariantProps {
        Model = modelB,
        VariantName = "variantB",
        InitialVariantWeight = 1
    } }
});

import sagemaker "github.com/aws/aws-cdk-go/awscdksagemakeralpha"

var modelA Model
var modelB Model


endpointConfig := sagemaker.NewEndpointConfig(this, jsii.String("EndpointConfig"), &EndpointConfigProps{
	InstanceProductionVariants: []InstanceProductionVariantProps{
		&InstanceProductionVariantProps{
			Model: modelA,
			VariantName: jsii.String("modelA"),
			InitialVariantWeight: jsii.Number(2),
		},
		&InstanceProductionVariantProps{
			Model: modelB,
			VariantName: jsii.String("variantB"),
			InitialVariantWeight: jsii.Number(1),
		},
	},
})

Container Startup Health Check Timeout

You can specify a timeout value for your inference container to pass health check by configuring the containerStartupHealthCheckTimeout property. This is useful when your model takes longer to initialize and you want to avoid premature health check failures:

import * as sagemaker from '@aws-cdk/aws-sagemaker-alpha';

declare const model: sagemaker.Model;

const endpointConfig = new sagemaker.EndpointConfig(this, 'EndpointConfig', {
  instanceProductionVariants: [
    {
      model: model,
      variantName: 'my-variant',
      containerStartupHealthCheckTimeout: cdk.Duration.minutes(5), // 5 minutes timeout
    },
  ]
});

import aws_cdk.aws_sagemaker_alpha as sagemaker

# model: sagemaker.Model


endpoint_config = sagemaker.EndpointConfig(self, "EndpointConfig",
    instance_production_variants=[sagemaker.InstanceProductionVariantProps(
        model=model,
        variant_name="my-variant",
        container_startup_health_check_timeout=cdk.Duration.minutes(5)
    )
    ]
)

import software.amazon.awscdk.services.sagemaker.alpha.*;

Model model;


EndpointConfig endpointConfig = EndpointConfig.Builder.create(this, "EndpointConfig")
        .instanceProductionVariants(List.of(InstanceProductionVariantProps.builder()
                .model(model)
                .variantName("my-variant")
                .containerStartupHealthCheckTimeout(Duration.minutes(5))
                .build()))
        .build();

using Amazon.CDK.AWS.Sagemaker.Alpha;

Model model;


var endpointConfig = new EndpointConfig(this, "EndpointConfig", new EndpointConfigProps {
    InstanceProductionVariants = new [] { new InstanceProductionVariantProps {
        Model = model,
        VariantName = "my-variant",
        ContainerStartupHealthCheckTimeout = Duration.Minutes(5)
    } }
});

import sagemaker "github.com/aws/aws-cdk-go/awscdksagemakeralpha"

var model Model


endpointConfig := sagemaker.NewEndpointConfig(this, jsii.String("EndpointConfig"), &EndpointConfigProps{
	InstanceProductionVariants: []InstanceProductionVariantProps{
		&InstanceProductionVariantProps{
			Model: model,
			VariantName: jsii.String("my-variant"),
			ContainerStartupHealthCheckTimeout: cdk.Duration_Minutes(jsii.Number(5)),
		},
	},
})

The timeout value must be between 60 seconds and 1 hour (3600 seconds). If not specified, Amazon SageMaker uses the default timeout behavior.

Serverless Inference

Amazon SageMaker Serverless Inference is a purpose-built inference option that makes it easy for you to deploy and scale ML models. Serverless endpoints automatically launch compute resources and scale them in and out depending on traffic, eliminating the need to choose instance types or manage scaling policies. For more information, see SageMaker Serverless Inference.

To create a serverless endpoint configuration, use the serverlessProductionVariant property:

import * as sagemaker from '@aws-cdk/aws-sagemaker-alpha';

declare const model: sagemaker.Model;

const endpointConfig = new sagemaker.EndpointConfig(this, 'ServerlessEndpointConfig', {
  serverlessProductionVariant: {
    model: model,
    variantName: 'serverlessVariant',
    maxConcurrency: 10,
    memorySizeInMB: 2048,
    provisionedConcurrency: 5, // optional
  },
});

import aws_cdk.aws_sagemaker_alpha as sagemaker

# model: sagemaker.Model


endpoint_config = sagemaker.EndpointConfig(self, "ServerlessEndpointConfig",
    serverless_production_variant=sagemaker.ServerlessProductionVariantProps(
        model=model,
        variant_name="serverlessVariant",
        max_concurrency=10,
        memory_size_in_mb=2048,
        provisioned_concurrency=5
    )
)

import software.amazon.awscdk.services.sagemaker.alpha.*;

Model model;


EndpointConfig endpointConfig = EndpointConfig.Builder.create(this, "ServerlessEndpointConfig")
        .serverlessProductionVariant(ServerlessProductionVariantProps.builder()
                .model(model)
                .variantName("serverlessVariant")
                .maxConcurrency(10)
                .memorySizeInMB(2048)
                .provisionedConcurrency(5)
                .build())
        .build();

using Amazon.CDK.AWS.Sagemaker.Alpha;

Model model;


var endpointConfig = new EndpointConfig(this, "ServerlessEndpointConfig", new EndpointConfigProps {
    ServerlessProductionVariant = new ServerlessProductionVariantProps {
        Model = model,
        VariantName = "serverlessVariant",
        MaxConcurrency = 10,
        MemorySizeInMB = 2048,
        ProvisionedConcurrency = 5
    }
});

import sagemaker "github.com/aws/aws-cdk-go/awscdksagemakeralpha"

var model Model


endpointConfig := sagemaker.NewEndpointConfig(this, jsii.String("ServerlessEndpointConfig"), &EndpointConfigProps{
	ServerlessProductionVariant: &ServerlessProductionVariantProps{
		Model: model,
		VariantName: jsii.String("serverlessVariant"),
		MaxConcurrency: jsii.Number(10),
		MemorySizeInMB: jsii.Number(2048),
		ProvisionedConcurrency: jsii.Number(5),
	},
})

Serverless inference is ideal for workloads with intermittent or unpredictable traffic patterns. You can configure:

• maxConcurrency: Maximum concurrent invocations (1-200)
• memorySizeInMB: Memory allocation in 1GB increments (1024, 2048, 3072, 4096, 5120, or 6144 MB)
• provisionedConcurrency: Optional pre-warmed capacity to reduce cold starts

Note: Provisioned concurrency incurs charges even when the endpoint is not processing requests. Use it only when you need to minimize cold start latency.

You cannot mix serverless and instance-based variants in the same endpoint configuration.

Endpoint

When you create an endpoint from an EndpointConfig, Amazon SageMaker launches the ML compute instances and deploys the model or models as specified in the configuration. To get inferences from the model, client applications send requests to the Amazon SageMaker Runtime HTTPS endpoint. For more information about the API, see the InvokeEndpoint API. Defining an endpoint requires at minimum the associated endpoint configuration:

import * as sagemaker from '@aws-cdk/aws-sagemaker-alpha';

declare const endpointConfig: sagemaker.EndpointConfig;

const endpoint = new sagemaker.Endpoint(this, 'Endpoint', { endpointConfig });

import aws_cdk.aws_sagemaker_alpha as sagemaker

# endpoint_config: sagemaker.EndpointConfig


endpoint = sagemaker.Endpoint(self, "Endpoint", endpoint_config=endpoint_config)

import software.amazon.awscdk.services.sagemaker.alpha.*;

EndpointConfig endpointConfig;


Endpoint endpoint = Endpoint.Builder.create(this, "Endpoint").endpointConfig(endpointConfig).build();

using Amazon.CDK.AWS.Sagemaker.Alpha;

EndpointConfig endpointConfig;


var endpoint = new Endpoint(this, "Endpoint", new EndpointProps { EndpointConfig = endpointConfig });

import sagemaker "github.com/aws/aws-cdk-go/awscdksagemakeralpha"

var endpointConfig EndpointConfig


endpoint := sagemaker.NewEndpoint(this, jsii.String("Endpoint"), &EndpointProps{
	EndpointConfig: EndpointConfig,
})

AutoScaling

To enable autoscaling on the production variant, use the autoScaleInstanceCount method:

import * as sagemaker from '@aws-cdk/aws-sagemaker-alpha';

declare const model: sagemaker.Model;

const variantName = 'my-variant';
const endpointConfig = new sagemaker.EndpointConfig(this, 'EndpointConfig', {
  instanceProductionVariants: [
    {
      model: model,
      variantName: variantName,
    },
  ]
});

const endpoint = new sagemaker.Endpoint(this, 'Endpoint', { endpointConfig });
const productionVariant = endpoint.findInstanceProductionVariant(variantName);
const instanceCount = productionVariant.autoScaleInstanceCount({
  maxCapacity: 3
});
instanceCount.scaleOnInvocations('LimitRPS', {
  maxRequestsPerSecond: 30,
});

import aws_cdk.aws_sagemaker_alpha as sagemaker

# model: sagemaker.Model


variant_name = "my-variant"
endpoint_config = sagemaker.EndpointConfig(self, "EndpointConfig",
    instance_production_variants=[sagemaker.InstanceProductionVariantProps(
        model=model,
        variant_name=variant_name
    )
    ]
)

endpoint = sagemaker.Endpoint(self, "Endpoint", endpoint_config=endpoint_config)
production_variant = endpoint.find_instance_production_variant(variant_name)
instance_count = production_variant.auto_scale_instance_count(
    max_capacity=3
)
instance_count.scale_on_invocations("LimitRPS",
    max_requests_per_second=30
)

import software.amazon.awscdk.services.sagemaker.alpha.*;

Model model;


String variantName = "my-variant";
EndpointConfig endpointConfig = EndpointConfig.Builder.create(this, "EndpointConfig")
        .instanceProductionVariants(List.of(InstanceProductionVariantProps.builder()
                .model(model)
                .variantName(variantName)
                .build()))
        .build();

Endpoint endpoint = Endpoint.Builder.create(this, "Endpoint").endpointConfig(endpointConfig).build();
IEndpointInstanceProductionVariant productionVariant = endpoint.findInstanceProductionVariant(variantName);
ScalableInstanceCount instanceCount = productionVariant.autoScaleInstanceCount(EnableScalingProps.builder()
        .maxCapacity(3)
        .build());
instanceCount.scaleOnInvocations("LimitRPS", InvocationsScalingProps.builder()
        .maxRequestsPerSecond(30)
        .build());

using Amazon.CDK.AWS.Sagemaker.Alpha;

Model model;


var variantName = "my-variant";
var endpointConfig = new EndpointConfig(this, "EndpointConfig", new EndpointConfigProps {
    InstanceProductionVariants = new [] { new InstanceProductionVariantProps {
        Model = model,
        VariantName = variantName
    } }
});

var endpoint = new Endpoint(this, "Endpoint", new EndpointProps { EndpointConfig = endpointConfig });
var productionVariant = endpoint.FindInstanceProductionVariant(variantName);
var instanceCount = productionVariant.AutoScaleInstanceCount(new EnableScalingProps {
    MaxCapacity = 3
});
instanceCount.ScaleOnInvocations("LimitRPS", new InvocationsScalingProps {
    MaxRequestsPerSecond = 30
});

import "github.com/aws/aws-cdk-go/awscdksagemakeralpha"

var model Model


variantName := "my-variant"
endpointConfig := sagemaker.NewEndpointConfig(this, jsii.String("EndpointConfig"), &EndpointConfigProps{
	InstanceProductionVariants: []InstanceProductionVariantProps{
		&InstanceProductionVariantProps{
			Model: model,
			VariantName: variantName,
		},
	},
})

endpoint := sagemaker.NewEndpoint(this, jsii.String("Endpoint"), &EndpointProps{
	EndpointConfig: EndpointConfig,
})
productionVariant := endpoint.FindInstanceProductionVariant(variantName)
instanceCount := productionVariant.AutoScaleInstanceCount(&EnableScalingProps{
	MaxCapacity: jsii.Number(3),
})
instanceCount.ScaleOnInvocations(jsii.String("LimitRPS"), &InvocationsScalingProps{
	MaxRequestsPerSecond: jsii.Number(30),
})

For load testing guidance on determining the maximum requests per second per instance, please see this documentation.

Metrics

To monitor CloudWatch metrics for a production variant, use one or more of the metric convenience methods:

import * as sagemaker from '@aws-cdk/aws-sagemaker-alpha';

declare const endpointConfig: sagemaker.EndpointConfig;

const endpoint = new sagemaker.Endpoint(this, 'Endpoint', { endpointConfig });
const productionVariant = endpoint.findInstanceProductionVariant('my-variant');
productionVariant.metricModelLatency().createAlarm(this, 'ModelLatencyAlarm', {
  threshold: 100000,
  evaluationPeriods: 3,
});

import aws_cdk.aws_sagemaker_alpha as sagemaker

# endpoint_config: sagemaker.EndpointConfig


endpoint = sagemaker.Endpoint(self, "Endpoint", endpoint_config=endpoint_config)
production_variant = endpoint.find_instance_production_variant("my-variant")
production_variant.metric_model_latency().create_alarm(self, "ModelLatencyAlarm",
    threshold=100000,
    evaluation_periods=3
)

import software.amazon.awscdk.services.sagemaker.alpha.*;

EndpointConfig endpointConfig;


Endpoint endpoint = Endpoint.Builder.create(this, "Endpoint").endpointConfig(endpointConfig).build();
IEndpointInstanceProductionVariant productionVariant = endpoint.findInstanceProductionVariant("my-variant");
productionVariant.metricModelLatency().createAlarm(this, "ModelLatencyAlarm", CreateAlarmOptions.builder()
        .threshold(100000)
        .evaluationPeriods(3)
        .build());

using Amazon.CDK.AWS.Sagemaker.Alpha;

EndpointConfig endpointConfig;


var endpoint = new Endpoint(this, "Endpoint", new EndpointProps { EndpointConfig = endpointConfig });
var productionVariant = endpoint.FindInstanceProductionVariant("my-variant");
productionVariant.MetricModelLatency().CreateAlarm(this, "ModelLatencyAlarm", new CreateAlarmOptions {
    Threshold = 100000,
    EvaluationPeriods = 3
});

import sagemaker "github.com/aws/aws-cdk-go/awscdksagemakeralpha"

var endpointConfig EndpointConfig


endpoint := sagemaker.NewEndpoint(this, jsii.String("Endpoint"), &EndpointProps{
	EndpointConfig: EndpointConfig,
})
productionVariant := endpoint.FindInstanceProductionVariant(jsii.String("my-variant"))
productionVariant.MetricModelLatency().CreateAlarm(this, jsii.String("ModelLatencyAlarm"), &CreateAlarmOptions{
	Threshold: jsii.Number(100000),
	EvaluationPeriods: jsii.Number(3),
})

Types

Name	Kind	Summary
Endpoint	Construct (L2)	Defines a SageMaker endpoint.
EndpointConfig	Construct (L2)	Defines a SageMaker EndpointConfig.
Model	Construct (L2)	Defines a SageMaker Model.
Pipeline	Construct (L2)	A SageMaker Pipeline.
AcceleratorType	Class	Supported Elastic Inference (EI) instance types for SageMaker instance-based production variants.
ContainerDefinition	Struct	Describes the container, as part of model definition.
ContainerImage	Class	Constructs for types of container images.
ContainerImageConfig	Struct	The configuration for creating a container image.
EndpointAttributes	Struct	Represents an Endpoint resource defined outside this stack.
EndpointConfigProps	Struct	Construction properties for a SageMaker EndpointConfig.
EndpointProps	Struct	Construction properties for a SageMaker Endpoint.
IEndpoint	Interface	The Interface for a SageMaker Endpoint resource.
IEndpointConfig	Interface	The interface for a SageMaker EndpointConfig resource.
IEndpointInstanceProductionVariant	Interface	Represents an instance production variant that has been associated with an endpoint.
IModel	Interface	Interface that defines a Model resource.
InstanceProductionVariantProps	Struct	Construction properties for an instance production variant.
InstanceType	Class	Supported instance types for SageMaker instance-based production variants.
InvocationHttpResponseCode	Enum	HTTP response codes for Endpoint invocations.
InvocationsScalingProps	Struct	Properties for enabling SageMaker Endpoint utilization tracking.
ModelAttributes	Struct	Represents a Model resource defined outside this stack.
ModelData	Class	Model data represents the source of model artifacts, which will ultimately be loaded from an S3 location.
ModelDataConfig	Struct	The configuration needed to reference model artifacts.
ModelProps	Struct	Construction properties for a SageMaker Model.
PipelineAttributes	Struct	Attributes for importing a SageMaker Pipeline.
PipelineProps	Struct	Properties for defining a SageMaker Pipeline.
ScalableInstanceCount	Construct	A scalable sagemaker endpoint attribute.
ScalableInstanceCountProps	Struct	The properties of a scalable attribute representing task count.
ServerlessProductionVariantProps	Struct	Construction properties for a serverless production variant.