Menu
AWS CloudFormation
User Guide (API Version 2010-05-15)

Amazon CloudWatch Logs Template Snippets

Send Logs to CloudWatch Logs from an Instance

Amazon CloudWatch Logs can monitor your system, application, and custom log files from Amazon EC2 instances or other sources. You can use AWS CloudFormation to provision and manage log groups and metric filters. For more information about getting started with Amazon CloudWatch Logs, see Monitoring System, Application, and Custom Log Files in the Amazon CloudWatch User Guide.

The following template describes a web server and its custom metrics. Log events from the web server's log provides the data for the custom metrics. To send log events to a custom metric, the UserData field installs a CloudWatch Logs agent on the Amazon EC2 instance. The configuration information for the agent, such as the location of the server log file, the log group name, and the log stream name, are defined in the /tmp/cwlogs/apacheaccess.conf file. The log stream is created after the web server starts sending log events to the /var/log/httpd/access_log file.

The two metric filters describe how the log information is transformed into CloudWatch metrics. The 404 metric counts the number of 404 occurrences. The size metric tracks the size of a request. The two CloudWatch alarms will send notifications if there are more than two 404s within two minutes or if the average request size is over 3500 KB over 10 minutes.

JSON

Copy
{ "AWSTemplateFormatVersion": "2010-09-09", "Description": "AWS CloudFormation Sample Template for CloudWatch Logs.", "Parameters": { "KeyName": { "Description": "Name of an existing EC2 KeyPair to enable SSH access to the instances", "Type": "AWS::EC2::KeyPair::KeyName", "ConstraintDescription" : "must be the name of an existing EC2 KeyPair." }, "SSHLocation" : { "Description" : "The IP address range that can be used to SSH to the EC2 instances", "Type": "String", "MinLength": "9", "MaxLength": "18", "Default": "0.0.0.0/0", "AllowedPattern": "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})", "ConstraintDescription": "must be a valid IP CIDR range of the form x.x.x.x/x." }, "OperatorEmail": { "Description": "Email address to notify if there are any scaling operations", "Type": "String" } }, "Mappings": { "RegionMap": { "us-east-1": { "AMI": "ami-fb8e9292" }, "us-west-1": { "AMI": "ami-7aba833f" }, "us-west-2": { "AMI": "ami-043a5034" }, "eu-west-1": { "AMI": "ami-2918e35e" }, "ap-southeast-1": { "AMI": "ami-b40d5ee6" }, "ap-southeast-2": { "AMI": "ami-3b4bd301" }, "ap-northeast-1": { "AMI": "ami-c9562fc8" }, "sa-east-1": { "AMI": "ami-215dff3c" }, "eu-central-1": { "AMI" : "ami-a03503bd" } } }, "Resources": { "LogRole": { "Type": "AWS::IAM::Role", "Properties": { "AssumeRolePolicyDocument": { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Principal": { "Service": [ "ec2.amazonaws.com" ] }, "Action": [ "sts:AssumeRole" ] } ] }, "Path": "/", "Policies": [ { "PolicyName": "LogRolePolicy", "PolicyDocument": { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Action": [ "logs:Create*", "logs:PutLogEvents", "s3:GetObject" ], "Resource": [ "arn:aws:logs:*:*:*", "arn:aws:s3:::*" ] } ] } } ] } }, "LogRoleInstanceProfile": { "Type": "AWS::IAM::InstanceProfile", "Properties": { "Path": "/", "Roles": [ { "Ref": "LogRole" } ] } }, "WebServerSecurityGroup": { "Type": "AWS::EC2::SecurityGroup", "Properties": { "GroupDescription": "Enable HTTP access via port 80 and SSH access via port 22", "SecurityGroupIngress" : [ {"IpProtocol" : "tcp", "FromPort" : "80", "ToPort" : "80", "CidrIp" : "0.0.0.0/0"}, {"IpProtocol" : "tcp", "FromPort" : "22", "ToPort" : "22", "CidrIp" : { "Ref" : "SSHLocation"}} ] } }, "WebServerHost": { "Type": "AWS::EC2::Instance", "Metadata": { "Comment": "Install a simple PHP application", "AWS::CloudFormation::Init": { "config": { "packages": { "yum": { "httpd": [], "php": [] } }, "files": { "/tmp/cwlogs/apacheaccess.conf": { "content": { "Fn::Join": [ "", [ "[general]\n", "state_file= /var/awslogs/agent-state\n", "[/var/log/httpd/access_log]\n", "file = /var/log/httpd/access_log\n", "log_group_name = ", {"Ref": "WebServerLogGroup"}, "\n", "log_stream_name = {instance_id}/apache.log\n", "datetime_format = %d/%b/%Y:%H:%M:%S" ] ] }, "mode": "000400", "owner": "apache", "group": "apache" }, "/var/www/html/index.php": { "content": { "Fn::Join": [ "", [ "<?php\n", "echo '<h1>AWS CloudFormation sample PHP application</h1>';\n", "?>\n" ] ] }, "mode": "000644", "owner": "apache", "group": "apache" }, "/etc/cfn/cfn-hup.conf": { "content": { "Fn::Join": [ "", [ "[main]\n", "stack=", { "Ref": "AWS::StackId" }, "\n", "region=", { "Ref": "AWS::Region" }, "\n" ] ] }, "mode": "000400", "owner": "root", "group": "root" }, "/etc/cfn/hooks.d/cfn-auto-reloader.conf": { "content": { "Fn::Join": [ "", [ "[cfn-auto-reloader-hook]\n", "triggers=post.update\n", "path=Resources.WebServerHost.Metadata.AWS::CloudFormation::Init\n", "action=/opt/aws/bin/cfn-init -s ", { "Ref": "AWS::StackId" }, " -r WebServerHost ", " --region ", { "Ref": "AWS::Region" }, "\n", "runas=root\n" ] ] } } }, "services": { "sysvinit": { "httpd": { "enabled": "true", "ensureRunning": "true" }, "sendmail": { "enabled": "false", "ensureRunning": "false" } } } } } }, "CreationPolicy" : { "ResourceSignal" : { "Timeout" : "PT5M" } }, "Properties": { "ImageId": { "Fn::FindInMap": [ "RegionMap", { "Ref": "AWS::Region" }, "AMI" ] }, "KeyName": { "Ref": "KeyName" }, "InstanceType": "t1.micro", "SecurityGroups": [ { "Ref": "WebServerSecurityGroup" } ], "IamInstanceProfile": { "Ref": "LogRoleInstanceProfile" }, "UserData": { "Fn::Base64": { "Fn::Join": [ "", [ "#!/bin/bash -xe\n", "# Get the latest CloudFormation package\n", "yum install -y aws-cfn-bootstrap\n", "# Start cfn-init\n", "/opt/aws/bin/cfn-init -s ", { "Ref": "AWS::StackId" }, " -r WebServerHost ", " --region ", { "Ref": "AWS::Region" }, " || error_exit 'Failed to run cfn-init'\n", "# Start up the cfn-hup daemon to listen for changes to the EC2 instance metadata\n", "/opt/aws/bin/cfn-hup || error_exit 'Failed to start cfn-hup'\n", "# Get the CloudWatch Logs agent\n", "wget https://s3.amazonaws.com/aws-cloudwatch/downloads/latest/awslogs-agent-setup.py\n", "# Install the CloudWatch Logs agent\n", "python awslogs-agent-setup.py -n -r ", { "Ref" : "AWS::Region" }, " -c /tmp/cwlogs/apacheaccess.conf || error_exit 'Failed to run CloudWatch Logs agent setup'\n", "# All done so signal success\n", "/opt/aws/bin/cfn-signal -e $? ", " --stack ", { "Ref" : "AWS::StackName" }, " --resource WebServerHost ", " --region ", { "Ref" : "AWS::Region" }, "\n" ] ] } } } }, "WebServerLogGroup": { "Type": "AWS::Logs::LogGroup", "Properties": { "RetentionInDays": 7 } }, "404MetricFilter": { "Type": "AWS::Logs::MetricFilter", "Properties": { "LogGroupName": { "Ref": "WebServerLogGroup" }, "FilterPattern": "[ip, identity, user_id, timestamp, request, status_code = 404, size, ...]", "MetricTransformations": [ { "MetricValue": "1", "MetricNamespace": "test/404s", "MetricName": "test404Count" } ] } }, "BytesTransferredMetricFilter": { "Type": "AWS::Logs::MetricFilter", "Properties": { "LogGroupName": { "Ref": "WebServerLogGroup" }, "FilterPattern": "[ip, identity, user_id, timestamp, request, status_code, size, ...]", "MetricTransformations": [ { "MetricValue": "$size", "MetricNamespace": "test/BytesTransferred", "MetricName": "testBytesTransferred" } ] } }, "404Alarm": { "Type": "AWS::CloudWatch::Alarm", "Properties": { "AlarmDescription": "The number of 404s is greater than 2 over 2 minutes", "MetricName": "test404Count", "Namespace": "test/404s", "Statistic": "Sum", "Period": "60", "EvaluationPeriods": "2", "Threshold": "2", "AlarmActions": [ { "Ref": "AlarmNotificationTopic" } ], "ComparisonOperator": "GreaterThanThreshold" } }, "BandwidthAlarm": { "Type": "AWS::CloudWatch::Alarm", "Properties": { "AlarmDescription": "The average volume of traffic is greater 3500 KB over 10 minutes", "MetricName": "testBytesTransferred", "Namespace": "test/BytesTransferred", "Statistic": "Average", "Period": "300", "EvaluationPeriods": "2", "Threshold": "3500", "AlarmActions": [ { "Ref": "AlarmNotificationTopic" } ], "ComparisonOperator": "GreaterThanThreshold" } }, "AlarmNotificationTopic": { "Type": "AWS::SNS::Topic", "Properties": { "Subscription": [ { "Endpoint": { "Ref": "OperatorEmail" }, "Protocol": "email" } ] } } }, "Outputs": { "InstanceId": { "Description": "The instance ID of the web server", "Value": { "Ref": "WebServerHost" } }, "WebsiteURL" : { "Value" : { "Fn::Join" : ["", ["http://", { "Fn::GetAtt" : [ "WebServerHost", "PublicDnsName" ]}]] }, "Description" : "URL for newly created LAMP stack" }, "PublicIP": { "Description": "Public IP address of the web server", "Value": { "Fn::GetAtt": [ "WebServerHost", "PublicIp" ] } }, "CloudWatchLogGroupName": { "Description": "The name of the CloudWatch log group", "Value": { "Ref": "WebServerLogGroup" } } } }

YAML

Copy
AWSTemplateFormatVersion: '2010-09-09' Description: AWS CloudFormation Sample Template for CloudWatch Logs. Parameters: KeyName: Description: Name of an existing EC2 KeyPair to enable SSH access to the instances Type: AWS::EC2::KeyPair::KeyName ConstraintDescription: must be the name of an existing EC2 KeyPair. SSHLocation: Description: The IP address range that can be used to SSH to the EC2 instances Type: String MinLength: '9' MaxLength: '18' Default: 0.0.0.0/0 AllowedPattern: "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})" ConstraintDescription: must be a valid IP CIDR range of the form x.x.x.x/x. OperatorEmail: Description: Email address to notify if there are any scaling operations Type: String Mappings: RegionMap: us-east-1: AMI: ami-fb8e9292 us-west-1: AMI: ami-7aba833f us-west-2: AMI: ami-043a5034 eu-west-1: AMI: ami-2918e35e ap-southeast-1: AMI: ami-b40d5ee6 ap-southeast-2: AMI: ami-3b4bd301 ap-northeast-1: AMI: ami-c9562fc8 sa-east-1: AMI: ami-215dff3c eu-central-1: AMI: ami-a03503bd Resources: LogRole: Type: AWS::IAM::Role Properties: AssumeRolePolicyDocument: Version: '2012-10-17' Statement: - Effect: Allow Principal: Service: - ec2.amazonaws.com Action: - sts:AssumeRole Path: "/" Policies: - PolicyName: LogRolePolicy PolicyDocument: Version: '2012-10-17' Statement: - Effect: Allow Action: - logs:Create* - logs:PutLogEvents - s3:GetObject Resource: - arn:aws:logs:*:*:* - arn:aws:s3:::* LogRoleInstanceProfile: Type: AWS::IAM::InstanceProfile Properties: Path: "/" Roles: - Ref: LogRole WebServerSecurityGroup: Type: AWS::EC2::SecurityGroup Properties: GroupDescription: Enable HTTP access via port 80 and SSH access via port 22 SecurityGroupIngress: - IpProtocol: tcp FromPort: '80' ToPort: '80' CidrIp: 0.0.0.0/0 - IpProtocol: tcp FromPort: '22' ToPort: '22' CidrIp: Ref: SSHLocation WebServerHost: Type: AWS::EC2::Instance Metadata: Comment: Install a simple PHP application AWS::CloudFormation::Init: config: packages: yum: httpd: [] php: [] files: "/tmp/cwlogs/apacheaccess.conf": content: !Sub | [general] state_file= /var/awslogs/agent-state [/var/log/httpd/access_log] file = /var/log/httpd/access_log log_group_name = ${WebServerLogGroup} log_stream_name = {instance_id}/apache.log datetime_format = %d/%b/%Y:%H:%M:%S mode: '000400' owner: apache group: apache "/var/www/html/index.php": content: !Sub | "<?php" "echo '<h1>AWS CloudFormation sample PHP application</h1>';" "?>" mode: '000644' owner: apache group: apache "/etc/cfn/cfn-hup.conf": content: !Sub | [main] stack= ${AWS::StackId} region=${AWS::Region} mode: "000400" owner: "root" group: "root" "/etc/cfn/hooks.d/cfn-auto-reloader.conf": content: !Sub | [cfn-auto-reloader-hook] triggers=post.update path=Resources.WebServerHost.Metadata.AWS::CloudFormation::Init action=/opt/aws/bin/cfn-init -v --stack ${AWS::StackName} --resource WebServerHost --region ${AWS::Region} mode: "000400" owner: "root" group: "root" services: sysvinit: httpd: enabled: 'true' ensureRunning: 'true' sendmail: enabled: 'false' ensureRunning: 'false' CreationPolicy: ResourceSignal: Timeout: PT5M Properties: ImageId: Fn::FindInMap: - RegionMap - Ref: AWS::Region - AMI KeyName: Ref: KeyName InstanceType: t1.micro SecurityGroups: - Ref: WebServerSecurityGroup IamInstanceProfile: Ref: LogRoleInstanceProfile UserData: "Fn::Base64": !Sub | #!/bin/bash -xe # Get the latest CloudFormation package yum update -y aws-cfn-bootstrap # Start cfn-init /opt/aws/bin/cfn-init -s ${AWS::StackId} -r WebServerHost --region ${AWS::Region} || error_exit 'Failed to run cfn-init' # Start up the cfn-hup daemon to listen for changes to the EC2 instance metadata /opt/aws/bin/cfn-hup || error_exit 'Failed to start cfn-hup' # Get the CloudWatch Logs agent wget https://s3.amazonaws.com/aws-cloudwatch/downloads/latest/awslogs-agent-setup.py # Install the CloudWatch Logs agent python awslogs-agent-setup.py -n -r ${AWS::Region} -c /tmp/cwlogs/apacheaccess.conf || error_exit 'Failed to run CloudWatch Logs agent setup' # All done so signal success /opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackId} --resource WebServerHost --region ${AWS::Region} WebServerLogGroup: Type: AWS::Logs::LogGroup Properties: RetentionInDays: 7 404MetricFilter: Type: AWS::Logs::MetricFilter Properties: LogGroupName: Ref: WebServerLogGroup FilterPattern: "[ip, identity, user_id, timestamp, request, status_code = 404, size, ...]" MetricTransformations: - MetricValue: '1' MetricNamespace: test/404s MetricName: test404Count BytesTransferredMetricFilter: Type: AWS::Logs::MetricFilter Properties: LogGroupName: Ref: WebServerLogGroup FilterPattern: "[ip, identity, user_id, timestamp, request, status_code, size, ...]" MetricTransformations: - MetricValue: "$size" MetricNamespace: test/BytesTransferred MetricName: testBytesTransferred 404Alarm: Type: AWS::CloudWatch::Alarm Properties: AlarmDescription: The number of 404s is greater than 2 over 2 minutes MetricName: test404Count Namespace: test/404s Statistic: Sum Period: '60' EvaluationPeriods: '2' Threshold: '2' AlarmActions: - Ref: AlarmNotificationTopic ComparisonOperator: GreaterThanThreshold BandwidthAlarm: Type: AWS::CloudWatch::Alarm Properties: AlarmDescription: The average volume of traffic is greater 3500 KB over 10 minutes MetricName: testBytesTransferred Namespace: test/BytesTransferred Statistic: Average Period: '300' EvaluationPeriods: '2' Threshold: '3500' AlarmActions: - Ref: AlarmNotificationTopic ComparisonOperator: GreaterThanThreshold AlarmNotificationTopic: Type: AWS::SNS::Topic Properties: Subscription: - Endpoint: Ref: OperatorEmail Protocol: email Outputs: InstanceId: Description: The instance ID of the web server Value: Ref: WebServerHost WebsiteURL: Value: !Sub 'http://${WebServerHost.PublicDnsName}' Description: URL for newly created LAMP stack PublicIP: Description: Public IP address of the web server Value: !GetAtt WebServerHost.PublicIp CloudWatchLogGroupName: Description: The name of the CloudWatch log group Value: !Ref WebServerLogGroup

See Also

For more information about CloudWatch Logs resources, see AWS::Logs::LogGroup or AWS::Logs::MetricFilter.