Container Insights performance log events for Amazon EKS and Kubernetes
The following are examples of the performance log events that Container Insights collects from Amazon EKS and Kubernetes clusters.
Type: Node
{ "AutoScalingGroupName": "eksctl-myCICluster-nodegroup-standard-workers-NodeGroup-1174PV2WHZAYU", "CloudWatchMetrics": [ { "Metrics": [ { "Unit": "Percent", "Name": "node_cpu_utilization" }, { "Unit": "Percent", "Name": "node_memory_utilization" }, { "Unit": "Bytes/Second", "Name": "node_network_total_bytes" }, { "Unit": "Percent", "Name": "node_cpu_reserved_capacity" }, { "Unit": "Percent", "Name": "node_memory_reserved_capacity" }, { "Unit": "Count", "Name": "node_number_of_running_pods" }, { "Unit": "Count", "Name": "node_number_of_running_containers" } ], "Dimensions": [ [ "NodeName", "InstanceId", "ClusterName" ] ], "Namespace": "ContainerInsights" }, { "Metrics": [ { "Unit": "Percent", "Name": "node_cpu_utilization" }, { "Unit": "Percent", "Name": "node_memory_utilization" }, { "Unit": "Bytes/Second", "Name": "node_network_total_bytes" }, { "Unit": "Percent", "Name": "node_cpu_reserved_capacity" }, { "Unit": "Percent", "Name": "node_memory_reserved_capacity" }, { "Unit": "Count", "Name": "node_number_of_running_pods" }, { "Unit": "Count", "Name": "node_number_of_running_containers" }, { "Name": "node_cpu_usage_total" }, { "Name": "node_cpu_limit" }, { "Unit": "Bytes", "Name": "node_memory_working_set" }, { "Unit": "Bytes", "Name": "node_memory_limit" } ], "Dimensions": [ [ "ClusterName" ] ], "Namespace": "ContainerInsights" } ], "ClusterName": "myCICluster", "InstanceId": "i-1234567890123456", "InstanceType": "t3.xlarge", "NodeName": "ip-192-0-2-0.us-west-2.compute.internal", "Sources": [ "cadvisor", "/proc", "pod", "calculated" ], "Timestamp": "1567096682364", "Type": "Node", "Version": "0", "kubernetes": { "host": "ip-192-168-75-26.us-west-2.compute.internal" }, "node_cpu_limit": 4000, "node_cpu_request": 1130, "node_cpu_reserved_capacity": 28.249999999999996, "node_cpu_usage_system": 33.794636630852764, "node_cpu_usage_total": 136.47852169244098, "node_cpu_usage_user": 71.67075111567326, "node_cpu_utilization": 3.4119630423110245, "node_memory_cache": 3103297536, "node_memory_failcnt": 0, "node_memory_hierarchical_pgfault": 0, "node_memory_hierarchical_pgmajfault": 0, "node_memory_limit": 16624865280, "node_memory_mapped_file": 406646784, "node_memory_max_usage": 4230746112, "node_memory_pgfault": 0, "node_memory_pgmajfault": 0, "node_memory_request": 1115684864, "node_memory_reserved_capacity": 6.7109407818311055, "node_memory_rss": 798146560, "node_memory_swap": 0, "node_memory_usage": 3901444096, "node_memory_utilization": 6.601302600149552, "node_memory_working_set": 1097457664, "node_network_rx_bytes": 35918.392817386324, "node_network_rx_dropped": 0, "node_network_rx_errors": 0, "node_network_rx_packets": 157.67565245448117, "node_network_total_bytes": 68264.20276554905, "node_network_tx_bytes": 32345.80994816272, "node_network_tx_dropped": 0, "node_network_tx_errors": 0, "node_network_tx_packets": 154.21455923431654, "node_number_of_running_containers": 16, "node_number_of_running_pods": 13 }
Type: NodeFS
{ "AutoScalingGroupName": "eksctl-myCICluster-nodegroup-standard-workers-NodeGroup-1174PV2WHZAYU", "CloudWatchMetrics": [ { "Metrics": [ { "Unit": "Percent", "Name": "node_filesystem_utilization" } ], "Dimensions": [ [ "NodeName", "InstanceId", "ClusterName" ], [ "ClusterName" ] ], "Namespace": "ContainerInsights" } ], "ClusterName": "myCICluster", "EBSVolumeId": "aws://us-west-2b/vol-0a53108976d4a2fda", "InstanceId": "i-1234567890123456", "InstanceType": "t3.xlarge", "NodeName": "ip-192-0-2-0.us-west-2.compute.internal", "Sources": [ "cadvisor", "calculated" ], "Timestamp": "1567097939726", "Type": "NodeFS", "Version": "0", "device": "/dev/nvme0n1p1", "fstype": "vfs", "kubernetes": { "host": "ip-192-168-75-26.us-west-2.compute.internal" }, "node_filesystem_available": 17298395136, "node_filesystem_capacity": 21462233088, "node_filesystem_inodes": 10484720, "node_filesystem_inodes_free": 10367158, "node_filesystem_usage": 4163837952, "node_filesystem_utilization": 19.400767547940255 }
Type: NodeDiskIO
{ "AutoScalingGroupName": "eksctl-myCICluster-nodegroup-standard-workers-NodeGroup-1174PV2WHZAYU", "ClusterName": "myCICluster", "EBSVolumeId": "aws://us-west-2b/vol-0a53108976d4a2fda", "InstanceId": "i-1234567890123456", "InstanceType": "t3.xlarge", "NodeName": "ip-192-0-2-0.us-west-2.compute.internal", "Sources": [ "cadvisor" ], "Timestamp": "1567096928131", "Type": "NodeDiskIO", "Version": "0", "device": "/dev/nvme0n1", "kubernetes": { "host": "ip-192-168-75-26.us-west-2.compute.internal" }, "node_diskio_io_service_bytes_async": 9750.505814277016, "node_diskio_io_service_bytes_read": 0, "node_diskio_io_service_bytes_sync": 230.6174506688036, "node_diskio_io_service_bytes_total": 9981.123264945818, "node_diskio_io_service_bytes_write": 9981.123264945818, "node_diskio_io_serviced_async": 1.153087253344018, "node_diskio_io_serviced_read": 0, "node_diskio_io_serviced_sync": 0.03603397666700056, "node_diskio_io_serviced_total": 1.1891212300110185, "node_diskio_io_serviced_write": 1.1891212300110185 }
Type: NodeNet
{ "AutoScalingGroupName": "eksctl-myCICluster-nodegroup-standard-workers-NodeGroup-1174PV2WHZAYU", "ClusterName": "myCICluster", "InstanceId": "i-1234567890123456", "InstanceType": "t3.xlarge", "NodeName": "ip-192-0-2-0.us-west-2.compute.internal", "Sources": [ "cadvisor", "calculated" ], "Timestamp": "1567096928131", "Type": "NodeNet", "Version": "0", "interface": "eni972f6bfa9a0", "kubernetes": { "host": "ip-192-168-75-26.us-west-2.compute.internal" }, "node_interface_network_rx_bytes": 3163.008420864309, "node_interface_network_rx_dropped": 0, "node_interface_network_rx_errors": 0, "node_interface_network_rx_packets": 16.575629266820258, "node_interface_network_total_bytes": 3518.3935157426017, "node_interface_network_tx_bytes": 355.385094878293, "node_interface_network_tx_dropped": 0, "node_interface_network_tx_errors": 0, "node_interface_network_tx_packets": 3.9997714100370625 }
Type: Pod
{ "AutoScalingGroupName": "eksctl-myCICluster-nodegroup-standard-workers-NodeGroup-1174PV2WHZAYU", "CloudWatchMetrics": [ { "Metrics": [ { "Unit": "Percent", "Name": "pod_cpu_utilization" }, { "Unit": "Percent", "Name": "pod_memory_utilization" }, { "Unit": "Bytes/Second", "Name": "pod_network_rx_bytes" }, { "Unit": "Bytes/Second", "Name": "pod_network_tx_bytes" }, { "Unit": "Percent", "Name": "pod_cpu_utilization_over_pod_limit" }, { "Unit": "Percent", "Name": "pod_memory_utilization_over_pod_limit" } ], "Dimensions": [ [ "PodName", "Namespace", "ClusterName" ], [ "Service", "Namespace", "ClusterName" ], [ "Namespace", "ClusterName" ], [ "ClusterName" ] ], "Namespace": "ContainerInsights" }, { "Metrics": [ { "Unit": "Percent", "Name": "pod_cpu_reserved_capacity" }, { "Unit": "Percent", "Name": "pod_memory_reserved_capacity" } ], "Dimensions": [ [ "PodName", "Namespace", "ClusterName" ], [ "ClusterName" ] ], "Namespace": "ContainerInsights" }, { "Metrics": [ { "Unit": "Count", "Name": "pod_number_of_container_restarts" } ], "Dimensions": [ [ "PodName", "Namespace", "ClusterName" ] ], "Namespace": "ContainerInsights" } ], "ClusterName": "myCICluster", "InstanceId": "i-1234567890123456", "InstanceType": "t3.xlarge", "Namespace": "amazon-cloudwatch", "NodeName": "ip-192-0-2-0.us-west-2.compute.internal", "PodName": "cloudwatch-agent-statsd", "Service": "cloudwatch-agent-statsd", "Sources": [ "cadvisor", "pod", "calculated" ], "Timestamp": "1567097351092", "Type": "Pod", "Version": "0", "kubernetes": { "host": "ip-192-168-75-26.us-west-2.compute.internal", "labels": { "app": "cloudwatch-agent-statsd", "pod-template-hash": "df44f855f" }, "namespace_name": "amazon-cloudwatch", "pod_id": "2f4ff5ac-c813-11e9-a31d-06e9dde32928", "pod_name": "cloudwatch-agent-statsd-df44f855f-ts4q2", "pod_owners": [ { "owner_kind": "Deployment", "owner_name": "cloudwatch-agent-statsd" } ], "service_name": "cloudwatch-agent-statsd" }, "pod_cpu_limit": 200, "pod_cpu_request": 200, "pod_cpu_reserved_capacity": 5, "pod_cpu_usage_system": 1.4504841104992765, "pod_cpu_usage_total": 5.817016867430125, "pod_cpu_usage_user": 1.1281543081661038, "pod_cpu_utilization": 0.14542542168575312, "pod_cpu_utilization_over_pod_limit": 2.9085084337150624, "pod_memory_cache": 8192, "pod_memory_failcnt": 0, "pod_memory_hierarchical_pgfault": 0, "pod_memory_hierarchical_pgmajfault": 0, "pod_memory_limit": 104857600, "pod_memory_mapped_file": 0, "pod_memory_max_usage": 25268224, "pod_memory_pgfault": 0, "pod_memory_pgmajfault": 0, "pod_memory_request": 104857600, "pod_memory_reserved_capacity": 0.6307275170893897, "pod_memory_rss": 22777856, "pod_memory_swap": 0, "pod_memory_usage": 25141248, "pod_memory_utilization": 0.10988455961791709, "pod_memory_utilization_over_pod_limit": 17.421875, "pod_memory_working_set": 18268160, "pod_network_rx_bytes": 9880.697124714186, "pod_network_rx_dropped": 0, "pod_network_rx_errors": 0, "pod_network_rx_packets": 107.80005532263283, "pod_network_total_bytes": 10158.829201483635, "pod_network_tx_bytes": 278.13207676944796, "pod_network_tx_dropped": 0, "pod_network_tx_errors": 0, "pod_network_tx_packets": 1.146027574644318, "pod_number_of_container_restarts": 0, "pod_number_of_containers": 1, "pod_number_of_running_containers": 1, "pod_status": "Running" }
Type: PodNet
{ "AutoScalingGroupName": "eksctl-myCICluster-nodegroup-standard-workers-NodeGroup-1174PV2WHZAYU", "ClusterName": "myCICluster", "InstanceId": "i-1234567890123456", "InstanceType": "t3.xlarge", "Namespace": "amazon-cloudwatch", "NodeName": "ip-192-0-2-0.us-west-2.compute.internal", "PodName": "cloudwatch-agent-statsd", "Service": "cloudwatch-agent-statsd", "Sources": [ "cadvisor", "calculated" ], "Timestamp": "1567097351092", "Type": "PodNet", "Version": "0", "interface": "eth0", "kubernetes": { "host": "ip-192-168-75-26.us-west-2.compute.internal", "labels": { "app": "cloudwatch-agent-statsd", "pod-template-hash": "df44f855f" }, "namespace_name": "amazon-cloudwatch", "pod_id": "2f4ff5ac-c813-11e9-a31d-06e9dde32928", "pod_name": "cloudwatch-agent-statsd-df44f855f-ts4q2", "pod_owners": [ { "owner_kind": "Deployment", "owner_name": "cloudwatch-agent-statsd" } ], "service_name": "cloudwatch-agent-statsd" }, "pod_interface_network_rx_bytes": 9880.697124714186, "pod_interface_network_rx_dropped": 0, "pod_interface_network_rx_errors": 0, "pod_interface_network_rx_packets": 107.80005532263283, "pod_interface_network_total_bytes": 10158.829201483635, "pod_interface_network_tx_bytes": 278.13207676944796, "pod_interface_network_tx_dropped": 0, "pod_interface_network_tx_errors": 0, "pod_interface_network_tx_packets": 1.146027574644318 }
Type: Container
{ "AutoScalingGroupName": "eksctl-myCICluster-nodegroup-standard-workers-NodeGroup-sample", "ClusterName": "myCICluster", "InstanceId": "i-1234567890123456", "InstanceType": "t3.xlarge", "Namespace": "amazon-cloudwatch", "NodeName": "ip-192-0-2-0.us-west-2.compute.internal", "PodName": "cloudwatch-agent-statsd", "Service": "cloudwatch-agent-statsd", "Sources": [ "cadvisor", "pod", "calculated" ], "Timestamp": "1567097399912", "Type": "Container", "Version": "0", "container_cpu_limit": 200, "container_cpu_request": 200, "container_cpu_usage_system": 1.87958283771964, "container_cpu_usage_total": 6.159993652997942, "container_cpu_usage_user": 1.6707403001952357, "container_cpu_utilization": 0.15399984132494854, "container_memory_cache": 8192, "container_memory_failcnt": 0, "container_memory_hierarchical_pgfault": 0, "container_memory_hierarchical_pgmajfault": 0, "container_memory_limit": 104857600, "container_memory_mapped_file": 0, "container_memory_max_usage": 24580096, "container_memory_pgfault": 0, "container_memory_pgmajfault": 0, "container_memory_request": 104857600, "container_memory_rss": 22736896, "container_memory_swap": 0, "container_memory_usage": 24453120, "container_memory_utilization": 0.10574541028701798, "container_memory_working_set": 17580032, "container_status": "Running", "kubernetes": { "container_name": "cloudwatch-agent", "docker": { "container_id": "8967b6b37da239dfad197c9fdea3e5dfd35a8a759ec86e2e4c3f7b401e232706" }, "host": "ip-192-168-75-26.us-west-2.compute.internal", "labels": { "app": "cloudwatch-agent-statsd", "pod-template-hash": "df44f855f" }, "namespace_name": "amazon-cloudwatch", "pod_id": "2f4ff5ac-c813-11e9-a31d-06e9dde32928", "pod_name": "cloudwatch-agent-statsd-df44f855f-ts4q2", "pod_owners": [ { "owner_kind": "Deployment", "owner_name": "cloudwatch-agent-statsd" } ], "service_name": "cloudwatch-agent-statsd" }, "number_of_container_restarts": 0 }
Type: ContainerFS
{ "AutoScalingGroupName": "eksctl-myCICluster-nodegroup-standard-workers-NodeGroup-1174PV2WHZAYU", "ClusterName": "myCICluster", "EBSVolumeId": "aws://us-west-2b/vol-0a53108976d4a2fda", "InstanceId": "i-1234567890123456", "InstanceType": "t3.xlarge", "Namespace": "amazon-cloudwatch", "NodeName": "ip-192-0-2-0.us-west-2.compute.internal", "PodName": "cloudwatch-agent-statsd", "Service": "cloudwatch-agent-statsd", "Sources": [ "cadvisor", "calculated" ], "Timestamp": "1567097399912", "Type": "ContainerFS", "Version": "0", "device": "/dev/nvme0n1p1", "fstype": "vfs", "kubernetes": { "container_name": "cloudwatch-agent", "docker": { "container_id": "8967b6b37da239dfad197c9fdea3e5dfd35a8a759ec86e2e4c3f7b401e232706" }, "host": "ip-192-168-75-26.us-west-2.compute.internal", "labels": { "app": "cloudwatch-agent-statsd", "pod-template-hash": "df44f855f" }, "namespace_name": "amazon-cloudwatch", "pod_id": "2f4ff5ac-c813-11e9-a31d-06e9dde32928", "pod_name": "cloudwatch-agent-statsd-df44f855f-ts4q2", "pod_owners": [ { "owner_kind": "Deployment", "owner_name": "cloudwatch-agent-statsd" } ], "service_name": "cloudwatch-agent-statsd" } }
Type: Cluster
{ "CloudWatchMetrics": [ { "Metrics": [ { "Unit": "Count", "Name": "cluster_node_count" }, { "Unit": "Count", "Name": "cluster_failed_node_count" } ], "Dimensions": [ [ "ClusterName" ] ], "Namespace": "ContainerInsights" } ], "ClusterName": "myCICluster", "Sources": [ "apiserver" ], "Timestamp": "1567097534160", "Type": "Cluster", "Version": "0", "cluster_failed_node_count": 0, "cluster_node_count": 3 }
Type: ClusterService
{ "CloudWatchMetrics": [ { "Metrics": [ { "Unit": "Count", "Name": "service_number_of_running_pods" } ], "Dimensions": [ [ "Service", "Namespace", "ClusterName" ], [ "ClusterName" ] ], "Namespace": "ContainerInsights" } ], "ClusterName": "myCICluster", "Namespace": "amazon-cloudwatch", "Service": "cloudwatch-agent-statsd", "Sources": [ "apiserver" ], "Timestamp": "1567097534160", "Type": "ClusterService", "Version": "0", "kubernetes": { "namespace_name": "amazon-cloudwatch", "service_name": "cloudwatch-agent-statsd" }, "service_number_of_running_pods": 1 }
Type: ClusterNamespace
{ "CloudWatchMetrics": [ { "Metrics": [ { "Unit": "Count", "Name": "namespace_number_of_running_pods" } ], "Dimensions": [ [ "Namespace", "ClusterName" ], [ "ClusterName" ] ], "Namespace": "ContainerInsights" } ], "ClusterName": "myCICluster", "Namespace": "amazon-cloudwatch", "Sources": [ "apiserver" ], "Timestamp": "1567097594160", "Type": "ClusterNamespace", "Version": "0", "kubernetes": { "namespace_name": "amazon-cloudwatch" }, "namespace_number_of_running_pods": 7 }
