{ "tags": [], "title": "1 - Deployment Performance & Health", "widgets": [ { "layout": { "x": 0, "y": 0, "width": 12, "height": 4 }, "definition": { "type": "group", "title": "At a Glance", "widgets": [ { "layout": { "x": 0, "y": 0, "width": 3, "height": 2 }, "definition": { "type": "query_value", "title": "Client Success Rate (non-5xx responses)", "requests": [ { "queries": [ { "name": "query1", "query": "sum:istio.mesh.request.count{reporter:source,destination_workload_namespace:$namespace.value,destination_workload:$workload.value,!response_code:5*,account:$awsaccount.value,$cluster} by {cluster}.as_rate()", "aggregator": "last", "data_source": "metrics" }, { "name": "query2", "query": "sum:istio.mesh.request.count{reporter:source,destination_workload_namespace:$namespace.value,destination_workload:$workload.value,account:$awsaccount.value,$cluster} by {cluster}.as_rate()", "aggregator": "last", "data_source": "metrics" } ], "formulas": [ { "formula": "(query1 / query2) * 100" } ], "response_format": "scalar", "conditional_formats": [ { "value": 0.9, "palette": "white_on_red", "comparator": ">=" }, { "value": 0.95, "palette": "white_on_red", "comparator": ">=" }, { "value": 0.99, "palette": "white_on_gray", "comparator": ">=" }, { "value": 0.9999, "palette": "white_on_green", "comparator": ">=" } ] } ], "autoscale": true, "precision": 2, "text_align": "center", "custom_unit": "%" } }, { "layout": { "x": 3, "y": 0, "width": 9, "height": 2 }, "definition": { "type": "query_value", "title": "Request Latency", "requests": [ { "queries": [ { "name": "query1", "query": "p50:istio.mesh.request.duration.milliseconds{reporter:source,destination_workload:$workload.value,destination_workload_namespace:$namespace.value,account:$awsaccount.value,$cluster}", "aggregator": "last", "data_source": "metrics" }, { "name": "query2", "query": "p90:istio.mesh.request.duration.milliseconds{reporter:source,destination_workload:$workload.value,destination_workload_namespace:$namespace.value,account:$awsaccount.value,$cluster}", "aggregator": "last", "data_source": "metrics" }, { "name": "query3", "query": "p95:istio.mesh.request.duration.milliseconds{reporter:source,destination_workload:$workload.value,destination_workload_namespace:$namespace.value,account:$awsaccount.value,$cluster}", "aggregator": "last", "data_source": "metrics" }, { "name": "query4", "query": "p99:istio.mesh.request.duration.milliseconds{reporter:source,destination_workload:$workload.value,destination_workload_namespace:$namespace.value,account:$awsaccount.value,$cluster}", "aggregator": "last", "data_source": "metrics" } ], "formulas": [ { "alias": "P50", "formula": "query1 / 1000" } ], "response_format": "scalar", "conditional_formats": [ { "value": 0.2, "palette": "white_on_red", "comparator": ">=" }, { "value": 0.5, "palette": "white_on_red", "comparator": ">=" } ] } ], "autoscale": true, "precision": 2, "text_align": "center", "custom_unit": "s", "timeseries_background": { "type": "area" } } }, { "layout": { "x": 0, "y": 2, "width": 3, "height": 2 }, "definition": { "type": "query_value", "title": "Client Request Volume", "requests": [ { "queries": [ { "name": "query1", "query": "sum:istio.mesh.request.count{reporter:source,destination_workload_namespace:$namespace.value,destination_workload:$workload.value,account:$awsaccount.value,$cluster} by {cluster}.as_rate()", "aggregator": "last", "data_source": "metrics" } ], "formulas": [ { "formula": "query1" } ], "response_format": "scalar" } ], "autoscale": true, "precision": 2, "text_align": "center", "custom_unit": "req/s", "timeseries_background": { "type": "area" } } }, { "layout": { "x": 3, "y": 2, "width": 3, "height": 2 }, "definition": { "type": "query_value", "title": "Deployment Healthy POD Replicas", "requests": [ { "queries": [ { "name": "query1", "query": "sum:kubernetes_state.deployment.replicas_available{$namespace,kube_deployment:$workload.value,account:$awsaccount.value,$cluster} by {cluster}", "aggregator": "last", "data_source": "metrics" } ], "formulas": [ { "formula": "query1" } ], "response_format": "scalar" } ], "autoscale": true, "precision": 2, "text_align": "center", "timeseries_background": { "type": "area" } } }, { "layout": { "x": 6, "y": 2, "width": 3, "height": 2 }, "definition": { "type": "query_value", "title": "CPU Throttling Time", "requests": [ { "queries": [ { "name": "query1", "query": "avg:kubernetes.cpu.cfs.throttled.seconds{$namespace,pod_name:$workload.value-*,account:$awsaccount.value,$cluster} by {cluster}.as_rate()", "aggregator": "last", "data_source": "metrics" } ], "formulas": [ { "formula": "query1" } ], "response_format": "scalar", "conditional_formats": [ { "value": 0.0001, "palette": "white_on_gray", "comparator": ">=" }, { "value": 0.1, "palette": "white_on_red", "comparator": ">=" }, { "value": 1, "palette": "white_on_red", "comparator": ">=" } ] } ], "autoscale": true, "precision": 2, "text_align": "center", "custom_unit": "s" } }, { "layout": { "x": 9, "y": 2, "width": 3, "height": 2 }, "definition": { "type": "query_value", "title": "Current Container Image Tag", "requests": [ { "queries": [ { "name": "query1", "query": "avg:kube_pod_container_info{$namespace AND pod:$workload.value-* AND container NOT IN (pod,istio-proxy) AND account:$awsaccount.value AND $cluster} by {cluster}", "aggregator": "last", "data_source": "metrics" } ], "formulas": [ { "formula": "query1" } ], "response_format": "scalar" } ], "autoscale": true, "precision": 2, "text_align": "center" } } ], "show_title": true, "layout_type": "ordered" } }, { "layout": { "x": 0, "y": 4, "width": 12, "height": 6 }, "definition": { "type": "group", "title": "Requests, Error, Duration (RED) - Only usable for Istio-enabled Services", "widgets": [ { "layout": { "x": 0, "y": 0, "width": 6, "height": 3 }, "definition": { "type": "timeseries", "title": "% of Responses By Response Code", "yaxis": { "max": "auto", "min": "auto", "label": "", "scale": "linear", "include_zero": true }, "markers": [ { "value": "y = 80", "display_type": "error dashed" } ], "requests": [ { "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "queries": [ { "name": "query1", "query": "sum:istio.mesh.request.count{reporter:source,response_code:2*,destination_workload:$workload.value,destination_workload_namespace:$namespace.value,account:$awsaccount.value,$cluster} by {cluster,destination_workload,destination_workload_namespace}.as_rate()", "data_source": "metrics" }, { "name": "query2", "query": "sum:istio.mesh.request.count{reporter:source,destination_workload:$workload.value,destination_workload_namespace:$namespace.value,account:$awsaccount.value,$cluster} by {cluster,destination_workload,destination_workload_namespace}.as_rate()", "data_source": "metrics" }, { "name": "query3", "query": "sum:istio.mesh.request.count{reporter:source,response_code:3*,destination_workload:$workload.value,destination_workload_namespace:$namespace.value,account:$awsaccount.value,$cluster} by {cluster,destination_workload,destination_workload_namespace}.as_rate()", "data_source": "metrics" }, { "name": "query4", "query": "sum:istio.mesh.request.count{reporter:source,destination_workload:$workload.value,destination_workload_namespace:$namespace.value,account:$awsaccount.value,$cluster} by {cluster,destination_workload,destination_workload_namespace}.as_rate()", "data_source": "metrics" }, { "name": "query5", "query": "sum:istio.mesh.request.count{reporter:source,response_code:4*,destination_workload:$workload.value,destination_workload_namespace:$namespace.value,account:$awsaccount.value,$cluster} by {cluster,destination_workload,destination_workload_namespace}.as_rate()", "data_source": "metrics" }, { "name": "query6", "query": "sum:istio.mesh.request.count{reporter:source,destination_workload:$workload.value,destination_workload_namespace:$namespace.value,account:$awsaccount.value,$cluster} by {cluster,destination_workload,destination_workload_namespace}.as_rate()", "data_source": "metrics" }, { "name": "query7", "query": "sum:istio.mesh.request.count{reporter:source,response_code:5*,destination_workload:$workload.value,destination_workload_namespace:$namespace.value,account:$awsaccount.value,$cluster} by {cluster,destination_workload,destination_workload_namespace}.as_rate()", "data_source": "metrics" }, { "name": "query8", "query": "sum:istio.mesh.request.count{reporter:source,destination_workload:$workload.value,destination_workload_namespace:$namespace.value,account:$awsaccount.value,$cluster} by {cluster,destination_workload,destination_workload_namespace}.as_rate()", "data_source": "metrics" }, { "name": "query9", "query": "sum:istio.mesh.request.count{reporter:source,response_code:0,destination_workload:$workload.value,destination_workload_namespace:$namespace.value,account:$awsaccount.value,$cluster} by {cluster,destination_workload,destination_workload_namespace}.as_rate()", "data_source": "metrics" }, { "name": "query10", "query": "sum:istio.mesh.request.count{reporter:source,destination_workload:$workload.value,destination_workload_namespace:$namespace.value,account:$awsaccount.value,$cluster} by {cluster,destination_workload,destination_workload_namespace}.as_rate()", "data_source": "metrics" } ], "formulas": [ { "formula": "(query1 / query2) * 100" }, { "formula": "(query3 / query4) * 100" }, { "formula": "(query5 / query6) * 100" }, { "formula": "(query7 / query8) * 100" }, { "formula": "(query9 / query10) * 100" } ], "display_type": "area", "response_format": "timeseries" } ], "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ] } }, { "layout": { "x": 6, "y": 0, "width": 6, "height": 3 }, "definition": { "type": "timeseries", "title": "Incoming Requests by Source And Response Code", "yaxis": { "max": "auto", "min": "auto", "label": "", "scale": "linear", "include_zero": false }, "markers": [ { "value": "y = 80", "display_type": "error dashed" } ], "requests": [ { "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "queries": [ { "name": "query1", "query": "sum:istio.mesh.request.count{connection_security_policy:mutual_tls,destination_workload_namespace:$namespace.value,destination_workload:$workload.value,reporter:source,account:$awsaccount.value,$cluster} by {cluster,source_workload,source_workload_namespace,response_code}.as_rate()", "data_source": "metrics" }, { "name": "query2", "query": "sum:istio.mesh.request.count{!connection_security_policy:mutual_tls,destination_workload_namespace:$namespace.value,destination_workload:$workload.value,reporter:source,account:$awsaccount.value,$cluster} by {cluster,source_workload,source_workload_namespace,response_code}.as_rate()", "data_source": "metrics" } ], "formulas": [ { "formula": "query1" }, { "formula": "query2" } ], "display_type": "line", "response_format": "timeseries" } ], "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ] } }, { "layout": { "x": 0, "y": 3, "width": 6, "height": 3 }, "definition": { "type": "timeseries", "title": "Request Duration", "yaxis": { "max": "auto", "min": "auto", "label": "", "scale": "linear", "include_zero": true }, "requests": [ { "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "queries": [ { "name": "query1", "query": "p50:istio.mesh.request.duration.milliseconds{reporter:source,destination_workload:$workload.value,destination_workload_namespace:$namespace.value,account:$awsaccount.value,$cluster} by {cluster}", "data_source": "metrics" }, { "name": "query2", "query": "p90:istio.mesh.request.duration.milliseconds{reporter:source,destination_workload:$workload.value,destination_workload_namespace:$namespace.value,account:$awsaccount.value,$cluster} by {cluster}", "data_source": "metrics" }, { "name": "query3", "query": "p99:istio.mesh.request.duration.milliseconds{reporter:source,destination_workload:$workload.value,destination_workload_namespace:$namespace.value,account:$awsaccount.value,$cluster} by {cluster}", "data_source": "metrics" }, { "name": "query4", "query": "p95:istio.mesh.request.duration.milliseconds{reporter:source,destination_workload:$workload.value,destination_workload_namespace:$namespace.value,account:$awsaccount.value,$cluster} by {cluster}", "data_source": "metrics" } ], "formulas": [ { "formula": "query1 / 1000" }, { "formula": "query2 / 1000" }, { "formula": "query3 / 1000" }, { "formula": "query4 / 1000" } ], "display_type": "area", "response_format": "timeseries" } ], "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ] } }, { "layout": { "x": 6, "y": 3, "width": 6, "height": 3 }, "definition": { "type": "timeseries", "title": "Outgoing Requests by Destination And Response Code", "yaxis": { "max": "auto", "min": "auto", "label": "", "scale": "linear", "include_zero": false }, "requests": [ { "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "queries": [ { "name": "query1", "query": "sum:istio.mesh.request.count{connection_security_policy:mutual_tls,source_workload_namespace:$namespace.value,source_workload:$workload.value,reporter:source,account:$awsaccount.value,$cluster} by {cluster,destination_service,response_code}.as_rate()", "data_source": "metrics" }, { "name": "query2", "query": "sum:istio.mesh.request.count{!connection_security_policy:mutual_tls,source_workload_namespace:$namespace.value,source_workload:$workload.value,reporter:source,account:$awsaccount.value,$cluster} by {cluster,destination_service,response_code}.as_rate()", "data_source": "metrics" } ], "formulas": [ { "formula": "query1" }, { "formula": "query2" } ], "display_type": "line", "response_format": "timeseries" } ], "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ] } } ], "show_title": true, "layout_type": "ordered" } }, { "layout": { "x": 0, "y": 10, "width": 12, "height": 8 }, "definition": { "type": "group", "title": "Utilization, Saturation, Errors (USE)", "widgets": [ { "layout": { "x": 0, "y": 0, "width": 6, "height": 2 }, "definition": { "type": "timeseries", "title": "CPU Usage (Avg of all replicas)", "yaxis": { "max": "auto", "min": "auto", "label": "", "scale": "linear", "include_zero": false }, "requests": [ { "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "queries": [ { "name": "query1", "query": "max:kubernetes.cpu.usage.total{$namespace,pod_name:$workload.value*,!kube_container_name:POD,account:$awsaccount.value,$cluster} by {cluster,kube_container_name}", "data_source": "metrics" }, { "name": "query2", "query": "avg:kubernetes_state.container.cpu_requested{$namespace,pod_name:$workload.value*,account:$awsaccount.value,$cluster} by {cluster,kube_container_name}", "data_source": "metrics" }, { "name": "query3", "query": "avg:kubernetes_state.container.cpu_limit{$namespace,pod_name:$workload.value*,account:$awsaccount.value,$cluster} by {cluster,kube_container_name}", "data_source": "metrics" } ], "formulas": [ { "formula": "query1 / 1000000000" }, { "formula": "query2" }, { "formula": "query3" } ], "display_type": "line", "response_format": "timeseries" } ], "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ] } }, { "layout": { "x": 6, "y": 0, "width": 6, "height": 2 }, "definition": { "type": "timeseries", "title": "Memory Usage (Max of all replicas)", "yaxis": { "max": "auto", "min": "auto", "label": "", "scale": "linear", "include_zero": false }, "requests": [ { "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "queries": [ { "name": "query1", "query": "max:kubernetes.memory.working_set{$namespace,pod_name:$workload.value*,!kube_container_name:POD,account:$awsaccount.value,$cluster} by {cluster,kube_container_name}", "data_source": "metrics" }, { "name": "query2", "query": "max:kubernetes_state.container.memory_requested{$namespace,pod_name:$workload.value*,!kube_container_name:POD,account:$awsaccount.value,$cluster} by {cluster,kube_container_name}", "data_source": "metrics" }, { "name": "query3", "query": "max:kubernetes_state.container.memory_limit{$namespace,pod_name:$workload.value*,!kube_container_name:POD,account:$awsaccount.value,$cluster} by {cluster,kube_container_name}", "data_source": "metrics" } ], "formulas": [ { "formula": "query1" }, { "formula": "query2" }, { "formula": "query3" } ], "display_type": "line", "response_format": "timeseries" } ], "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ] } }, { "layout": { "x": 0, "y": 2, "width": 6, "height": 2 }, "definition": { "type": "timeseries", "title": "CPU HPA", "yaxis": { "max": "auto", "min": "auto", "label": "", "scale": "linear", "include_zero": false }, "requests": [ { "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "queries": [ { "name": "query1", "query": "max:kubernetes.cpu.usage.total{$namespace,pod_name:$workload.value*,!kube_container_name:POD,account:$awsaccount.value,$cluster} by {cluster}", "data_source": "metrics" }, { "name": "query2", "query": "max:kubernetes_state.container.cpu_requested{$namespace,pod_name:$workload.value*,account:$awsaccount.value,$cluster} by {cluster}", "data_source": "metrics" }, { "name": "query3", "query": "avg:kube_horizontalpodautoscaler_spec_target_metric{$namespace,horizontalpodautoscaler:$workload.value,account:$awsaccount.value,$cluster,metric_name:cpu} by {cluster,horizontalpodautoscaler}", "data_source": "metrics" } ], "formulas": [ { "alias": "Max of all pods - CPU % based on request", "formula": "(query1 / 1000000000) / query2 * 100" }, { "alias": "CPU HPA target", "formula": "query3" } ], "display_type": "line", "response_format": "timeseries" } ], "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ] } }, { "layout": { "x": 6, "y": 2, "width": 6, "height": 2 }, "definition": { "type": "timeseries", "title": "Deployment Healthy POD Replicas", "yaxis": { "max": "auto", "min": "auto", "label": "", "scale": "linear", "include_zero": false }, "requests": [ { "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "queries": [ { "name": "query1", "query": "sum:kubernetes_state.deployment.replicas_available{$namespace,kube_deployment:$workload.value,account:$awsaccount.value,$cluster} by {cluster}", "data_source": "metrics" } ], "formulas": [ { "formula": "query1" } ], "display_type": "line", "response_format": "timeseries" } ], "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ] } }, { "layout": { "x": 0, "y": 4, "width": 6, "height": 2 }, "definition": { "type": "timeseries", "title": "Disk I/O (Bytes)", "yaxis": { "max": "auto", "min": "auto", "label": "", "scale": "linear", "include_zero": true }, "requests": [ { "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "queries": [ { "name": "query1", "query": "sum:kubernetes.io.read_bytes{$namespace,pod_name:$workload.value*,account:$awsaccount.value,$cluster} by {cluster}", "data_source": "metrics" }, { "name": "query2", "query": "sum:kubernetes.io.write_bytes{$namespace,pod_name:$workload.value*,account:$awsaccount.value,$cluster} by {cluster}", "data_source": "metrics" } ], "formulas": [ { "formula": "query1" }, { "formula": "query2 * -1" } ], "display_type": "area", "response_format": "timeseries" } ], "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ] } }, { "layout": { "x": 6, "y": 4, "width": 6, "height": 2 }, "definition": { "type": "timeseries", "title": "Disk I/O (IOPS)", "yaxis": { "max": "auto", "min": "auto", "label": "", "scale": "linear", "include_zero": true }, "requests": [ { "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "queries": [ { "name": "query1", "query": "sum:container_fs_reads.count{$namespace,pod:$workload.value*,account:$awsaccount.value,$cluster} by {cluster}.as_rate()", "data_source": "metrics" }, { "name": "query2", "query": "sum:container_fs_writes.count{$namespace,pod:$workload.value*,account:$awsaccount.value,$cluster} by {cluster}.as_rate()", "data_source": "metrics" } ], "formulas": [ { "formula": "query1" }, { "formula": "query2 * -1" } ], "display_type": "area", "response_format": "timeseries" } ], "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ] } }, { "layout": { "x": 0, "y": 6, "width": 6, "height": 2 }, "definition": { "type": "timeseries", "title": "Network I/O", "yaxis": { "max": "auto", "min": "auto", "label": "", "scale": "linear", "include_zero": true }, "requests": [ { "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "queries": [ { "name": "query1", "query": "sum:kubernetes.network.rx_bytes{$namespace,pod_name:$workload.value*,account:$awsaccount.value,$cluster} by {cluster}", "data_source": "metrics" }, { "name": "query2", "query": "sum:kubernetes.network.tx_bytes{$namespace,pod_name:$workload.value*,account:$awsaccount.value,$cluster} by {cluster}", "data_source": "metrics" } ], "formulas": [ { "formula": "query1" }, { "formula": "query2 * -1" } ], "display_type": "area", "response_format": "timeseries" } ], "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ] } }, { "layout": { "x": 6, "y": 6, "width": 6, "height": 2 }, "definition": { "type": "timeseries", "title": "Open TCP Connections", "yaxis": { "max": "auto", "min": "auto", "label": "", "scale": "linear", "include_zero": true }, "requests": [ { "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "queries": [ { "name": "query1", "query": "sum:istio.envoy.server.total_connections{kubernetes_namespace:$namespace.value,service_istio_io_canonical_name:$workload.value,$cluster,account:$awsaccount.value} by {cluster,service_istio_io_canonical_name}", "data_source": "metrics" } ], "formulas": [ { "formula": "query1" } ], "display_type": "area", "response_format": "timeseries" } ], "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ] } } ], "show_title": true, "layout_type": "ordered" } }, { "layout": { "x": 0, "y": 18, "width": 12, "height": 7 }, "definition": { "type": "group", "title": "Infrastructure Resources and Events", "widgets": [ { "layout": { "x": 0, "y": 0, "width": 6, "height": 3 }, "definition": { "type": "note", "content": "**Node & AZ Distribution of Workload**\n\nUnsupported panel (timeseries)\n\n- nested count() aggregations over instance/node/zone labels (distinct-count topology) cannot be expressed with Datadog's query grammar\n\nOriginal PromQL:\n```\nB: sum(count(count(container_cpu_usage_seconds_total{namespace=\"$namespace\",pod=~\"$workload-.*\",container!~\"POD|istio-proxy\",account=~\"$awsaccount\",cluster=~\"$cluster\"}) by (cluster,instance,node_type,node_kubernetes_io_instance_type,failure_domain_beta_kubernetes_io_zone)) by (cluster,instance,node_type,node_kubernetes_io_instance_type,failure_domain_beta_kubernetes_io_zone)) by (cluster,node_type,failure_domain_beta_kubernetes_io_zone)\n```", "tick_pos": "50%", "font_size": "14", "show_tick": false, "tick_edge": "left", "text_align": "left", "has_padding": true, "vertical_align": "top", "background_color": "white" } }, { "layout": { "x": 6, "y": 0, "width": 6, "height": 3 }, "definition": { "type": "timeseries", "title": "Workload Replicas (PODs)", "yaxis": { "max": "auto", "min": "auto", "label": "", "scale": "linear", "include_zero": true }, "requests": [ { "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "queries": [ { "name": "query1", "query": "avg:kubernetes_state.hpa.current_replicas{$namespace,account:$awsaccount.value,$cluster} by {horizontalpodautoscaler,cluster}", "data_source": "metrics" }, { "name": "query2", "query": "avg:kubernetes_state.hpa.desired_replicas{$namespace,horizontalpodautoscaler:$workload.value*,account:$awsaccount.value,$cluster} by {horizontalpodautoscaler,cluster}", "data_source": "metrics" }, { "name": "query3", "query": "avg:kubernetes_state.hpa.min_replicas{$namespace,horizontalpodautoscaler:$workload.value*,account:$awsaccount.value,$cluster} by {horizontalpodautoscaler,cluster}", "data_source": "metrics" }, { "name": "query4", "query": "avg:kubernetes_state.hpa.max_replicas{$namespace,horizontalpodautoscaler:$workload.value*,account:$awsaccount.value,$cluster} by {horizontalpodautoscaler,cluster}", "data_source": "metrics" }, { "name": "query5", "query": "avg:kubernetes_state.deployment.replicas_unavailable{$namespace,kube_deployment:$workload.value,account:$awsaccount.value,$cluster} by {kube_deployment,cluster}", "data_source": "metrics" } ], "formulas": [ { "formula": "query1" }, { "formula": "query2" }, { "formula": "query3" }, { "formula": "query4" }, { "formula": "query5" } ], "display_type": "area", "response_format": "timeseries" } ], "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ] } }, { "layout": { "x": 0, "y": 3, "width": 4, "height": 2 }, "definition": { "type": "timeseries", "title": "Running PODs per Container Image Tag", "yaxis": { "max": "auto", "min": "auto", "label": "", "scale": "linear", "include_zero": true }, "requests": [ { "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "queries": [ { "name": "query1", "query": "sum:kube_pod_container_info{$namespace AND pod:$workload.value-* AND container NOT IN (pod,istio-proxy) AND account:$awsaccount.value AND $cluster} by {cluster,image}", "data_source": "metrics" } ], "formulas": [ { "formula": "query1" } ], "display_type": "area", "response_format": "timeseries" } ], "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ] } }, { "layout": { "x": 4, "y": 3, "width": 4, "height": 2 }, "definition": { "type": "timeseries", "title": "Total vCPU's Allocated to Workload", "yaxis": { "max": "auto", "min": "auto", "label": "", "scale": "linear", "include_zero": true }, "markers": [ { "value": "y = 2", "display_type": "error dashed" } ], "requests": [ { "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "queries": [ { "name": "query1", "query": "sum:kubernetes_state.container.cpu_requested{$namespace AND pod_name:$workload.value* AND kube_container_name NOT IN (pod,istio-proxy) AND account:$awsaccount.value AND $cluster} by {cluster}", "data_source": "metrics" }, { "name": "query2", "query": "sum:kubernetes_state.container.cpu_requested{$namespace AND pod_name:$workload.value* AND kube_container_name IN (pod,istio-proxy) AND account:$awsaccount.value AND $cluster} by {cluster}", "data_source": "metrics" } ], "formulas": [ { "formula": "query1" }, { "formula": "query2" } ], "display_type": "area", "response_format": "timeseries" } ], "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ] } }, { "layout": { "x": 8, "y": 3, "width": 4, "height": 2 }, "definition": { "type": "timeseries", "title": "Total Memory Allocated to Workload", "yaxis": { "max": "auto", "min": "auto", "label": "", "scale": "linear", "include_zero": true }, "requests": [ { "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "queries": [ { "name": "query1", "query": "sum:kubernetes_state.container.memory_requested{$namespace AND pod_name:$workload.value* AND kube_container_name NOT IN (pod,istio-proxy) AND account:$awsaccount.value AND $cluster} by {cluster}", "data_source": "metrics" }, { "name": "query2", "query": "sum:kubernetes_state.container.memory_requested{$namespace AND pod_name:$workload.value* AND kube_container_name IN (pod,istio-proxy) AND account:$awsaccount.value AND $cluster} by {cluster}", "data_source": "metrics" } ], "formulas": [ { "formula": "query1" }, { "formula": "query2" } ], "display_type": "area", "response_format": "timeseries" } ], "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ] } }, { "layout": { "x": 0, "y": 5, "width": 4, "height": 2 }, "definition": { "type": "timeseries", "title": "CPU Throttling", "yaxis": { "max": "auto", "min": "auto", "label": "", "scale": "linear", "include_zero": true }, "requests": [ { "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "queries": [ { "name": "query1", "query": "avg:kubernetes.cpu.cfs.throttled.seconds{$namespace,pod_name:$workload.value-*,account:$awsaccount.value,$cluster} by {cluster,kube_container_name}.as_rate()", "data_source": "metrics" } ], "formulas": [ { "formula": "query1" } ], "display_type": "area", "response_format": "timeseries" } ], "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ] } }, { "layout": { "x": 4, "y": 5, "width": 4, "height": 2 }, "definition": { "type": "timeseries", "title": "POD Restarts", "yaxis": { "max": "auto", "min": "auto", "label": "", "scale": "linear", "include_zero": true }, "markers": [ { "value": "y = 2", "display_type": "error dashed" } ], "requests": [ { "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "queries": [ { "name": "query1", "query": "sum:kubernetes_state.container.restarts{$namespace,pod_name:$workload.value-*,account:$awsaccount.value,$cluster} by {cluster,pod_name,host}", "data_source": "metrics" } ], "formulas": [ { "formula": "count_not_null(diff(query1))" } ], "display_type": "area", "response_format": "timeseries" } ], "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ] } }, { "layout": { "x": 8, "y": 5, "width": 4, "height": 2 }, "definition": { "type": "timeseries", "title": "OOM Kills", "yaxis": { "max": "auto", "min": "auto", "label": "", "scale": "linear", "include_zero": true }, "markers": [ { "value": "y = 1", "display_type": "warning dashed" } ], "requests": [ { "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "queries": [ { "name": "query1", "query": "sum:node_vmstat_oom_kill{kubernetes_namespace:$namespace.value,app:$workload.value,account:$awsaccount.value,$cluster} by {cluster}", "data_source": "metrics" } ], "formulas": [ { "formula": "query1" } ], "display_type": "area", "response_format": "timeseries" } ], "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ] } } ], "show_title": true, "layout_type": "ordered" } } ], "description": "Converted from Grafana dashboard https://grafana.com/grafana/dashboards/15160 (by roku). Watch performance, errors, latency, and infrastructure metrics for your workload", "layout_type": "ordered", "reflow_type": "fixed", "template_variables": [ { "name": "awsaccount", "prefix": "awsaccount", "default": "*", "available_values": [] }, { "name": "cluster", "prefix": "cluster", "default": "*", "available_values": [] }, { "name": "namespace", "prefix": "kube_namespace", "default": "*", "available_values": [] }, { "name": "workload", "prefix": "workload", "default": "*", "available_values": [] } ] }