{
  "tags": [],
  "title": "AWS CNI Metrics",
  "widgets": [
    {
      "layout": {
        "x": 0,
        "y": 0,
        "width": 12,
        "height": 6
      },
      "definition": {
        "type": "group",
        "title": "Headlines",
        "widgets": [
          {
            "layout": {
              "x": 0,
              "y": 0,
              "width": 1,
              "height": 2
            },
            "definition": {
              "type": "query_value",
              "title": "CNI Pods",
              "requests": [
                {
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:awscni_eni_max{*} by {host}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query2",
                      "query": "sum:kube_node_labels{*}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query3",
                      "query": "sum:kubernetes_state.pod.count{!priority_class:overprovisioning AND NOT created_by_name IN (aws-node,kube-proxy,kube-prometheus-stack-prometheus-node-exporter)}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query4",
                      "query": "sum:kubernetes_state.pod.count{priority_class:overprovisioning}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "alias": "Agents",
                      "formula": "count_not_null(query1)"
                    }
                  ],
                  "response_format": "scalar"
                }
              ],
              "autoscale": true,
              "precision": 2,
              "text_align": "center"
            }
          },
          {
            "layout": {
              "x": 1,
              "y": 0,
              "width": 2,
              "height": 2
            },
            "definition": {
              "type": "query_value",
              "title": "ENI in the Cluster",
              "requests": [
                {
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:awscni_eni_max{*}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query2",
                      "query": "sum:awscni_eni_allocated{*}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "alias": "Max",
                      "formula": "query1"
                    }
                  ],
                  "response_format": "scalar"
                }
              ],
              "autoscale": true,
              "precision": 2,
              "text_align": "center"
            }
          },
          {
            "layout": {
              "x": 3,
              "y": 0,
              "width": 1,
              "height": 2
            },
            "definition": {
              "type": "query_value",
              "title": "IP Addresses in the Cluster",
              "requests": [
                {
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:awscni_ip_max{*}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query2",
                      "query": "sum:awscni_ip_max{*} by {host}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query3",
                      "query": "sum:awscni_assigned_ip_addresses{*}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "alias": "Max",
                      "formula": "query1 - count_not_null(query2)"
                    }
                  ],
                  "response_format": "scalar"
                }
              ],
              "autoscale": true,
              "precision": 2,
              "text_align": "center"
            }
          },
          {
            "layout": {
              "x": 4,
              "y": 0,
              "width": 2,
              "height": 2
            },
            "definition": {
              "type": "query_value",
              "title": "Available IP Addresses in the Cluster",
              "requests": [
                {
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:awscni_total_ip_addresses{*}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query2",
                      "query": "sum:awscni_assigned_ip_addresses{*}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query3",
                      "query": "sum:awscni_total_ip_addresses{*} by {host}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query4",
                      "query": "sum:awscni_ip_max{*}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query5",
                      "query": "sum:awscni_total_ip_addresses{*}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "alias": "Warm",
                      "formula": "query1 - count_not_null(query3) - query2"
                    }
                  ],
                  "response_format": "scalar"
                }
              ],
              "autoscale": true,
              "precision": 2,
              "text_align": "center"
            }
          },
          {
            "layout": {
              "x": 6,
              "y": 0,
              "width": 5,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "IP Addresses in the Cluster",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": true
              },
              "markers": [
                {
                  "value": "y = 80",
                  "display_type": "error dashed"
                }
              ],
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:awscni_assigned_ip_addresses{*}",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query2",
                      "query": "sum:awscni_total_ip_addresses{*}",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query3",
                      "query": "sum:awscni_total_ip_addresses{*} by {host}",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query4",
                      "query": "sum:awscni_ip_max{*}",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query5",
                      "query": "sum:awscni_ip_max{*} by {host}",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "alias": "assigned to pods",
                      "formula": "query1"
                    },
                    {
                      "alias": "total number",
                      "formula": "query2 - count_not_null(query3)"
                    },
                    {
                      "alias": "max that can be allocated",
                      "formula": "query4 - count_not_null(query5)"
                    }
                  ],
                  "display_type": "area",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          },
          {
            "layout": {
              "x": 0,
              "y": 2,
              "width": 3,
              "height": 3
            },
            "definition": {
              "type": "note",
              "content": "**ENI Usage (node count per usage %)**\n\nUnsupported panel (piechart)\n\n- count_values() has no Datadog equivalent (cannot build a histogram of metric values)\n\nOriginal PromQL:\n```\nA: count_values(\"value\", 100*awscni_eni_allocated/awscni_eni_max)\n```",
              "tick_pos": "50%",
              "font_size": "14",
              "show_tick": false,
              "tick_edge": "left",
              "text_align": "left",
              "has_padding": true,
              "vertical_align": "top",
              "background_color": "white"
            }
          },
          {
            "layout": {
              "x": 3,
              "y": 2,
              "width": 1,
              "height": 2
            },
            "definition": {
              "type": "query_value",
              "title": "ENI Usage (Cluster)",
              "requests": [
                {
                  "queries": [
                    {
                      "name": "query1",
                      "query": "avg:awscni_eni_allocated{*}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query2",
                      "query": "avg:awscni_eni_max{*}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "(query1 / query2) * 100"
                    }
                  ],
                  "response_format": "scalar"
                }
              ],
              "autoscale": true,
              "precision": 2,
              "text_align": "center",
              "custom_unit": "%"
            }
          },
          {
            "layout": {
              "x": 4,
              "y": 2,
              "width": 2,
              "height": 2
            },
            "definition": {
              "type": "query_value",
              "title": "IP Addresses Usage (Cluster)",
              "requests": [
                {
                  "queries": [
                    {
                      "name": "query1",
                      "query": "avg:awscni_assigned_ip_addresses{*}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query2",
                      "query": "avg:awscni_total_ip_addresses{*}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "(query1 / (query2 - 1)) * 100"
                    }
                  ],
                  "response_format": "scalar"
                }
              ],
              "autoscale": true,
              "precision": 2,
              "text_align": "center",
              "custom_unit": "%"
            }
          },
          {
            "layout": {
              "x": 6,
              "y": 2,
              "width": 5,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "Add/Del IP Address Rate",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": false
              },
              "markers": [
                {
                  "value": "y = 1",
                  "display_type": "error dashed"
                }
              ],
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:awscni_add_ip_req_count.count{*}.as_rate()",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query2",
                      "query": "sum:awscni_del_ip_req_count.count{*}.as_rate()",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "alias": "add",
                      "formula": "query1"
                    },
                    {
                      "alias": "del",
                      "formula": "query2"
                    }
                  ],
                  "display_type": "line",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          },
          {
            "layout": {
              "x": 3,
              "y": 4,
              "width": 1,
              "height": 2
            },
            "definition": {
              "type": "query_value",
              "title": "Force Removed ENI / IP",
              "requests": [
                {
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:awscni_force_removed_enis.count{*}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query2",
                      "query": "sum:awscni_force_removed_ips.count{*}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "alias": "ENI",
                      "formula": "query1"
                    }
                  ],
                  "response_format": "scalar",
                  "conditional_formats": [
                    {
                      "value": 1,
                      "palette": "white_on_red",
                      "comparator": ">="
                    }
                  ]
                }
              ],
              "autoscale": true,
              "precision": 2,
              "text_align": "center",
              "timeseries_background": {
                "type": "area"
              }
            }
          },
          {
            "layout": {
              "x": 4,
              "y": 4,
              "width": 2,
              "height": 2
            },
            "definition": {
              "type": "query_value",
              "title": "Delete IP Address Errors",
              "requests": [
                {
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:awscni_del_ip_req_count.count{!reason:poddeleted}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1"
                    }
                  ],
                  "response_format": "scalar",
                  "conditional_formats": [
                    {
                      "value": 1,
                      "palette": "white_on_red",
                      "comparator": ">="
                    }
                  ]
                }
              ],
              "autoscale": true,
              "precision": 2,
              "text_align": "center",
              "timeseries_background": {
                "type": "area"
              }
            }
          },
          {
            "layout": {
              "x": 6,
              "y": 4,
              "width": 1,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "Delete IP Address Errors (by Reason)",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": true
              },
              "markers": [
                {
                  "value": "y = 80",
                  "display_type": "error dashed"
                }
              ],
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:awscni_del_ip_req_count.count{!reason:poddeleted} by {reason}",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1"
                    }
                  ],
                  "display_type": "area",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          },
          {
            "layout": {
              "x": 7,
              "y": 4,
              "width": 2,
              "height": 2
            },
            "definition": {
              "type": "query_value",
              "title": "API Errors on the Period",
              "requests": [
                {
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:awscni_aws_api_error_count.count{*} by {api}.rollup(sum, 300)",
                      "aggregator": "sum",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1"
                    }
                  ],
                  "response_format": "scalar",
                  "conditional_formats": [
                    {
                      "value": 1,
                      "palette": "white_on_red",
                      "comparator": ">="
                    }
                  ]
                }
              ],
              "autoscale": true,
              "precision": 2,
              "text_align": "center"
            }
          },
          {
            "layout": {
              "x": 9,
              "y": 4,
              "width": 2,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "API Errors on the Period (by API)",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": true
              },
              "markers": [
                {
                  "value": "y = 80",
                  "display_type": "error dashed"
                }
              ],
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:awscni_aws_api_error_count.count{*} by {api}.rollup(sum, 300)",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1"
                    }
                  ],
                  "display_type": "area",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          }
        ],
        "show_title": true,
        "layout_type": "ordered"
      }
    },
    {
      "layout": {
        "x": 0,
        "y": 6,
        "width": 12,
        "height": 4
      },
      "definition": {
        "type": "group",
        "title": "Activity",
        "widgets": [
          {
            "layout": {
              "x": 0,
              "y": 0,
              "width": 11,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "Average AWS API Latency (5min)",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": true
              },
              "markers": [
                {
                  "value": "y = 80",
                  "display_type": "error dashed"
                }
              ],
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "avg:awscni_aws_api_latency_ms.sum{*} by {api}.as_rate()",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query2",
                      "query": "avg:awscni_aws_api_latency_ms.count{*} by {api}.as_rate()",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1 / query2"
                    }
                  ],
                  "display_type": "area",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          },
          {
            "layout": {
              "x": 0,
              "y": 2,
              "width": 5,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "ipamd Actions in Progress",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": true
              },
              "markers": [
                {
                  "value": "y = 80",
                  "display_type": "error dashed"
                }
              ],
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:awscni_ipamd_action_inprogress{*} by {fn}",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1"
                    }
                  ],
                  "display_type": "area",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          },
          {
            "layout": {
              "x": 5,
              "y": 2,
              "width": 6,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "The number of times ipamd reconciles on ENIs and IP addresses",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": true
              },
              "markers": [
                {
                  "value": "y = 80",
                  "display_type": "error dashed"
                }
              ],
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:awscni_reconcile_count.count{*} by {fn}.as_rate()",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1"
                    }
                  ],
                  "display_type": "area",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          }
        ],
        "show_title": true,
        "layout_type": "ordered"
      }
    },
    {
      "layout": {
        "x": 0,
        "y": 10,
        "width": 12,
        "height": 3
      },
      "definition": {
        "type": "group",
        "title": "CNI Metrics Per Node",
        "widgets": [
          {
            "layout": {
              "x": 0,
              "y": 0,
              "width": 3,
              "height": 3
            },
            "definition": {
              "type": "query_table",
              "title": "Assigned IP Addresses",
              "requests": [
                {
                  "queries": [
                    {
                      "name": "query1",
                      "query": "avg:awscni_assigned_ip_addresses{*}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "limit": {
                        "count": 50,
                        "order": "desc"
                      },
                      "formula": "query1"
                    }
                  ],
                  "response_format": "scalar"
                }
              ],
              "has_search_bar": "auto"
            }
          },
          {
            "layout": {
              "x": 3,
              "y": 0,
              "width": 3,
              "height": 3
            },
            "definition": {
              "type": "query_table",
              "title": "IP Deletion Requests (Cumulated)",
              "requests": [
                {
                  "queries": [
                    {
                      "name": "query1",
                      "query": "avg:awscni_del_ip_req_count.count{*}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "limit": {
                        "count": 50,
                        "order": "desc"
                      },
                      "formula": "query1"
                    }
                  ],
                  "response_format": "scalar"
                }
              ],
              "has_search_bar": "auto"
            }
          },
          {
            "layout": {
              "x": 6,
              "y": 0,
              "width": 5,
              "height": 3
            },
            "definition": {
              "type": "query_table",
              "title": "API Errors (cumulated)",
              "requests": [
                {
                  "queries": [
                    {
                      "name": "query1",
                      "query": "avg:awscni_aws_api_error_count.count{*}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "limit": {
                        "count": 50,
                        "order": "desc"
                      },
                      "formula": "query1"
                    }
                  ],
                  "response_format": "scalar"
                }
              ],
              "has_search_bar": "auto"
            }
          }
        ],
        "show_title": true,
        "layout_type": "ordered"
      }
    }
  ],
  "description": "Converted from Grafana dashboard https://grafana.com/grafana/dashboards/16032 (by ndegory). USE (Utilization Saturation Errors) metrics dashboards for troubleshooting pod IP management in EKS",
  "layout_type": "ordered",
  "reflow_type": "fixed",
  "template_variables": [
    {
      "name": "instance",
      "prefix": "instance",
      "default": "*",
      "available_values": []
    },
    {
      "name": "ip_deletion_reason",
      "prefix": "reason",
      "default": "*",
      "available_values": []
    }
  ]
}