{
  "tags": [],
  "title": "Kubernetes / Kubelet",
  "widgets": [
    {
      "layout": {
        "x": 0,
        "y": 0,
        "width": 2,
        "height": 2
      },
      "definition": {
        "type": "note",
        "content": "**Up**\n\nUnsupported panel (singlestat)\n\n- Prometheus `up` has no Datadog metric; use the openmetrics.health service check\n\nOriginal PromQL:\n```\nA: sum(up{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\"})\n```",
        "tick_pos": "50%",
        "font_size": "14",
        "show_tick": false,
        "tick_edge": "left",
        "text_align": "left",
        "has_padding": true,
        "vertical_align": "top",
        "background_color": "white"
      }
    },
    {
      "layout": {
        "x": 2,
        "y": 0,
        "width": 2,
        "height": 2
      },
      "definition": {
        "type": "query_value",
        "title": "Running Pods",
        "requests": [
          {
            "queries": [
              {
                "name": "query1",
                "query": "sum:kubernetes.pods.running{$cluster,$instance}",
                "aggregator": "last",
                "data_source": "metrics"
              }
            ],
            "formulas": [
              {
                "formula": "query1"
              }
            ],
            "response_format": "scalar"
          }
        ],
        "autoscale": true,
        "precision": 2,
        "text_align": "center"
      }
    },
    {
      "layout": {
        "x": 4,
        "y": 0,
        "width": 2,
        "height": 2
      },
      "definition": {
        "type": "query_value",
        "title": "Running Container",
        "requests": [
          {
            "queries": [
              {
                "name": "query1",
                "query": "sum:kubernetes.containers.running{$cluster,$instance}",
                "aggregator": "last",
                "data_source": "metrics"
              }
            ],
            "formulas": [
              {
                "formula": "query1"
              }
            ],
            "response_format": "scalar"
          }
        ],
        "autoscale": true,
        "precision": 2,
        "text_align": "center"
      }
    },
    {
      "layout": {
        "x": 6,
        "y": 0,
        "width": 2,
        "height": 2
      },
      "definition": {
        "type": "query_value",
        "title": "Actual Volume Count",
        "requests": [
          {
            "queries": [
              {
                "name": "query1",
                "query": "sum:volume_manager_total_volumes{$cluster,$instance,job:kubelet,metrics_path:/metrics,state:actual_state_of_world}",
                "aggregator": "last",
                "data_source": "metrics"
              }
            ],
            "formulas": [
              {
                "formula": "query1"
              }
            ],
            "response_format": "scalar"
          }
        ],
        "autoscale": true,
        "precision": 2,
        "text_align": "center"
      }
    },
    {
      "layout": {
        "x": 8,
        "y": 0,
        "width": 2,
        "height": 2
      },
      "definition": {
        "type": "query_value",
        "title": "Desired Volume Count",
        "requests": [
          {
            "queries": [
              {
                "name": "query1",
                "query": "sum:volume_manager_total_volumes{$cluster,$instance,job:kubelet,metrics_path:/metrics,state:desired_state_of_world}",
                "aggregator": "last",
                "data_source": "metrics"
              }
            ],
            "formulas": [
              {
                "formula": "query1"
              }
            ],
            "response_format": "scalar"
          }
        ],
        "autoscale": true,
        "precision": 2,
        "text_align": "center"
      }
    },
    {
      "layout": {
        "x": 10,
        "y": 0,
        "width": 2,
        "height": 2
      },
      "definition": {
        "type": "query_value",
        "title": "Config Error Count",
        "requests": [
          {
            "queries": [
              {
                "name": "query1",
                "query": "sum:kubelet_node_config_error{$cluster,$instance,job:kubelet,metrics_path:/metrics}.as_rate()",
                "aggregator": "last",
                "data_source": "metrics"
              }
            ],
            "formulas": [
              {
                "formula": "query1"
              }
            ],
            "response_format": "scalar"
          }
        ],
        "autoscale": true,
        "precision": 2,
        "text_align": "center"
      }
    },
    {
      "layout": {
        "x": 0,
        "y": 2,
        "width": 6,
        "height": 2
      },
      "definition": {
        "type": "timeseries",
        "title": "Operation Rate",
        "yaxis": {
          "max": "auto",
          "min": "auto",
          "label": "",
          "scale": "linear",
          "include_zero": true
        },
        "requests": [
          {
            "style": {
              "palette": "dog_classic",
              "line_type": "solid",
              "line_width": "normal"
            },
            "queries": [
              {
                "name": "query1",
                "query": "sum:kubelet_runtime_operations.count{$cluster,$instance,job:kubelet,metrics_path:/metrics} by {operation_type,instance}.as_rate()",
                "data_source": "metrics"
              }
            ],
            "formulas": [
              {
                "formula": "query1"
              }
            ],
            "display_type": "area",
            "response_format": "timeseries"
          }
        ],
        "show_legend": true,
        "legend_layout": "auto",
        "legend_columns": [
          "avg",
          "min",
          "max",
          "value",
          "sum"
        ]
      }
    },
    {
      "layout": {
        "x": 6,
        "y": 2,
        "width": 6,
        "height": 2
      },
      "definition": {
        "type": "timeseries",
        "title": "Operation Error Rate",
        "yaxis": {
          "max": "auto",
          "min": "auto",
          "label": "",
          "scale": "linear",
          "include_zero": true
        },
        "requests": [
          {
            "style": {
              "palette": "dog_classic",
              "line_type": "solid",
              "line_width": "normal"
            },
            "queries": [
              {
                "name": "query1",
                "query": "sum:kubernetes.kubelet.runtime.errors{$cluster,$instance} by {host,operation_type}.as_rate()",
                "data_source": "metrics"
              }
            ],
            "formulas": [
              {
                "formula": "query1"
              }
            ],
            "display_type": "area",
            "response_format": "timeseries"
          }
        ],
        "show_legend": true,
        "legend_layout": "auto",
        "legend_columns": [
          "avg",
          "min",
          "max",
          "value",
          "sum"
        ]
      }
    },
    {
      "layout": {
        "x": 0,
        "y": 4,
        "width": 12,
        "height": 2
      },
      "definition": {
        "type": "timeseries",
        "title": "Operation duration 99th quantile",
        "yaxis": {
          "max": "auto",
          "min": "auto",
          "label": "",
          "scale": "linear",
          "include_zero": true
        },
        "requests": [
          {
            "style": {
              "palette": "dog_classic",
              "line_type": "solid",
              "line_width": "normal"
            },
            "queries": [
              {
                "name": "query1",
                "query": "p99:kubelet_runtime_operations_duration_seconds{$cluster,$instance,job:kubelet,metrics_path:/metrics} by {instance,operation_type}",
                "data_source": "metrics"
              }
            ],
            "formulas": [
              {
                "formula": "query1"
              }
            ],
            "display_type": "area",
            "response_format": "timeseries"
          }
        ],
        "show_legend": true,
        "legend_layout": "auto",
        "legend_columns": [
          "avg",
          "min",
          "max",
          "value",
          "sum"
        ]
      }
    },
    {
      "layout": {
        "x": 0,
        "y": 6,
        "width": 6,
        "height": 2
      },
      "definition": {
        "type": "timeseries",
        "title": "Pod Start Rate",
        "yaxis": {
          "max": "auto",
          "min": "auto",
          "label": "",
          "scale": "linear",
          "include_zero": true
        },
        "requests": [
          {
            "style": {
              "palette": "dog_classic",
              "line_type": "solid",
              "line_width": "normal"
            },
            "queries": [
              {
                "name": "query1",
                "query": "sum:kubelet_pod_start_duration_seconds.count{$cluster,$instance,job:kubelet,metrics_path:/metrics} by {instance}.as_rate()",
                "data_source": "metrics"
              },
              {
                "name": "query2",
                "query": "sum:kubelet_pod_worker_duration_seconds.count{$cluster,$instance,job:kubelet,metrics_path:/metrics} by {instance}.as_rate()",
                "data_source": "metrics"
              }
            ],
            "formulas": [
              {
                "formula": "query1"
              },
              {
                "formula": "query2"
              }
            ],
            "display_type": "area",
            "response_format": "timeseries"
          }
        ],
        "show_legend": true,
        "legend_layout": "auto",
        "legend_columns": [
          "avg",
          "min",
          "max",
          "value",
          "sum"
        ]
      }
    },
    {
      "layout": {
        "x": 6,
        "y": 6,
        "width": 6,
        "height": 2
      },
      "definition": {
        "type": "timeseries",
        "title": "Pod Start Duration",
        "yaxis": {
          "max": "auto",
          "min": "auto",
          "label": "",
          "scale": "linear",
          "include_zero": true
        },
        "requests": [
          {
            "style": {
              "palette": "dog_classic",
              "line_type": "solid",
              "line_width": "normal"
            },
            "queries": [
              {
                "name": "query1",
                "query": "p99:kubelet_pod_start_duration_seconds{$cluster,$instance,job:kubelet,metrics_path:/metrics} by {instance}",
                "data_source": "metrics"
              },
              {
                "name": "query2",
                "query": "p99:kubelet_pod_worker_duration_seconds{$cluster,$instance,job:kubelet,metrics_path:/metrics} by {instance}",
                "data_source": "metrics"
              }
            ],
            "formulas": [
              {
                "formula": "query1"
              },
              {
                "formula": "query2"
              }
            ],
            "display_type": "area",
            "response_format": "timeseries"
          }
        ],
        "show_legend": true,
        "legend_layout": "auto",
        "legend_columns": [
          "avg",
          "min",
          "max",
          "value",
          "sum"
        ]
      }
    },
    {
      "layout": {
        "x": 0,
        "y": 8,
        "width": 6,
        "height": 2
      },
      "definition": {
        "type": "timeseries",
        "title": "Storage Operation Rate",
        "yaxis": {
          "max": "auto",
          "min": "auto",
          "label": "",
          "scale": "linear",
          "include_zero": true
        },
        "requests": [
          {
            "style": {
              "palette": "dog_classic",
              "line_type": "solid",
              "line_width": "normal"
            },
            "queries": [
              {
                "name": "query1",
                "query": "sum:storage_operation_duration_seconds.count{$cluster,$instance,job:kubelet,metrics_path:/metrics} by {instance,operation_name,volume_plugin}.as_rate()",
                "data_source": "metrics"
              }
            ],
            "formulas": [
              {
                "formula": "query1"
              }
            ],
            "display_type": "area",
            "response_format": "timeseries"
          }
        ],
        "show_legend": true,
        "legend_layout": "auto",
        "legend_columns": [
          "avg",
          "min",
          "max",
          "value",
          "sum"
        ]
      }
    },
    {
      "layout": {
        "x": 6,
        "y": 8,
        "width": 6,
        "height": 2
      },
      "definition": {
        "type": "timeseries",
        "title": "Storage Operation Error Rate",
        "yaxis": {
          "max": "auto",
          "min": "auto",
          "label": "",
          "scale": "linear",
          "include_zero": true
        },
        "requests": [
          {
            "style": {
              "palette": "dog_classic",
              "line_type": "solid",
              "line_width": "normal"
            },
            "queries": [
              {
                "name": "query1",
                "query": "sum:storage_operation_errors.count{$cluster,$instance,job:kubelet,metrics_path:/metrics} by {instance,operation_name,volume_plugin}.as_rate()",
                "data_source": "metrics"
              }
            ],
            "formulas": [
              {
                "formula": "query1"
              }
            ],
            "display_type": "area",
            "response_format": "timeseries"
          }
        ],
        "show_legend": true,
        "legend_layout": "auto",
        "legend_columns": [
          "avg",
          "min",
          "max",
          "value",
          "sum"
        ]
      }
    },
    {
      "layout": {
        "x": 0,
        "y": 10,
        "width": 12,
        "height": 2
      },
      "definition": {
        "type": "timeseries",
        "title": "Storage Operation Duration 99th quantile",
        "yaxis": {
          "max": "auto",
          "min": "auto",
          "label": "",
          "scale": "linear",
          "include_zero": true
        },
        "requests": [
          {
            "style": {
              "palette": "dog_classic",
              "line_type": "solid",
              "line_width": "normal"
            },
            "queries": [
              {
                "name": "query1",
                "query": "p99:storage_operation_duration_seconds{$cluster,$instance,job:kubelet,metrics_path:/metrics} by {instance,operation_name,volume_plugin}",
                "data_source": "metrics"
              }
            ],
            "formulas": [
              {
                "formula": "query1"
              }
            ],
            "display_type": "area",
            "response_format": "timeseries"
          }
        ],
        "show_legend": true,
        "legend_layout": "auto",
        "legend_columns": [
          "avg",
          "min",
          "max",
          "value",
          "sum"
        ]
      }
    },
    {
      "layout": {
        "x": 0,
        "y": 12,
        "width": 6,
        "height": 2
      },
      "definition": {
        "type": "timeseries",
        "title": "Cgroup manager operation rate",
        "yaxis": {
          "max": "auto",
          "min": "auto",
          "label": "",
          "scale": "linear",
          "include_zero": true
        },
        "requests": [
          {
            "style": {
              "palette": "dog_classic",
              "line_type": "solid",
              "line_width": "normal"
            },
            "queries": [
              {
                "name": "query1",
                "query": "sum:kubelet_cgroup_manager_duration_seconds.count{$cluster,$instance,job:kubelet,metrics_path:/metrics} by {instance,operation_type}.as_rate()",
                "data_source": "metrics"
              }
            ],
            "formulas": [
              {
                "formula": "query1"
              }
            ],
            "display_type": "area",
            "response_format": "timeseries"
          }
        ],
        "show_legend": true,
        "legend_layout": "auto",
        "legend_columns": [
          "avg",
          "min",
          "max",
          "value",
          "sum"
        ]
      }
    },
    {
      "layout": {
        "x": 6,
        "y": 12,
        "width": 6,
        "height": 2
      },
      "definition": {
        "type": "timeseries",
        "title": "Cgroup manager 99th quantile",
        "yaxis": {
          "max": "auto",
          "min": "auto",
          "label": "",
          "scale": "linear",
          "include_zero": true
        },
        "requests": [
          {
            "style": {
              "palette": "dog_classic",
              "line_type": "solid",
              "line_width": "normal"
            },
            "queries": [
              {
                "name": "query1",
                "query": "p99:kubelet_cgroup_manager_duration_seconds{$cluster,$instance,job:kubelet,metrics_path:/metrics} by {instance,operation_type}",
                "data_source": "metrics"
              }
            ],
            "formulas": [
              {
                "formula": "query1"
              }
            ],
            "display_type": "area",
            "response_format": "timeseries"
          }
        ],
        "show_legend": true,
        "legend_layout": "auto",
        "legend_columns": [
          "avg",
          "min",
          "max",
          "value",
          "sum"
        ]
      }
    },
    {
      "layout": {
        "x": 0,
        "y": 14,
        "width": 6,
        "height": 2
      },
      "definition": {
        "type": "timeseries",
        "title": "PLEG relist rate",
        "yaxis": {
          "max": "auto",
          "min": "auto",
          "label": "",
          "scale": "linear",
          "include_zero": true
        },
        "requests": [
          {
            "style": {
              "palette": "dog_classic",
              "line_type": "solid",
              "line_width": "normal"
            },
            "queries": [
              {
                "name": "query1",
                "query": "sum:kubelet_pleg_relist_duration_seconds.count{$cluster,$instance,job:kubelet,metrics_path:/metrics} by {instance}.as_rate()",
                "data_source": "metrics"
              }
            ],
            "formulas": [
              {
                "formula": "query1"
              }
            ],
            "display_type": "area",
            "response_format": "timeseries"
          }
        ],
        "show_legend": true,
        "legend_layout": "auto",
        "legend_columns": [
          "avg",
          "min",
          "max",
          "value",
          "sum"
        ]
      }
    },
    {
      "layout": {
        "x": 6,
        "y": 14,
        "width": 6,
        "height": 2
      },
      "definition": {
        "type": "timeseries",
        "title": "PLEG relist interval",
        "yaxis": {
          "max": "auto",
          "min": "auto",
          "label": "",
          "scale": "linear",
          "include_zero": true
        },
        "requests": [
          {
            "style": {
              "palette": "dog_classic",
              "line_type": "solid",
              "line_width": "normal"
            },
            "queries": [
              {
                "name": "query1",
                "query": "p99:kubelet_pleg_relist_interval_seconds{$cluster,$instance,job:kubelet,metrics_path:/metrics} by {instance}",
                "data_source": "metrics"
              }
            ],
            "formulas": [
              {
                "formula": "query1"
              }
            ],
            "display_type": "area",
            "response_format": "timeseries"
          }
        ],
        "show_legend": true,
        "legend_layout": "auto",
        "legend_columns": [
          "avg",
          "min",
          "max",
          "value",
          "sum"
        ]
      }
    },
    {
      "layout": {
        "x": 0,
        "y": 16,
        "width": 12,
        "height": 2
      },
      "definition": {
        "type": "timeseries",
        "title": "PLEG relist duration",
        "yaxis": {
          "max": "auto",
          "min": "auto",
          "label": "",
          "scale": "linear",
          "include_zero": true
        },
        "requests": [
          {
            "style": {
              "palette": "dog_classic",
              "line_type": "solid",
              "line_width": "normal"
            },
            "queries": [
              {
                "name": "query1",
                "query": "p99:kubernetes.kubelet.pleg.relist_duration{$cluster,$instance} by {host}",
                "data_source": "metrics"
              }
            ],
            "formulas": [
              {
                "formula": "query1"
              }
            ],
            "display_type": "area",
            "response_format": "timeseries"
          }
        ],
        "show_legend": true,
        "legend_layout": "auto",
        "legend_columns": [
          "avg",
          "min",
          "max",
          "value",
          "sum"
        ]
      }
    },
    {
      "layout": {
        "x": 0,
        "y": 18,
        "width": 12,
        "height": 2
      },
      "definition": {
        "type": "timeseries",
        "title": "RPC Rate",
        "yaxis": {
          "max": "auto",
          "min": "auto",
          "label": "",
          "scale": "linear",
          "include_zero": true
        },
        "requests": [
          {
            "style": {
              "palette": "dog_classic",
              "line_type": "solid",
              "line_width": "normal"
            },
            "queries": [
              {
                "name": "query1",
                "query": "sum:rest_client_requests.count{$cluster,$instance,job:kubelet,metrics_path:/metrics,code:2*}.as_rate()",
                "data_source": "metrics"
              },
              {
                "name": "query2",
                "query": "sum:rest_client_requests.count{$cluster,$instance,job:kubelet,metrics_path:/metrics,code:3*}.as_rate()",
                "data_source": "metrics"
              },
              {
                "name": "query3",
                "query": "sum:rest_client_requests.count{$cluster,$instance,job:kubelet,metrics_path:/metrics,code:4*}.as_rate()",
                "data_source": "metrics"
              },
              {
                "name": "query4",
                "query": "sum:rest_client_requests.count{$cluster,$instance,job:kubelet,metrics_path:/metrics,code:5*}.as_rate()",
                "data_source": "metrics"
              }
            ],
            "formulas": [
              {
                "alias": "2xx",
                "formula": "query1"
              },
              {
                "alias": "3xx",
                "formula": "query2"
              },
              {
                "alias": "4xx",
                "formula": "query3"
              },
              {
                "alias": "5xx",
                "formula": "query4"
              }
            ],
            "display_type": "area",
            "response_format": "timeseries"
          }
        ],
        "show_legend": true,
        "legend_layout": "auto",
        "legend_columns": [
          "avg",
          "min",
          "max",
          "value",
          "sum"
        ]
      }
    },
    {
      "layout": {
        "x": 0,
        "y": 20,
        "width": 12,
        "height": 2
      },
      "definition": {
        "type": "timeseries",
        "title": "Request duration 99th quantile",
        "yaxis": {
          "max": "auto",
          "min": "auto",
          "label": "",
          "scale": "linear",
          "include_zero": true
        },
        "requests": [
          {
            "style": {
              "palette": "dog_classic",
              "line_type": "solid",
              "line_width": "normal"
            },
            "queries": [
              {
                "name": "query1",
                "query": "p99:rest_client_request_latency_seconds{$cluster,$instance,job:kubelet,metrics_path:/metrics} by {instance,verb,url}",
                "data_source": "metrics"
              }
            ],
            "formulas": [
              {
                "formula": "query1"
              }
            ],
            "display_type": "area",
            "response_format": "timeseries"
          }
        ],
        "show_legend": true,
        "legend_layout": "auto",
        "legend_columns": [
          "avg",
          "min",
          "max",
          "value",
          "sum"
        ]
      }
    },
    {
      "layout": {
        "x": 0,
        "y": 22,
        "width": 4,
        "height": 2
      },
      "definition": {
        "type": "timeseries",
        "title": "Memory",
        "yaxis": {
          "max": "auto",
          "min": "auto",
          "label": "",
          "scale": "linear",
          "include_zero": true
        },
        "requests": [
          {
            "style": {
              "palette": "dog_classic",
              "line_type": "solid",
              "line_width": "normal"
            },
            "queries": [
              {
                "name": "query1",
                "query": "avg:process_resident_memory_bytes{$cluster,$instance,job:kubelet,metrics_path:/metrics} by {instance}",
                "data_source": "metrics"
              }
            ],
            "formulas": [
              {
                "formula": "query1"
              }
            ],
            "display_type": "area",
            "response_format": "timeseries"
          }
        ],
        "show_legend": true,
        "legend_layout": "auto",
        "legend_columns": [
          "avg",
          "min",
          "max",
          "value",
          "sum"
        ]
      }
    },
    {
      "layout": {
        "x": 4,
        "y": 22,
        "width": 4,
        "height": 2
      },
      "definition": {
        "type": "timeseries",
        "title": "CPU usage",
        "yaxis": {
          "max": "auto",
          "min": "auto",
          "label": "",
          "scale": "linear",
          "include_zero": true
        },
        "requests": [
          {
            "style": {
              "palette": "dog_classic",
              "line_type": "solid",
              "line_width": "normal"
            },
            "queries": [
              {
                "name": "query1",
                "query": "avg:process_cpu_seconds.count{$cluster,$instance,job:kubelet,metrics_path:/metrics} by {instance}.as_rate()",
                "data_source": "metrics"
              }
            ],
            "formulas": [
              {
                "formula": "query1"
              }
            ],
            "display_type": "area",
            "response_format": "timeseries"
          }
        ],
        "show_legend": true,
        "legend_layout": "auto",
        "legend_columns": [
          "avg",
          "min",
          "max",
          "value",
          "sum"
        ]
      }
    },
    {
      "layout": {
        "x": 8,
        "y": 22,
        "width": 4,
        "height": 2
      },
      "definition": {
        "type": "timeseries",
        "title": "Goroutines",
        "yaxis": {
          "max": "auto",
          "min": "auto",
          "label": "",
          "scale": "linear",
          "include_zero": true
        },
        "requests": [
          {
            "style": {
              "palette": "dog_classic",
              "line_type": "solid",
              "line_width": "normal"
            },
            "queries": [
              {
                "name": "query1",
                "query": "avg:go_goroutines{$cluster,$instance,job:kubelet,metrics_path:/metrics} by {instance}",
                "data_source": "metrics"
              }
            ],
            "formulas": [
              {
                "formula": "query1"
              }
            ],
            "display_type": "area",
            "response_format": "timeseries"
          }
        ],
        "show_legend": true,
        "legend_layout": "auto",
        "legend_columns": [
          "avg",
          "min",
          "max",
          "value",
          "sum"
        ]
      }
    }
  ],
  "description": "Converted from Grafana dashboard https://grafana.com/grafana/dashboards/12123 (by andreynovik1993). prometheus operator ",
  "layout_type": "ordered",
  "reflow_type": "fixed",
  "template_variables": [
    {
      "name": "cluster",
      "prefix": "cluster",
      "default": "*",
      "available_values": []
    },
    {
      "name": "instance",
      "prefix": "host",
      "default": "*",
      "available_values": []
    }
  ]
}