{
  "tags": [],
  "title": "K8S Cluster Monitor",
  "widgets": [
    {
      "layout": {
        "x": 0,
        "y": 0,
        "width": 12,
        "height": 2
      },
      "definition": {
        "type": "group",
        "title": "Status",
        "widgets": [
          {
            "layout": {
              "x": 0,
              "y": 0,
              "width": 4,
              "height": 1
            },
            "definition": {
              "type": "query_value",
              "title": "Pods: Ready",
              "requests": [
                {
                  "queries": [
                    {
                      "name": "query1",
                      "query": "avg:kubernetes_state.container.ready{kube_namespace:kube-system}.rollup(avg, 300)",
                      "aggregator": "last",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "(query1) * 100"
                    }
                  ],
                  "response_format": "scalar",
                  "conditional_formats": [
                    {
                      "value": 1,
                      "palette": "white_on_green",
                      "comparator": ">="
                    }
                  ]
                }
              ],
              "autoscale": true,
              "precision": 2,
              "text_align": "center",
              "custom_unit": "%"
            }
          },
          {
            "layout": {
              "x": 4,
              "y": 0,
              "width": 8,
              "height": 2
            },
            "definition": {
              "type": "query_table",
              "title": "Node Info",
              "requests": [
                {
                  "queries": [
                    {
                      "name": "query1",
                      "query": "avg:kubernetes_state.node.count{*} by {host}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query2",
                      "query": "sum:container_spec_memory_reservation_limit_bytes{pod:*} by {node}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query3",
                      "query": "max:kubernetes.pods.running{*} by {host}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query4",
                      "query": "sum:instance_node_cpu_ratio{*} by {node}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query5",
                      "query": "sum:kubernetes_state.node.cpu_capacity{*} by {node}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query6",
                      "query": "avg:kube_node_role{*}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query7",
                      "query": "avg:system.uptime{*} by {host}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "limit": {
                        "count": 50,
                        "order": "desc"
                      },
                      "formula": "query1"
                    },
                    {
                      "formula": "count_not_null(query2)"
                    },
                    {
                      "formula": "query3"
                    },
                    {
                      "formula": "query4 / query5"
                    },
                    {
                      "formula": "query6"
                    },
                    {
                      "formula": "query7"
                    }
                  ],
                  "response_format": "scalar"
                }
              ],
              "has_search_bar": "auto"
            }
          },
          {
            "layout": {
              "x": 0,
              "y": 1,
              "width": 1,
              "height": 1
            },
            "definition": {
              "type": "query_value",
              "title": "Namespaces",
              "requests": [
                {
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:kube_namespace_created{*} by {host}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "count_not_null(query1)"
                    }
                  ],
                  "response_format": "scalar",
                  "conditional_formats": [
                    {
                      "value": 80,
                      "palette": "white_on_red",
                      "comparator": ">="
                    }
                  ]
                }
              ],
              "autoscale": true,
              "precision": 2,
              "text_align": "center"
            }
          },
          {
            "layout": {
              "x": 1,
              "y": 1,
              "width": 2,
              "height": 1
            },
            "definition": {
              "type": "query_value",
              "title": "Pods",
              "requests": [
                {
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:container_spec_memory_reservation_limit_bytes{pod:*} by {pod}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "count_not_null(query1)"
                    }
                  ],
                  "response_format": "scalar",
                  "conditional_formats": [
                    {
                      "value": 80,
                      "palette": "white_on_red",
                      "comparator": ">="
                    }
                  ]
                }
              ],
              "autoscale": true,
              "precision": 2,
              "text_align": "center"
            }
          },
          {
            "layout": {
              "x": 3,
              "y": 1,
              "width": 1,
              "height": 1
            },
            "definition": {
              "type": "query_value",
              "title": "PVC",
              "requests": [
                {
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:kube_persistentvolumeclaim_info{*} by {host}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "count_not_null(query1)"
                    }
                  ],
                  "response_format": "scalar",
                  "conditional_formats": [
                    {
                      "value": 80,
                      "palette": "white_on_red",
                      "comparator": ">="
                    }
                  ]
                }
              ],
              "autoscale": true,
              "precision": 2,
              "text_align": "center"
            }
          }
        ],
        "show_title": true,
        "layout_type": "ordered"
      }
    },
    {
      "layout": {
        "x": 0,
        "y": 2,
        "width": 12,
        "height": 2
      },
      "definition": {
        "type": "group",
        "title": "Nodes",
        "widgets": [
          {
            "layout": {
              "x": 0,
              "y": 0,
              "width": 3,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "CPU / Node",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": true
              },
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "avg:system.cpu.user{*} by {host}",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query2",
                      "query": "avg:system.cpu.system{*} by {host}",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query3",
                      "query": "avg:system.cpu.interrupt{*} by {host}",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query4",
                      "query": "avg:system.cpu.stolen{*} by {host}",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "((query1 + query2 + query3 + query4) / 100) * 100"
                    }
                  ],
                  "display_type": "area",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          },
          {
            "layout": {
              "x": 3,
              "y": 0,
              "width": 3,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "Memory / Node",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": false
              },
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "avg:system.mem.total{*} by {host}",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query2",
                      "query": "avg:system.mem.usable{*} by {host}",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "(query1 - query2) * 1048576"
                    }
                  ],
                  "display_type": "line",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          },
          {
            "layout": {
              "x": 6,
              "y": 0,
              "width": 3,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "Network In/out",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": true
              },
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:system.net.bytes_rcvd{*}",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query2",
                      "query": "sum:system.net.bytes_sent{*}",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1"
                    },
                    {
                      "formula": "query2"
                    }
                  ],
                  "display_type": "bars",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          },
          {
            "layout": {
              "x": 9,
              "y": 0,
              "width": 3,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "K8S API-Server Usage",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": true
              },
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:kube_apiserver.apiserver_request_total.count{*} by {verb}.as_rate()",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1"
                    }
                  ],
                  "display_type": "bars",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          }
        ],
        "show_title": true,
        "layout_type": "ordered"
      }
    },
    {
      "layout": {
        "x": 0,
        "y": 4,
        "width": 12,
        "height": 2
      },
      "definition": {
        "type": "group",
        "title": "$node",
        "widgets": [
          {
            "layout": {
              "x": 0,
              "y": 0,
              "width": 2,
              "height": 2
            },
            "definition": {
              "type": "query_value",
              "title": "CPU / Node",
              "requests": [
                {
                  "queries": [
                    {
                      "name": "query1",
                      "query": "avg:system.cpu.user{$node}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query2",
                      "query": "avg:system.cpu.system{$node}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query3",
                      "query": "avg:system.cpu.interrupt{$node}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query4",
                      "query": "avg:system.cpu.stolen{$node}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "((query1 + query2 + query3 + query4) / 100) * 100"
                    }
                  ],
                  "response_format": "scalar",
                  "conditional_formats": [
                    {
                      "value": 0.6,
                      "palette": "white_on_yellow",
                      "comparator": ">="
                    },
                    {
                      "value": 0.7,
                      "palette": "white_on_red",
                      "comparator": ">="
                    }
                  ]
                }
              ],
              "autoscale": true,
              "precision": 2,
              "text_align": "center",
              "custom_unit": "%"
            }
          },
          {
            "layout": {
              "x": 2,
              "y": 0,
              "width": 2,
              "height": 2
            },
            "definition": {
              "type": "query_value",
              "title": "Disk",
              "requests": [
                {
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:system.disk.total{$node} by {host}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query2",
                      "query": "sum:system.disk.free{$node} by {host}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "((query1 - query2) / query1) * 100"
                    }
                  ],
                  "response_format": "scalar",
                  "conditional_formats": [
                    {
                      "value": 0.8,
                      "palette": "white_on_red",
                      "comparator": ">="
                    },
                    {
                      "value": 0.9,
                      "palette": "white_on_red",
                      "comparator": ">="
                    }
                  ]
                }
              ],
              "autoscale": true,
              "precision": 2,
              "text_align": "center",
              "custom_unit": "%"
            }
          },
          {
            "layout": {
              "x": 4,
              "y": 0,
              "width": 3,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "File descriptor / Context switches",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": false
              },
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "avg:system.fs.file_handles.allocated{$instance}",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query2",
                      "query": "avg:system.cpu.context_switches{$instance}.as_rate()",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "alias": "used filefd",
                      "formula": "query1"
                    },
                    {
                      "alias": "switches",
                      "formula": "query2"
                    }
                  ],
                  "display_type": "line",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          },
          {
            "layout": {
              "x": 7,
              "y": 0,
              "width": 5,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "Disk IOPS",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": false
              },
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "avg:system.io.r_s{$instance} by {device}",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query2",
                      "query": "avg:system.io.w_s{$instance} by {device}",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query3",
                      "query": "avg:node_disk_io_now{$instance} by {device}",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1"
                    },
                    {
                      "formula": "query2"
                    },
                    {
                      "formula": "query3"
                    }
                  ],
                  "display_type": "line",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          }
        ],
        "show_title": true,
        "layout_type": "ordered"
      }
    },
    {
      "layout": {
        "x": 0,
        "y": 6,
        "width": 12,
        "height": 4
      },
      "definition": {
        "type": "group",
        "title": "Pod $pod",
        "widgets": [
          {
            "layout": {
              "x": 0,
              "y": 0,
              "width": 2,
              "height": 2
            },
            "definition": {
              "type": "query_value",
              "title": "Pod Uptime",
              "requests": [
                {
                  "queries": [
                    {
                      "name": "query1",
                      "query": "avg:kubernetes_state.pod.uptime{$pod}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1"
                    }
                  ],
                  "response_format": "scalar"
                }
              ],
              "autoscale": true,
              "precision": 2,
              "text_align": "center",
              "custom_unit": "s",
              "timeseries_background": {
                "type": "area"
              }
            }
          },
          {
            "layout": {
              "x": 2,
              "y": 0,
              "width": 3,
              "height": 2
            },
            "definition": {
              "type": "query_value",
              "title": "Pod Status",
              "requests": [
                {
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:kubernetes_state.container.status_report.count.waiting{$pod,!reason:containercreating}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1"
                    }
                  ],
                  "response_format": "scalar",
                  "conditional_formats": [
                    {
                      "value": 1,
                      "palette": "white_on_red",
                      "comparator": ">="
                    }
                  ]
                }
              ],
              "autoscale": true,
              "precision": 2,
              "text_align": "center",
              "timeseries_background": {
                "type": "area"
              }
            }
          },
          {
            "layout": {
              "x": 5,
              "y": 0,
              "width": 1,
              "height": 2
            },
            "definition": {
              "type": "query_value",
              "title": "Pod Restarts",
              "requests": [
                {
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:kubernetes_state.container.restarts{$pod}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "diff(query1)"
                    }
                  ],
                  "response_format": "scalar",
                  "conditional_formats": [
                    {
                      "value": 2,
                      "palette": "white_on_yellow",
                      "comparator": ">="
                    },
                    {
                      "value": 5,
                      "palette": "white_on_red",
                      "comparator": ">="
                    }
                  ]
                }
              ],
              "autoscale": true,
              "precision": 2,
              "text_align": "center",
              "timeseries_background": {
                "type": "area"
              }
            }
          },
          {
            "layout": {
              "x": 6,
              "y": 0,
              "width": 4,
              "height": 2
            },
            "definition": {
              "type": "toplist",
              "title": "Pod Mem/CPU/Disk Usage",
              "requests": [
                {
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:kubernetes.memory.usage{$pod}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query2",
                      "query": "sum:kubernetes.cpu.usage.total{$pod}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query3",
                      "query": "sum:kubernetes.filesystem.usage{$pod}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "alias": "Memory(bytes)",
                      "limit": {
                        "count": 10,
                        "order": "desc"
                      },
                      "formula": "query1"
                    }
                  ],
                  "response_format": "scalar"
                }
              ]
            }
          },
          {
            "layout": {
              "x": 10,
              "y": 0,
              "width": 2,
              "height": 2
            },
            "definition": {
              "type": "query_value",
              "title": "Pod Resource Limits",
              "requests": [
                {
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:kubernetes_state.container.cpu_limit{$pod}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query2",
                      "query": "sum:kubernetes_state.container.memory_limit{$pod}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "alias": "CPU core",
                      "formula": "query1"
                    }
                  ],
                  "response_format": "scalar"
                }
              ],
              "autoscale": true,
              "precision": 2,
              "text_align": "center"
            }
          },
          {
            "layout": {
              "x": 0,
              "y": 2,
              "width": 6,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "Pod Network",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": true
              },
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:kubernetes.network.rx_bytes{$pod}",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query2",
                      "query": "sum:kubernetes.network.tx_bytes{$pod}",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "alias": "in",
                      "formula": "query1"
                    },
                    {
                      "alias": "out",
                      "formula": "query2"
                    }
                  ],
                  "display_type": "bars",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          },
          {
            "layout": {
              "x": 6,
              "y": 2,
              "width": 6,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "Pod Threads",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": true
              },
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "avg:container_threads{$pod,!container:pod}",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "alias": "Thread",
                      "formula": "query1"
                    }
                  ],
                  "display_type": "area",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          }
        ],
        "show_title": true,
        "layout_type": "ordered"
      }
    }
  ],
  "description": "Converted from Grafana dashboard https://grafana.com/grafana/dashboards/14249 (by chaosengineers). English version of an Asian dashboard found on grafana.com/dashboards",
  "layout_type": "ordered",
  "reflow_type": "fixed",
  "template_variables": [
    {
      "name": "node",
      "prefix": "host",
      "default": "*",
      "available_values": []
    },
    {
      "name": "instance",
      "prefix": "host",
      "default": "*",
      "available_values": []
    },
    {
      "name": "namespace",
      "prefix": "kube_namespace",
      "default": "*",
      "available_values": []
    },
    {
      "name": "pod",
      "prefix": "pod_name",
      "default": "*",
      "available_values": []
    },
    {
      "name": "agent",
      "prefix": "agent",
      "default": "*",
      "available_values": []
    }
  ]
}