{
  "tags": [],
  "title": "Prometheus",
  "widgets": [
    {
      "layout": {
        "x": 0,
        "y": 0,
        "width": 12,
        "height": 1
      },
      "definition": {
        "type": "group",
        "title": "Information",
        "widgets": [
          {
            "layout": {
              "x": 0,
              "y": 0,
              "width": 3,
              "height": 1
            },
            "definition": {
              "type": "query_value",
              "title": "Prometheus version",
              "requests": [
                {
                  "queries": [
                    {
                      "name": "query1",
                      "query": "avg:prometheus_build_info{$pod,$cluster} by {version}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1"
                    }
                  ],
                  "response_format": "scalar"
                }
              ],
              "autoscale": true,
              "precision": 2,
              "text_align": "center"
            }
          },
          {
            "layout": {
              "x": 3,
              "y": 0,
              "width": 3,
              "height": 1
            },
            "definition": {
              "type": "note",
              "content": "**Instance Down**\n\nUnsupported panel (stat)\n\n- Prometheus `up` has no Datadog metric; use the openmetrics.health service check\n\nOriginal PromQL:\n```\nA: up{pod=~\"$pod\", cluster=~\"$cluster\"} < 1\n```",
              "tick_pos": "50%",
              "font_size": "14",
              "show_tick": false,
              "tick_edge": "left",
              "text_align": "left",
              "has_padding": true,
              "vertical_align": "top",
              "background_color": "white"
            }
          },
          {
            "layout": {
              "x": 6,
              "y": 0,
              "width": 3,
              "height": 1
            },
            "definition": {
              "type": "query_value",
              "title": "TSDB Head Series",
              "requests": [
                {
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:prometheus_tsdb_head_series{$pod,$cluster} by {pod}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1"
                    }
                  ],
                  "response_format": "scalar"
                }
              ],
              "autoscale": true,
              "precision": 2,
              "text_align": "center",
              "timeseries_background": {
                "type": "area"
              }
            }
          },
          {
            "layout": {
              "x": 9,
              "y": 0,
              "width": 3,
              "height": 1
            },
            "definition": {
              "type": "query_value",
              "title": "Discovered Targets",
              "requests": [
                {
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:prometheus_sd_discovered_targets{$pod,$cluster} by {pod}",
                      "aggregator": "last",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1"
                    }
                  ],
                  "response_format": "scalar"
                }
              ],
              "autoscale": true,
              "precision": 2,
              "text_align": "center"
            }
          }
        ],
        "show_title": true,
        "layout_type": "ordered"
      }
    },
    {
      "layout": {
        "x": 0,
        "y": 1,
        "width": 12,
        "height": 6
      },
      "definition": {
        "type": "group",
        "title": "Prometheus",
        "widgets": [
          {
            "layout": {
              "x": 0,
              "y": 0,
              "width": 6,
              "height": 2
            },
            "definition": {
              "type": "note",
              "content": "**Liveness by pod**\n\nUnsupported panel (timeseries)\n\n- Prometheus `up` has no Datadog metric; use the openmetrics.health service check\n\nOriginal PromQL:\n```\nA: up{pod=~\"$pod\", cluster=~\"$cluster\"}\n```",
              "tick_pos": "50%",
              "font_size": "14",
              "show_tick": false,
              "tick_edge": "left",
              "text_align": "left",
              "has_padding": true,
              "vertical_align": "top",
              "background_color": "white"
            }
          },
          {
            "layout": {
              "x": 6,
              "y": 0,
              "width": 6,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "Config - Last Successful Reload by pod",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": true
              },
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:prometheus_config_last_reload_successful{$pod,$cluster} by {pod}",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1"
                    }
                  ],
                  "display_type": "area",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          },
          {
            "layout": {
              "x": 0,
              "y": 2,
              "width": 6,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "Target Scrapes Errors by pod",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": true
              },
              "markers": [
                {
                  "value": "y = 80",
                  "display_type": "error dashed"
                }
              ],
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:prometheus_target_scrapes_exceeded_body_size_limit.count{$pod,$cluster} by {pod}.as_rate()",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query2",
                      "query": "sum:prometheus_target_scrapes_exceeded_sample_limit.count{$pod,$cluster} by {pod}.as_rate()",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query3",
                      "query": "sum:prometheus_target_scrapes_sample_duplicate_timestamp.count{$pod,$cluster} by {pod}.as_rate()",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query4",
                      "query": "sum:prometheus_target_scrapes_sample_out_of_bounds.count{$pod,$cluster} by {pod}.as_rate()",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query5",
                      "query": "sum:prometheus_target_scrapes_sample_out_of_order.count{$pod,$cluster} by {pod}.as_rate()",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1"
                    },
                    {
                      "formula": "query2"
                    },
                    {
                      "formula": "query3"
                    },
                    {
                      "formula": "query4"
                    },
                    {
                      "formula": "query5"
                    }
                  ],
                  "display_type": "area",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          },
          {
            "layout": {
              "x": 6,
              "y": 2,
              "width": 6,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "Number of Targets by pod",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": true
              },
              "markers": [
                {
                  "value": "y = 80",
                  "display_type": "error dashed"
                }
              ],
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:prometheus_sd_discovered_targets{$pod,$cluster} by {pod}",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1"
                    }
                  ],
                  "display_type": "area",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          },
          {
            "layout": {
              "x": 0,
              "y": 4,
              "width": 6,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "Target Sync by pod, scrape_job",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": true
              },
              "markers": [
                {
                  "value": "y = 80",
                  "display_type": "error dashed"
                }
              ],
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:prometheus_target_sync_length_seconds.sum{$pod,$cluster} by {pod,scrape_job}.as_rate()",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1 * 1000"
                    }
                  ],
                  "display_type": "area",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          },
          {
            "layout": {
              "x": 6,
              "y": 4,
              "width": 6,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "Average Scrape Interval by pod",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": true
              },
              "markers": [
                {
                  "value": "y = 80",
                  "display_type": "error dashed"
                }
              ],
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:prometheus_target_interval_length_seconds.sum{$pod,$cluster} by {pod}.as_rate()",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query2",
                      "query": "sum:prometheus_target_interval_length_seconds.count{$pod,$cluster} by {pod}.as_rate()",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1 / query2"
                    }
                  ],
                  "display_type": "area",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          }
        ],
        "show_title": true,
        "layout_type": "ordered"
      }
    },
    {
      "layout": {
        "x": 0,
        "y": 7,
        "width": 12,
        "height": 8
      },
      "definition": {
        "type": "group",
        "title": "Prometheus TSDB / Query Engine",
        "widgets": [
          {
            "layout": {
              "x": 0,
              "y": 0,
              "width": 6,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "TSDB Head Series & Chunks by pod",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": true
              },
              "markers": [
                {
                  "value": "y = 80",
                  "display_type": "error dashed"
                }
              ],
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:prometheus_tsdb_head_series{$pod,$cluster} by {pod}",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query2",
                      "query": "sum:prometheus_tsdb_head_chunks{$pod,$cluster} by {pod}",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1"
                    },
                    {
                      "formula": "query2"
                    }
                  ],
                  "display_type": "area",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          },
          {
            "layout": {
              "x": 6,
              "y": 0,
              "width": 6,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "TSDB Head samples appended - rate by pod",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": true
              },
              "markers": [
                {
                  "value": "y = 80",
                  "display_type": "error dashed"
                }
              ],
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:prometheus_tsdb_head_samples_appended.count{$pod,$cluster} by {pod}.as_rate()",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1"
                    }
                  ],
                  "display_type": "area",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          },
          {
            "layout": {
              "x": 0,
              "y": 2,
              "width": 6,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "TSDB Blocks Loaded by pod",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": true
              },
              "markers": [
                {
                  "value": "y = 80",
                  "display_type": "error dashed"
                }
              ],
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:prometheus_tsdb_blocks_loaded{$pod,$cluster} by {pod}",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1"
                    }
                  ],
                  "display_type": "area",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          },
          {
            "layout": {
              "x": 6,
              "y": 2,
              "width": 6,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "TSDB Rate of Compactions by pod",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": true
              },
              "markers": [
                {
                  "value": "y = 80",
                  "display_type": "error dashed"
                }
              ],
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:prometheus_tsdb_compactions.count{$pod,$cluster} by {pod}.as_rate()",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query2",
                      "query": "sum:prometheus_tsdb_compactions_triggered.count{$pod,$cluster} by {pod}.as_rate()",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query3",
                      "query": "sum:prometheus_tsdb_compactions_skipped.count{$pod,$cluster} by {pod}.as_rate()",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query4",
                      "query": "sum:prometheus_tsdb_compactions_failed.count{$pod,$cluster} by {pod}.as_rate()",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1"
                    },
                    {
                      "formula": "query2"
                    },
                    {
                      "formula": "query3"
                    },
                    {
                      "formula": "query4"
                    }
                  ],
                  "display_type": "area",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          },
          {
            "layout": {
              "x": 0,
              "y": 4,
              "width": 6,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "TSDB Reload Failures by pod",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": true
              },
              "markers": [
                {
                  "value": "y = 80",
                  "display_type": "error dashed"
                }
              ],
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:prometheus_tsdb_reloads_failures.count{$pod,$cluster} by {pod}.as_rate()",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1"
                    }
                  ],
                  "display_type": "area",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          },
          {
            "layout": {
              "x": 6,
              "y": 4,
              "width": 6,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "TSDB Created & Deleted series by pod",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": true
              },
              "markers": [
                {
                  "value": "y = 80",
                  "display_type": "error dashed"
                }
              ],
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:prometheus_tsdb_head_series_created.count{$pod,$cluster} by {pod}.as_rate()",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query2",
                      "query": "sum:prometheus_tsdb_head_series_removed.count{$pod,$cluster} by {pod}.as_rate()",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1"
                    },
                    {
                      "formula": "query2"
                    }
                  ],
                  "display_type": "area",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          },
          {
            "layout": {
              "x": 0,
              "y": 6,
              "width": 6,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "Engine Query Count by pod",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": true
              },
              "markers": [
                {
                  "value": "y = 80",
                  "display_type": "error dashed"
                }
              ],
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:prometheus_engine_query_duration_seconds.count{$pod,$cluster,slice:inner_eval} by {pod}.as_rate()",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1"
                    }
                  ],
                  "display_type": "area",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          },
          {
            "layout": {
              "x": 6,
              "y": 6,
              "width": 6,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "Engine Query Duration by pod, slice",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": true
              },
              "markers": [
                {
                  "value": "y = 80",
                  "display_type": "error dashed"
                }
              ],
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "max:prometheus_engine_query_duration_seconds.quantile{$pod,$cluster} by {pod,slice}",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1 * 1000"
                    }
                  ],
                  "display_type": "area",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          }
        ],
        "show_title": true,
        "layout_type": "ordered"
      }
    },
    {
      "layout": {
        "x": 0,
        "y": 15,
        "width": 12,
        "height": 2
      },
      "definition": {
        "type": "group",
        "title": "Resources",
        "widgets": [
          {
            "layout": {
              "x": 0,
              "y": 0,
              "width": 6,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "CPU Usage by pod, container",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": true
              },
              "markers": [
                {
                  "value": "y = 80",
                  "display_type": "error dashed"
                }
              ],
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:kubernetes.cpu.usage.total{$pod,$cluster,kube_container_name:*} by {pod_name,kube_container_name}",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1 / 1000000000"
                    }
                  ],
                  "display_type": "area",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          },
          {
            "layout": {
              "x": 6,
              "y": 0,
              "width": 6,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "Memory Usage by container",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": true
              },
              "markers": [
                {
                  "value": "y = 80",
                  "display_type": "error dashed"
                }
              ],
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:kubernetes.memory.working_set{$pod,$cluster,kube_container_name:*} by {pod_name,kube_container_name}",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1"
                    }
                  ],
                  "display_type": "area",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          }
        ],
        "show_title": true,
        "layout_type": "ordered"
      }
    },
    {
      "layout": {
        "x": 0,
        "y": 17,
        "width": 12,
        "height": 4
      },
      "definition": {
        "type": "group",
        "title": "Storage",
        "widgets": [
          {
            "layout": {
              "x": 0,
              "y": 0,
              "width": 6,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "Persistent Volumes - Capacity and usage in %",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": true
              },
              "markers": [
                {
                  "value": "y = 80",
                  "display_type": "error dashed"
                }
              ],
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "max:kubernetes.kubelet.volume.stats.used_bytes{$cluster} by {persistentvolumeclaim,kube_namespace}",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query2",
                      "query": "max:kubernetes.kubelet.volume.stats.capacity_bytes{$cluster} by {persistentvolumeclaim,kube_namespace}",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "(query1 / query2) * 100"
                    }
                  ],
                  "display_type": "area",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          },
          {
            "layout": {
              "x": 6,
              "y": 0,
              "width": 6,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "Persistent Volumes - Capacity and usage in bytes",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": true
              },
              "markers": [
                {
                  "value": "y = 80",
                  "display_type": "error dashed"
                }
              ],
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "max:kubernetes.kubelet.volume.stats.used_bytes{$cluster} by {persistentvolumeclaim,kube_namespace}",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query2",
                      "query": "max:kubernetes.kubelet.volume.stats.capacity_bytes{$cluster} by {persistentvolumeclaim,kube_namespace}",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1"
                    },
                    {
                      "formula": "query2"
                    }
                  ],
                  "display_type": "area",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          },
          {
            "layout": {
              "x": 0,
              "y": 2,
              "width": 6,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "Persistent Volumes - Inodes",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": true
              },
              "markers": [
                {
                  "value": "y = 80",
                  "display_type": "error dashed"
                }
              ],
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:kubernetes.kubelet.volume.stats.inodes_used{$cluster} by {persistentvolumeclaim}",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query2",
                      "query": "sum:kubernetes.kubelet.volume.stats.inodes{$cluster} by {persistentvolumeclaim}",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "(1 - (query1 / query2)) * 100"
                    }
                  ],
                  "display_type": "area",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          }
        ],
        "show_title": true,
        "layout_type": "ordered"
      }
    },
    {
      "layout": {
        "x": 0,
        "y": 21,
        "width": 12,
        "height": 4
      },
      "definition": {
        "type": "group",
        "title": "Network",
        "widgets": [
          {
            "layout": {
              "x": 0,
              "y": 0,
              "width": 6,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "Network - Bandwidth by pod",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": true
              },
              "markers": [
                {
                  "value": "y = 80",
                  "display_type": "error dashed"
                }
              ],
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:kubernetes.network.rx_bytes{$pod,$cluster} by {pod_name}",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query2",
                      "query": "sum:kubernetes.network.tx_bytes{$pod,$cluster} by {pod_name}",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1"
                    },
                    {
                      "formula": "-query2"
                    }
                  ],
                  "display_type": "area",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          },
          {
            "layout": {
              "x": 6,
              "y": 0,
              "width": 6,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "Network - Packets rate by pod",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": true
              },
              "markers": [
                {
                  "value": "y = 80",
                  "display_type": "error dashed"
                }
              ],
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:container_network_receive_packets.count{$pod,$cluster} by {pod}.as_rate()",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query2",
                      "query": "sum:container_network_transmit_packets.count{$pod,$cluster} by {pod}.as_rate()",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1"
                    },
                    {
                      "formula": "-query2"
                    }
                  ],
                  "display_type": "area",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          },
          {
            "layout": {
              "x": 0,
              "y": 2,
              "width": 6,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "Network - Packets Dropped by pod",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": true
              },
              "markers": [
                {
                  "value": "y = 80",
                  "display_type": "error dashed"
                }
              ],
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:kubernetes.network.rx_dropped{$pod,$cluster} by {pod_name}",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query2",
                      "query": "sum:kubernetes.network.tx_dropped{$pod,$cluster} by {pod_name}",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1"
                    },
                    {
                      "formula": "-query2"
                    }
                  ],
                  "display_type": "area",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          },
          {
            "layout": {
              "x": 6,
              "y": 2,
              "width": 6,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "Network - Errors by pod",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": true
              },
              "markers": [
                {
                  "value": "y = 80",
                  "display_type": "error dashed"
                }
              ],
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:kubernetes.network.rx_errors{$pod,$cluster} by {pod_name}",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query2",
                      "query": "sum:kubernetes.network.tx_errors{$pod} by {pod_name}",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1"
                    },
                    {
                      "formula": "query2 * -1"
                    }
                  ],
                  "display_type": "area",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          }
        ],
        "show_title": true,
        "layout_type": "ordered"
      }
    }
  ],
  "description": "Converted from Grafana dashboard https://grafana.com/grafana/dashboards/19105 (by dotdc). This is a modern 'Prometheus' dashboard for your Kubernetes cluster(s). Made for kube-prometheus-stack and take advantage of the latest Grafana features. GitHub repository: https://github.com/dotdc/grafana-dashboards-kubernetes",
  "layout_type": "ordered",
  "reflow_type": "fixed",
  "template_variables": [
    {
      "name": "cluster",
      "prefix": "kube_cluster_name",
      "default": "*",
      "available_values": []
    },
    {
      "name": "pod",
      "prefix": "pod_name",
      "default": "*",
      "available_values": []
    },
    {
      "name": "resolution",
      "prefix": "resolution",
      "default": "30s",
      "available_values": [
        "1s",
        "15s",
        "30s",
        "1m",
        "3m",
        "5m"
      ]
    }
  ]
}