{ "tags": [], "title": "kubernetes-persistent-volumes", "widgets": [ { "layout": { "x": 0, "y": 0, "width": 12, "height": 2 }, "definition": { "type": "group", "title": "Alerts", "widgets": [ { "layout": { "x": 0, "y": 0, "width": 4, "height": 1 }, "definition": { "type": "note", "content": "**PVCs Full in 2 days - Based on Daily Usage**\n\nUnsupported panel (stat)\n\n- predict_linear() has no Datadog formula equivalent; forecast-based PVC exhaustion count cannot be expressed\n\nOriginal PromQL:\n```\nA: (\n count (\n (kubelet_volume_stats_available_bytes{namespace=~\"${k8s_namespace}\"})\n and\n (predict_linear(kubelet_volume_stats_available_bytes{namespace=~\"${k8s_namespace}\"}[1d], 2 * 24 * 60 * 60) < 0)\n )\n)\nor\nvector(0)\n```", "tick_pos": "50%", "font_size": "14", "show_tick": false, "tick_edge": "left", "text_align": "left", "has_padding": true, "vertical_align": "top", "background_color": "white" } }, { "layout": { "x": 4, "y": 0, "width": 4, "height": 1 }, "definition": { "type": "note", "content": "**PVCs Full in 5 days - Based on Daily Usage**\n\nUnsupported panel (stat)\n\n- predict_linear() has no Datadog formula equivalent; forecast-based PVC exhaustion count cannot be expressed\n\nOriginal PromQL:\n```\nA: (\n count (\n (kubelet_volume_stats_available_bytes{namespace=~\"${k8s_namespace}\"})\n and\n (predict_linear(kubelet_volume_stats_available_bytes{namespace=~\"${k8s_namespace}\"}[1d], 5 * 24 * 60 * 60) < 0)\n )\n)\nor\nvector(0)\n```", "tick_pos": "50%", "font_size": "14", "show_tick": false, "tick_edge": "left", "text_align": "left", "has_padding": true, "vertical_align": "top", "background_color": "white" } }, { "layout": { "x": 8, "y": 0, "width": 4, "height": 1 }, "definition": { "type": "note", "content": "**PVCs Full in 1 Week - Based on Daily Usage**\n\nUnsupported panel (stat)\n\n- predict_linear() has no Datadog formula equivalent; forecast-based PVC exhaustion count cannot be expressed\n\nOriginal PromQL:\n```\nA: (\n count (\n (kubelet_volume_stats_available_bytes{namespace=~\"${k8s_namespace}\"})\n and\n (predict_linear(kubelet_volume_stats_available_bytes{namespace=~\"${k8s_namespace}\"}[1d], 7 * 24 * 60 * 60) < 0)\n )\n)\nor\nvector(0)\n```", "tick_pos": "50%", "font_size": "14", "show_tick": false, "tick_edge": "left", "text_align": "left", "has_padding": true, "vertical_align": "top", "background_color": "white" } }, { "layout": { "x": 0, "y": 1, "width": 4, "height": 1 }, "definition": { "type": "query_value", "title": "PVCs Above Warning Threshold", "requests": [ { "queries": [ { "name": "query1", "query": "max:kubernetes.kubelet.volume.stats.used_bytes{$k8s_namespace} by {persistentvolumeclaim,kube_namespace}", "aggregator": "last", "data_source": "metrics" } ], "formulas": [ { "formula": "default_zero(count_not_null(query1))" } ], "response_format": "scalar", "conditional_formats": [ { "value": -0.0001, "palette": "white_on_green", "comparator": ">=" }, { "value": 0.0001, "palette": "white_on_red", "comparator": ">=" } ] } ], "autoscale": true, "precision": 2, "text_align": "center", "timeseries_background": { "type": "area" } } }, { "layout": { "x": 4, "y": 1, "width": 4, "height": 1 }, "definition": { "type": "query_value", "title": "PVCs in Pending State", "requests": [ { "queries": [ { "name": "query1", "query": "sum:kubernetes_state.persistentvolumeclaim.status{$k8s_namespace,phase:pending} by {persistentvolumeclaim,kube_namespace}", "aggregator": "last", "data_source": "metrics" } ], "formulas": [ { "formula": "default_zero(count_not_null(query1))" } ], "response_format": "scalar", "conditional_formats": [ { "value": -0.0001, "palette": "white_on_green", "comparator": ">=" }, { "value": 0.0001, "palette": "white_on_red", "comparator": ">=" } ] } ], "autoscale": true, "precision": 2, "text_align": "center", "timeseries_background": { "type": "area" } } }, { "layout": { "x": 8, "y": 1, "width": 4, "height": 1 }, "definition": { "type": "query_value", "title": "PVCs in Lost State", "requests": [ { "queries": [ { "name": "query1", "query": "sum:kubernetes_state.persistentvolumeclaim.status{$k8s_namespace,phase:lost} by {persistentvolumeclaim,kube_namespace}", "aggregator": "last", "data_source": "metrics" } ], "formulas": [ { "formula": "default_zero(count_not_null(query1))" } ], "response_format": "scalar", "conditional_formats": [ { "value": -0.0001, "palette": "white_on_green", "comparator": ">=" }, { "value": 0.0001, "palette": "white_on_red", "comparator": ">=" } ] } ], "autoscale": true, "precision": 2, "text_align": "center", "timeseries_background": { "type": "area" } } } ], "show_title": true, "layout_type": "ordered" } }, { "layout": { "x": 0, "y": 2, "width": 12, "height": 4 }, "definition": { "type": "group", "title": "Usage statistics", "widgets": [ { "layout": { "x": 0, "y": 0, "width": 12, "height": 3 }, "definition": { "type": "query_table", "title": "Persistent Volume Claim", "requests": [ { "queries": [ { "name": "query1", "query": "sum:kube_persistentvolumeclaim_info{$k8s_namespace} by {persistentvolumeclaim,namespace,storageclass,volumename}", "aggregator": "last", "data_source": "metrics" }, { "name": "query2", "query": "sum:kubernetes.kubelet.volume.stats.capacity_bytes{$k8s_namespace} by {persistentvolumeclaim}", "aggregator": "last", "data_source": "metrics" }, { "name": "query3", "query": "sum:kubernetes.kubelet.volume.stats.used_bytes{$k8s_namespace} by {persistentvolumeclaim}", "aggregator": "last", "data_source": "metrics" }, { "name": "query4", "query": "sum:kubernetes.kubelet.volume.stats.available_bytes{$k8s_namespace} by {persistentvolumeclaim}", "aggregator": "last", "data_source": "metrics" }, { "name": "query5", "query": "sum:kubernetes_state.persistentvolumeclaim.status{$k8s_namespace AND phase IN (pending,lost)} by {persistentvolumeclaim}", "aggregator": "last", "data_source": "metrics" }, { "name": "query6", "query": "sum:kubernetes_state.persistentvolumeclaim.status{$k8s_namespace,phase:lost} by {persistentvolumeclaim}", "aggregator": "last", "data_source": "metrics" }, { "name": "query7", "query": "sum:kubernetes.kubelet.volume.stats.used_bytes{$k8s_namespace} by {persistentvolumeclaim}", "aggregator": "last", "data_source": "metrics" }, { "name": "query8", "query": "sum:kubernetes.kubelet.volume.stats.capacity_bytes{$k8s_namespace} by {persistentvolumeclaim}", "aggregator": "last", "data_source": "metrics" } ], "formulas": [ { "limit": { "count": 50, "order": "desc" }, "formula": "query1" }, { "formula": "query2 / 1073741824" }, { "formula": "query3 / 1073741824" }, { "formula": "query4 / 1073741824" }, { "formula": "query5 + query6" }, { "formula": "(query7 / query8) * 100" } ], "response_format": "scalar" } ], "has_search_bar": "auto" } }, { "layout": { "x": 0, "y": 3, "width": 12, "height": 1 }, "definition": { "type": "query_table", "title": "Storage Class", "requests": [ { "queries": [ { "name": "query1", "query": "avg:kube_storageclass_info{*}", "aggregator": "last", "data_source": "metrics" } ], "formulas": [ { "limit": { "count": 50, "order": "desc" }, "formula": "query1" } ], "response_format": "scalar" } ], "has_search_bar": "auto" } } ], "show_title": true, "layout_type": "ordered" } }, { "layout": { "x": 0, "y": 6, "width": 12, "height": 3 }, "definition": { "type": "group", "title": "Graphical usage data ", "widgets": [ { "layout": { "x": 0, "y": 0, "width": 12, "height": 3 }, "definition": { "type": "timeseries", "title": "All Running PVCs Used Bytes", "yaxis": { "max": "auto", "min": "auto", "label": "", "scale": "linear", "include_zero": false }, "requests": [ { "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "queries": [ { "name": "query1", "query": "max:kubernetes.kubelet.volume.stats.used_bytes{$k8s_namespace} by {persistentvolumeclaim,kube_namespace}", "data_source": "metrics" } ], "formulas": [ { "formula": "query1" } ], "display_type": "line", "response_format": "timeseries" } ], "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ] } } ], "show_title": true, "layout_type": "ordered" } }, { "layout": { "x": 0, "y": 9, "width": 12, "height": 6 }, "definition": { "type": "group", "title": "Usage Rate", "widgets": [ { "layout": { "x": 0, "y": 0, "width": 12, "height": 2 }, "definition": { "type": "timeseries", "title": "Hourly Volume Usage Rate", "yaxis": { "max": "auto", "min": "auto", "label": "", "scale": "linear", "include_zero": false }, "requests": [ { "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "queries": [ { "name": "query1", "query": "avg:kubernetes.kubelet.volume.stats.used_bytes{$k8s_namespace} by {persistentvolumeclaim,kube_namespace}", "data_source": "metrics" } ], "formulas": [ { "formula": "derivative(query1)" } ], "display_type": "line", "response_format": "timeseries" } ], "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ] } }, { "layout": { "x": 0, "y": 2, "width": 12, "height": 2 }, "definition": { "type": "timeseries", "title": "Daily Volume Usage Rate", "yaxis": { "max": "auto", "min": "auto", "label": "", "scale": "linear", "include_zero": false }, "requests": [ { "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "queries": [ { "name": "query1", "query": "avg:kubernetes.kubelet.volume.stats.used_bytes{$k8s_namespace} by {persistentvolumeclaim,kube_namespace}.rollup(avg, 86400)", "data_source": "metrics" } ], "formulas": [ { "formula": "derivative(query1)" } ], "display_type": "line", "response_format": "timeseries" } ], "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ] } }, { "layout": { "x": 0, "y": 4, "width": 12, "height": 2 }, "definition": { "type": "timeseries", "title": "Weekly Volume Usage Rate", "yaxis": { "max": "auto", "min": "auto", "label": "", "scale": "linear", "include_zero": false }, "requests": [ { "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "queries": [ { "name": "query1", "query": "avg:kubernetes.kubelet.volume.stats.used_bytes{$k8s_namespace} by {persistentvolumeclaim,kube_namespace}.rollup(avg, 604800)", "data_source": "metrics" } ], "formulas": [ { "formula": "derivative(query1)" } ], "display_type": "line", "response_format": "timeseries" } ], "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ] } } ], "show_title": true, "layout_type": "ordered" } } ], "description": "Converted from Grafana dashboard https://grafana.com/grafana/dashboards/13646 (by Navodit). Monitors all persistent volumes that exists within a Kubernetes cluster. Prometheus is used as datasource. ", "layout_type": "ordered", "reflow_type": "fixed", "template_variables": [ { "name": "k8s_namespace", "prefix": "kube_namespace", "default": "*", "available_values": [] }, { "name": "warning_threshold", "prefix": "warning_threshold", "default": "80", "available_values": [] } ] }