{
  "tags": [],
  "title": "Loki stack monitoring (Promtail, Loki)",
  "widgets": [
    {
      "layout": {
        "x": 0,
        "y": 0,
        "width": 12,
        "height": 2
      },
      "definition": {
        "type": "note",
        "content": "**Summary of alerts status**\n\nUnsupported panel (alertlist)\n\n- no Datadog equivalent for plugin alertlist\n\nOriginal PromQL:\n```\n(none)\n```",
        "tick_pos": "50%",
        "font_size": "14",
        "show_tick": false,
        "tick_edge": "left",
        "text_align": "left",
        "has_padding": true,
        "vertical_align": "top",
        "background_color": "white"
      }
    },
    {
      "layout": {
        "x": 0,
        "y": 2,
        "width": 12,
        "height": 4
      },
      "definition": {
        "type": "group",
        "title": "Problems in the Loki Stack",
        "widgets": [
          {
            "layout": {
              "x": 0,
              "y": 0,
              "width": 4,
              "height": 1
            },
            "definition": {
              "type": "timeseries",
              "title": "Number of error/warning messages logged by Loki itself",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": true
              },
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:log_messages.count{app:loki AND level IN (error,warn)} by {level}.as_rate()",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1"
                    }
                  ],
                  "display_type": "bars",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          },
          {
            "layout": {
              "x": 4,
              "y": 0,
              "width": 8,
              "height": 1
            },
            "definition": {
              "sort": {
                "order": "desc",
                "column": "time"
              },
              "type": "log_stream",
              "query": "",
              "title": "Error/warning messages logged by Loki itself",
              "columns": [
                "host",
                "service"
              ],
              "indexes": [],
              "message_display": "expanded-md",
              "show_date_column": true,
              "show_message_column": true
            }
          },
          {
            "layout": {
              "x": 0,
              "y": 1,
              "width": 4,
              "height": 1
            },
            "definition": {
              "type": "timeseries",
              "title": "Error/warning messages logged by Promtail",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": true
              },
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:log_messages.count{app_kubernetes_io_name:promtail AND level IN (error,warn)} by {level}.as_rate()",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1"
                    }
                  ],
                  "display_type": "bars",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          },
          {
            "layout": {
              "x": 4,
              "y": 1,
              "width": 8,
              "height": 1
            },
            "definition": {
              "sort": {
                "order": "desc",
                "column": "time"
              },
              "type": "log_stream",
              "query": "",
              "title": "Error/warning messages logged by Promtail",
              "columns": [
                "host",
                "service"
              ],
              "indexes": [],
              "message_display": "expanded-md",
              "show_date_column": true,
              "show_message_column": true
            }
          },
          {
            "layout": {
              "x": 0,
              "y": 2,
              "width": 12,
              "height": 1
            },
            "definition": {
              "type": "timeseries",
              "title": "Loki (distributor) - failed batch appends sent to ingesters",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": true
              },
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "avg:loki_distributor_ingester_append_failures.count{*}.as_rate()",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1 * 60"
                    }
                  ],
                  "display_type": "area",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          },
          {
            "layout": {
              "x": 0,
              "y": 3,
              "width": 12,
              "height": 1
            },
            "definition": {
              "type": "timeseries",
              "title": "Promtail - dropped log entries",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": true
              },
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "avg:promtail_dropped_entries.count{*} by {kubernetes_pod_name,instance}.as_rate()",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1 * 60"
                    }
                  ],
                  "display_type": "area",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          }
        ],
        "show_title": true,
        "layout_type": "ordered"
      }
    },
    {
      "layout": {
        "x": 0,
        "y": 6,
        "width": 12,
        "height": 6
      },
      "definition": {
        "type": "group",
        "title": "Logging activity",
        "widgets": [
          {
            "layout": {
              "x": 0,
              "y": 0,
              "width": 12,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "Number of Streams in Loki",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": true
              },
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "avg:loki_ingester_memory_streams{*}",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "alias": "Streams",
                      "formula": "query1"
                    }
                  ],
                  "display_type": "area",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          },
          {
            "layout": {
              "x": 0,
              "y": 2,
              "width": 12,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "Loki (distributor) - log entries received per second",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": true
              },
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "avg:loki_distributor_lines_received.count{*}.as_rate()",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1"
                    }
                  ],
                  "display_type": "area",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          },
          {
            "layout": {
              "x": 0,
              "y": 4,
              "width": 12,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "Bytes received in ingester per second",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": true
              },
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "sum:loki_distributor_bytes_received.count{*}.as_rate()",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1"
                    }
                  ],
                  "display_type": "area",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          }
        ],
        "show_title": true,
        "layout_type": "ordered"
      }
    },
    {
      "layout": {
        "x": 0,
        "y": 12,
        "width": 12,
        "height": 8
      },
      "definition": {
        "type": "group",
        "title": "Memory usage",
        "widgets": [
          {
            "layout": {
              "x": 0,
              "y": 0,
              "width": 12,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "Loki memory usage (% of Kubernetes memory limit)",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": false
              },
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "max:kubernetes.memory.working_set{kube_container_name:*,!kube_container_name:pod,pod_name:loki*} by {pod_name,kube_container_name}",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query2",
                      "query": "avg:kubernetes_state.container.memory_limit{pod_name:loki*} by {pod_name,kube_container_name}",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "100 * query1 / query2"
                    }
                  ],
                  "display_type": "line",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          },
          {
            "layout": {
              "x": 0,
              "y": 2,
              "width": 12,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "Loki memory usage vs Kubernetes limit and request",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": false
              },
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "avg:kubernetes.memory.working_set{kube_container_name:*,!kube_container_name:pod,pod_name:loki*} by {pod_name,kube_container_name}",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query2",
                      "query": "min:kubernetes_state.container.memory_limit{pod_name:loki*}",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query3",
                      "query": "min:kubernetes_state.container.memory_requested{pod_name:loki*}",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1"
                    },
                    {
                      "alias": "Limit",
                      "formula": "query2"
                    },
                    {
                      "alias": "Request",
                      "formula": "query3"
                    }
                  ],
                  "display_type": "line",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          },
          {
            "layout": {
              "x": 0,
              "y": 4,
              "width": 12,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "Promtail memory usage (% of Kubernetes memory limit)",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": false
              },
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "max:kubernetes.memory.working_set{kube_container_name:*,!kube_container_name:pod,pod_name:promtail*} by {pod_name,kube_container_name}",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query2",
                      "query": "avg:kubernetes_state.container.memory_limit{pod_name:promtail*} by {pod_name,kube_container_name}",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "100 * query1 / query2"
                    }
                  ],
                  "display_type": "line",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          },
          {
            "layout": {
              "x": 0,
              "y": 6,
              "width": 12,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "Promtail memory usage vs Kubernetes limit and request",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": false
              },
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "avg:kubernetes.memory.working_set{kube_container_name:*,!kube_container_name:pod,pod_name:promtail*} by {pod_name,kube_container_name}",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query2",
                      "query": "min:kubernetes_state.container.memory_limit{pod_name:promtail*}",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query3",
                      "query": "min:kubernetes_state.container.memory_requested{pod_name:promtail*}",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1"
                    },
                    {
                      "alias": "Limit",
                      "formula": "query2"
                    },
                    {
                      "alias": "Request",
                      "formula": "query3"
                    }
                  ],
                  "display_type": "line",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          }
        ],
        "show_title": true,
        "layout_type": "ordered"
      }
    },
    {
      "layout": {
        "x": 0,
        "y": 20,
        "width": 12,
        "height": 8
      },
      "definition": {
        "type": "group",
        "title": "CPU Usage",
        "widgets": [
          {
            "layout": {
              "x": 0,
              "y": 0,
              "width": 12,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "Loki CPU usage (% of Kubernetes CPU limit)",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": false
              },
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "avg:kubernetes.cpu.usage.total{!kube_container_name:pod,kube_container_name:*,pod_name:loki*} by {pod_name,kube_container_name}",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query2",
                      "query": "avg:kubernetes_state.container.cpu_limit{kube_container_name:loki} by {pod_name,kube_container_name}",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "100 * (query1 / 1000000000) / query2"
                    }
                  ],
                  "display_type": "line",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          },
          {
            "layout": {
              "x": 0,
              "y": 2,
              "width": 12,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "Loki CPU usage vs Kubernetes limit and request",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": false
              },
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "avg:kubernetes.cpu.usage.total{kube_container_name:*,!kube_container_name:pod,pod_name:loki*} by {pod_name,kube_container_name}",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query2",
                      "query": "min:kubernetes_state.container.cpu_limit{pod_name:loki*}",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query3",
                      "query": "min:kubernetes_state.container.cpu_requested{pod_name:loki*}",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1 / 1000000000"
                    },
                    {
                      "alias": "CPU Limit",
                      "formula": "query2"
                    },
                    {
                      "alias": "CPU Request",
                      "formula": "query3"
                    }
                  ],
                  "display_type": "line",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          },
          {
            "layout": {
              "x": 0,
              "y": 4,
              "width": 12,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "Promtail CPU usage (% of Kubernetes limit)",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": false
              },
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "avg:kubernetes.cpu.usage.total{kube_container_name:*,!kube_container_name:pod,pod_name:promtail*} by {pod_name,kube_container_name}",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query2",
                      "query": "avg:kubernetes_state.container.cpu_limit{pod_name:promtail*} by {pod_name,kube_container_name}",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "100 * (query1 / 1000000000) / query2"
                    }
                  ],
                  "display_type": "line",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          },
          {
            "layout": {
              "x": 0,
              "y": 6,
              "width": 12,
              "height": 2
            },
            "definition": {
              "type": "timeseries",
              "title": "Promtail CPU usage vs Kubernetes limit and request",
              "yaxis": {
                "max": "auto",
                "min": "auto",
                "label": "",
                "scale": "linear",
                "include_zero": false
              },
              "requests": [
                {
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "queries": [
                    {
                      "name": "query1",
                      "query": "avg:kubernetes.cpu.usage.total{kube_container_name:*,!kube_container_name:pod,pod_name:promtail*} by {pod_name,kube_container_name}",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query2",
                      "query": "min:kubernetes_state.container.cpu_limit{pod_name:promtail*}",
                      "data_source": "metrics"
                    },
                    {
                      "name": "query3",
                      "query": "min:kubernetes_state.container.cpu_requested{pod_name:promtail*}",
                      "data_source": "metrics"
                    }
                  ],
                  "formulas": [
                    {
                      "formula": "query1 / 1000000000"
                    },
                    {
                      "alias": "CPU Limit",
                      "formula": "query2"
                    },
                    {
                      "alias": "CPU Request",
                      "formula": "query3"
                    }
                  ],
                  "display_type": "line",
                  "response_format": "timeseries"
                }
              ],
              "show_legend": true,
              "legend_layout": "auto",
              "legend_columns": [
                "avg",
                "min",
                "max",
                "value",
                "sum"
              ]
            }
          }
        ],
        "show_title": true,
        "layout_type": "ordered"
      }
    }
  ],
  "description": "Converted from Grafana dashboard https://grafana.com/grafana/dashboards/14055 (by Quortex). This dashboard can be used to detect issues on the Loki stack, when deployed in Kubernetes. Shows: some error metrics published by Promtail/Loki. Error and warning logs emitted by Promtail/Loki. Memory and CPU usage of Promtail/Loki compared against the Kubernetes memory/cpu limits and requests.",
  "layout_type": "ordered",
  "reflow_type": "fixed",
  "template_variables": []
}