{ "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "datasource", "uid": "grafana" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "description": "", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "links": [ { "asDropdown": true, "icon": "external link", "includeVars": true, "keepTime": true, "tags": [ "kafka" ], "targetBlank": true, "title": "Related Dashboards", "type": "dashboards" } ], "liveNow": false, "panels": [ { "collapsed": false, "datasource": { "type": "prometheus", "uid": "$data_source" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 10, "panels": [], "title": "Topic & Consumer Group Stats", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "Message count is only the difference between all partitions' high and low watermarks. Therefore this count is only an approximation on compact topics", "fieldConfig": { "defaults": { "unit": "short", "min": 0, "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "gradientMode": "none", "spanNulls": false, "showPoints": "never", "pointSize": 2, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto", "axisLabel": "", "lineInterpolation": "linear", "barAlignment": 0, "insertNulls": false, "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "thresholdsStyle": { "mode": "off" } } }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 1 }, "id": 15, "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "$data_source" }, "expr": "avg by (topic_name) (kminion_kafka_topic_high_water_mark_sum{namespace=\"$kubernetes_namespace\", topic_name=\"$kafka_topic\"} - kminion_kafka_topic_low_water_mark_sum{namespace=\"$kubernetes_namespace\", topic_name=\"$kafka_topic\"})", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "Approx Messages", "refId": "A" } ], "title": "Message Count $kafka_topic", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "fieldConfig": { "defaults": { "mappings": [ { "options": { "match": "null", "result": { "text": "N/A" } }, "type": "special" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 5, "w": 3, "x": 8, "y": 1 }, "id": 35, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "fieldOptions": { "calcs": [ "first" ] }, "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "text": {}, "textMode": "value_and_name" }, "targets": [ { "exemplar": true, "expr": "count(kafka:cluster:partition:isr{namespace=\"$kubernetes_namespace\", topic=\"$kafka_topic\"} > 0)", "hide": false, "instant": false, "interval": "", "legendFormat": "total", "refId": "C" }, { "exemplar": true, "expr": "max(kminion_kafka_consumer_group_topic_assigned_partitions{namespace=\"$kubernetes_namespace\", topic_name=\"$kafka_topic\", group_id=\"$consumer_group\"})", "format": "time_series", "hide": false, "instant": false, "interval": "", "legendFormat": "subscribed", "refId": "B" } ], "title": "Partitions", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "Display the total number of consumers in a consumer group.\nFor multi-topic consumer groups, it also displays the topic_consumers count for the selected topic\nIf there are consumers not assigned to any partition in the consumer group. the count is also displayed", "fieldConfig": { "defaults": { "mappings": [ { "options": { "match": "null", "result": { "text": "N/A" } }, "type": "special" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 145 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 5, "w": 3, "x": 11, "y": 1 }, "id": 46, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "fieldOptions": { "calcs": [ "first" ] }, "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "targets": [ { "exemplar": true, "expr": "max(kminion_kafka_consumer_group_members{namespace=\"$kubernetes_namespace\", group_id=\"$consumer_group\"})", "format": "time_series", "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "group_consumers", "refId": "A" }, { "exemplar": true, "expr": "max(kminion_kafka_consumer_group_empty_members{namespace=\"$kubernetes_namespace\", group_id=\"$consumer_group\"})", "hide": false, "interval": "", "legendFormat": "empty_members", "refId": "B" }, { "exemplar": true, "expr": "max(kminion_kafka_consumer_group_topic_members{namespace=\"$kubernetes_namespace\", group_id=\"$consumer_group\", topic_name=\"$kafka_topic\"})", "hide": false, "instant": false, "interval": "", "legendFormat": "topic_consumers", "refId": "C" } ], "title": "Consumers", "type": "stat" }, { "fieldConfig": { "defaults": { "custom": { "align": "auto", "displayMode": "auto" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "Topics" }, "properties": [ { "id": "custom.width", "value": 72 } ] }, { "matcher": { "id": "byName", "options": "state" }, "properties": [ { "id": "custom.width", "value": 117 } ] }, { "matcher": { "id": "byName", "options": "protocol_type" }, "properties": [ { "id": "custom.width", "value": 113 } ] }, { "matcher": { "id": "byName", "options": "coordinator_id" }, "properties": [ { "id": "custom.width", "value": 114 } ] }, { "matcher": { "id": "byName", "options": "Lag" }, "properties": [ { "id": "custom.width", "value": 157 } ] } ] }, "gridPos": { "h": 4, "w": 10, "x": 14, "y": 1 }, "id": 43, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [] }, "targets": [ { "exemplar": true, "expr": "kminion_kafka_consumer_group_info{namespace=\"$kubernetes_namespace\", group_id=\"$consumer_group\"}", "format": "table", "instant": true, "interval": "", "legendFormat": "", "refId": "A" }, { "expr": "avg by (group_id) (kminion_kafka_consumer_group_topic_lag{namespace=\"$kubernetes_namespace\", group_id=\"$consumer_group\", topic_name=\"$kafka_topic\"})", "format": "table", "instant": true, "interval": "", "legendFormat": "", "refId": "B" }, { "exemplar": true, "expr": "count by (group_id) (kminion_kafka_consumer_group_topic_offset_sum{namespace=\"$kubernetes_namespace\", group_id=\"$consumer_group\"})", "format": "table", "hide": false, "instant": true, "interval": "", "legendFormat": "", "refId": "C" } ], "title": "Consumer Group Info", "transformations": [ { "id": "seriesToColumns", "options": { "byField": "group_id" } }, { "id": "filterFieldsByName", "options": { "include": { "names": [ "group_id", "coordinator_id", "protocol", "protocol_type", "state", "Value #B", "Value #C" ] } } }, { "id": "organize", "options": { "excludeByName": {}, "indexByName": { "Value #B": 6, "Value #C": 2, "coordinator_id": 1, "group_id": 0, "protocol": 3, "protocol_type": 4, "state": 5 }, "renameByName": { "Value #A": "", "Value #B": "Lag", "Value #C": "Topics" } } } ], "type": "table" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "hidden", "axisSoftMax": 3, "axisSoftMin": 0, "barAlignment": 0, "drawStyle": "bars", "fillOpacity": 41, "gradientMode": "hue", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "PreparingRebalance" }, "properties": [ { "id": "color", "value": { "fixedColor": "semi-dark-orange", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Stable" }, "properties": [ { "id": "color", "value": { "fixedColor": "light-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byType", "options": "time" }, "properties": [ { "id": "custom.axisPlacement", "value": "auto" } ] } ] }, "gridPos": { "h": 4, "w": 10, "x": 14, "y": 5 }, "id": 47, "options": { "legend": { "calcs": [], "displayMode": "hidden", "placement": "bottom" }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "exemplar": false, "expr": "max by(namespace, group_id, state) (kminion_kafka_consumer_group_info{namespace=\"$kubernetes_namespace\", group_id=\"$consumer_group\"}) + 1", "format": "time_series", "instant": false, "interval": "30s", "legendFormat": "{{state}}", "refId": "A" } ], "title": "Consumer Group State", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "fieldConfig": { "defaults": { "mappings": [ { "options": { "match": "null", "result": { "text": "N/A" } }, "type": "special" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 8, "y": 6 }, "id": 36, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "fieldOptions": { "calcs": [ "first" ] }, "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": [ "last" ], "fields": "/^replication_factor$/", "values": false }, "text": {}, "textMode": "auto" }, "targets": [ { "expr": "kminion_kafka_topic_info{namespace=\"$kubernetes_namespace\", topic_name=\"$kafka_topic\"}", "format": "table", "instant": true, "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "title": "Replication Factor", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "fieldConfig": { "defaults": { "mappings": [ { "options": { "match": "null", "result": { "text": "N/A" } }, "type": "special" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 11, "y": 6 }, "id": 20, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "fieldOptions": { "calcs": [ "first" ] }, "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "last" ], "fields": "/^cleanup_policy$/", "values": false }, "text": {}, "textMode": "auto" }, "targets": [ { "expr": "kminion_kafka_topic_info{namespace=\"$kubernetes_namespace\", topic_name=\"$kafka_topic\"}", "format": "table", "instant": true, "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "title": "Cleanup Policy", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "Messages", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "hue", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 1, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 9 }, "id": 4, "links": [], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "exemplar": true, "expr": "avg by (group_id,topic_name) (kminion_kafka_consumer_group_topic_lag{namespace=\"$kubernetes_namespace\", group_id=\"$consumer_group\", topic_name=\"$kafka_topic\"})", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{group_id}} => {{topic_name}}", "refId": "A" } ], "title": "Consumergroup => Topic Lag", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "fieldConfig": { "defaults": { "mappings": [ { "options": { "match": "null", "result": { "text": "N/A" } }, "type": "special" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "#299c46", "value": null }, { "color": "rgba(237, 129, 40, 0.89)", "value": 200 }, { "color": "#d44a3a", "value": 1000 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 4, "x": 8, "y": 9 }, "id": 7, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "fieldOptions": { "calcs": [ "lastNotNull" ] }, "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "last" ], "fields": "", "limit": 1, "values": false }, "text": {}, "textMode": "value" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "$data_source" }, "exemplar": false, "expr": "max(kminion_kafka_consumer_group_topic_lag{namespace=\"$kubernetes_namespace\", group_id=\"$consumer_group\", topic_name=\"$kafka_topic\"})", "format": "time_series", "instant": true, "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "title": "Topic Lag (messages)", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "fieldConfig": { "defaults": { "mappings": [ { "options": { "match": "null", "result": { "text": "N/A" } }, "type": "special" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "#299c46", "value": null } ] }, "unit": "msg/s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 4, "x": 12, "y": 9 }, "id": 23, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "fieldOptions": { "calcs": [ "lastNotNull" ] }, "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "last" ], "fields": "/^Value$/", "values": false }, "text": {}, "textMode": "auto" }, "targets": [ { "exemplar": false, "expr": "avg (sum by (kubernetes_pod_name) (delta(kminion_kafka_consumer_group_topic_offset_sum{namespace=\"$kubernetes_namespace\", group_id=\"$consumer_group\", topic_name=\"$kafka_topic\"}[90s])/90))", "format": "table", "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "title": "Consume Rate", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "Messages/s", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 9, "gradientMode": "hue", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "smooth", "lineStyle": { "fill": "solid" }, "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 1, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": "/.*Produce.*/g" }, "properties": [ { "id": "custom.lineStyle", "value": { "dash": [ 0, 10 ], "fill": "dot" } } ] } ] }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 9 }, "id": 5, "links": [], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "exemplar": true, "expr": "avg by (group_id, topic_name) (sum by (group_id, topic_name, kubernetes_pod_name) (delta(kminion_kafka_consumer_group_topic_offset_sum{namespace=\"$kubernetes_namespace\", group_id=\"$consumer_group\", topic_name=\"$kafka_topic\"}[90s])/90))", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "Consume Rate: {{ group_id }} => {{ topic_name }}", "refId": "A" }, { "exemplar": true, "expr": "sum by(topic_name) (rate(kminion_kafka_topic_high_water_mark_sum{namespace=\"$kubernetes_namespace\",topic_name=\"$kafka_topic\"}[$__rate_interval]))", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "Produce Rate: {{ topic_name }}", "refId": "B" } ], "title": "Messages In vs Consumed", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "", "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "bars", "lineWidth": 1, "fillOpacity": 0, "gradientMode": "none", "spanNulls": false, "showPoints": "never", "pointSize": 2, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto", "axisLabel": "", "lineInterpolation": "linear", "barAlignment": 0, "insertNulls": false, "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "thresholdsStyle": { "mode": "off" } } }, "overrides": [] }, "gridPos": { "h": 9, "w": 24, "x": 0, "y": 17 }, "id": 45, "options": { "legend": { "displayMode": "table", "placement": "right", "showLegend": true, "calcs": [ "last" ], "sortBy": "last", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "$data_source" }, "expr": "avg by (group_id,topic_name,partition_id) (kminion_kafka_consumer_group_topic_partition_lag{namespace=\"$kubernetes_namespace\", group_id=\"$consumer_group\", topic_name=\"$kafka_topic\"})", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "Partiton {{partition_id}}", "refId": "A" } ], "title": "Lag per Partition: $consumer_group => $kafka_topic", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "fieldConfig": { "defaults": { "custom": { "align": "auto", "displayMode": "auto", "filterable": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 10, "w": 24, "x": 0, "y": 26 }, "id": 41, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [] }, "targets": [ { "expr": "avg by (partition) (kafka_log_log_logstartoffset{namespace=\"$kubernetes_namespace\", topic=\"$kafka_topic\"})", "format": "table", "instant": true, "interval": "", "legendFormat": "", "refId": "A" }, { "expr": "avg by (partition) (kafka_log_log_logendoffset{namespace=\"$kubernetes_namespace\",topic=\"$kafka_topic\"})", "format": "table", "instant": true, "interval": "", "legendFormat": "", "refId": "B" }, { "expr": "avg by (partition_id) (kminion_kafka_consumer_group_topic_partition_lag{namespace=\"$kubernetes_namespace\",topic_name=\"$kafka_topic\", group_id=\"$consumer_group\"})", "format": "table", "hide": false, "instant": true, "interval": "", "legendFormat": "", "refId": "C" } ], "title": "Partitions Stats", "transformations": [ { "id": "renameByRegex", "options": { "regex": "(partition_id)", "renamePattern": "partition" } }, { "id": "seriesToColumns", "options": { "byField": "partition" } }, { "id": "organize", "options": { "excludeByName": { "Time": true, "Time 1": true, "Time 2": true, "Time 3": true }, "indexByName": { "Time 1": 4, "Time 2": 5, "Time 3": 6, "Value #A": 1, "Value #B": 2, "Value #C": 3, "partition": 0 }, "renameByName": { "Time": "", "Time 1": "", "Time 2": "", "Value #A": "Low Water Mark", "Value #B": "High Water Mark", "Value #C": "Lag", "partition": "Partition", "partition_id": "Partition ID" } } }, { "id": "calculateField", "options": { "alias": "Message Count", "mode": "reduceRow", "reduce": { "include": [ "High Water Mark", "Low Water Mark" ], "reducer": "delta" }, "replaceFields": false } } ], "type": "table" }, { "collapsed": false, "datasource": { "type": "prometheus", "uid": "$data_source" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 36 }, "id": 25, "panels": [], "title": "Cluster Totals", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "A consumer group which has at least one non expired consumer group offset for any partition is considered a \"subscriber\" for this topic. This metric can be used to identify topics which may not be consumed by anyone (anymore).", "fieldConfig": { "defaults": { "custom": { "align": "auto", "displayMode": "auto" }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "Value" }, "properties": [ { "id": "custom.displayMode", "value": "gradient-gauge" }, { "id": "custom.width", "value": 300 } ] }, { "matcher": { "id": "byName", "options": "topic" }, "properties": [ { "id": "custom.displayMode", "value": "color-background" } ] } ] }, "gridPos": { "h": 8, "w": 6, "x": 0, "y": 37 }, "id": 27, "links": [], "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [ { "desc": true, "displayName": "Value" } ] }, "targets": [ { "expr": "count by (topic_name) (kminion_kafka_consumer_group_topic_lag{namespace=\"$kubernetes_namespace\"})>0", "format": "table", "instant": true, "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "title": "Topics by number of active consumer groups", "transformations": [ { "id": "organize", "options": { "excludeByName": { "Time": true }, "indexByName": { "Time": 0, "Value": 2, "topic": 1 }, "renameByName": { "Time": "" } } } ], "type": "table" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "fieldConfig": { "defaults": { "custom": { "align": "auto", "displayMode": "auto" }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "green", "value": null }, { "color": "#EAB839", "value": 70 }, { "color": "red", "value": 90 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "Value" }, "properties": [ { "id": "custom.displayMode", "value": "gradient-gauge" }, { "id": "unit", "value": "op/s" } ] } ] }, "gridPos": { "h": 8, "w": 10, "x": 6, "y": 37 }, "id": 29, "links": [], "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true }, "targets": [ { "expr": "topk(10, max_over_time(sum by (kubernetes_pod_name, topic_name, group_id) (irate(kminion_kafka_consumer_group_topic_offset_sum{namespace=\"$kubernetes_namespace\"}[3m]))[5m:]) > 0)", "format": "table", "instant": true, "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "title": "Top 10 Consumer Groups by Consumer Rate", "transformations": [ { "id": "organize", "options": { "excludeByName": { "Time": true }, "indexByName": {}, "renameByName": {} } } ], "type": "table" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "Topics by their total size in bytes", "fieldConfig": { "defaults": { "custom": { "align": "auto", "displayMode": "auto" }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 50 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "Value" }, "properties": [ { "id": "unit", "value": "bytes" }, { "id": "custom.displayMode", "value": "gradient-gauge" }, { "id": "decimals", "value": 1 } ] } ] }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 37 }, "id": 30, "links": [], "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [ { "desc": true, "displayName": "Value" } ] }, "targets": [ { "expr": "avg by (topic_name) (kminion_kafka_topic_log_dir_size_total_bytes{namespace=\"$kubernetes_namespace\"})", "format": "table", "instant": true, "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "title": "Topics by size", "transformations": [ { "id": "organize", "options": { "excludeByName": { "Time": true }, "indexByName": {}, "renameByName": {} } } ], "type": "table" }, { "collapsed": true, "datasource": { "type": "prometheus", "uid": "$data_source" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 45 }, "id": 32, "panels": [ { "datasource": { "type": "prometheus", "uid": "$data_source" }, "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "gradientMode": "none", "spanNulls": false, "showPoints": "never", "pointSize": 2, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto", "axisLabel": "", "lineInterpolation": "linear", "barAlignment": 0, "insertNulls": false, "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "thresholdsStyle": { "mode": "off" } } }, "overrides": [] }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 47 }, "id": 34, "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "$data_source" }, "expr": "rate(kafka_server_fetchsessioncache_incrementalfetchsessionevictions_total{namespace=\"$kubernetes_namespace\"}[1m])", "interval": "", "legendFormat": "broker-{{brokerId}}", "refId": "A" } ], "title": "Broker Consumer Incremental Fetch Sessions Evictions", "type": "timeseries" } ], "title": "Broker Consumers Metrics", "type": "row" } ], "refresh": "30s", "schemaVersion": 41, "style": "dark", "tags": [ "kafka", "kafka-minion", "consumers" ], "templating": { "list": [ { "current": { "selected": false, "text": "", "value": "" }, "datasource": { "type": "prometheus", "uid": "$data_source" }, "definition": "kminion_exporter_up", "hide": 0, "includeAll": false, "label": "Namespace", "multi": false, "name": "kubernetes_namespace", "options": [], "query": { "query": "kminion_exporter_up", "refId": "Prometheus-kubernetes_namespace-Variable-Query" }, "refresh": 1, "regex": "/.*namespace=\"([^\"]*).*/", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false }, { "current": { "selected": false, "text": "", "value": "" }, "datasource": { "type": "prometheus", "uid": "$data_source" }, "definition": "query_result(sort_desc(count by (topic_name) (kminion_kafka_consumer_group_topic_lag{namespace=\"$kubernetes_namespace\"})))", "hide": 0, "includeAll": false, "label": "Topic", "multi": false, "name": "kafka_topic", "options": [], "query": { "query": "query_result(sort_desc(count by (topic_name) (kminion_kafka_consumer_group_topic_lag{namespace=\"$kubernetes_namespace\"})))", "refId": "Prometheus-kafka_topic-Variable-Query" }, "refresh": 1, "regex": "/.*topic_name=\\\"([^\\\"]*).*/", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": ".+", "current": { "selected": false, "text": "", "value": "" }, "datasource": { "type": "prometheus", "uid": "$data_source" }, "definition": "label_values(kminion_kafka_consumer_group_topic_lag{namespace=\"$kubernetes_namespace\", topic_name=\"$kafka_topic\"}, group_id)", "hide": 0, "includeAll": false, "label": "Consumer Group", "multi": false, "name": "consumer_group", "options": [], "query": { "query": "label_values(kminion_kafka_consumer_group_topic_lag{namespace=\"$kubernetes_namespace\", topic_name=\"$kafka_topic\"}, group_id)", "refId": "Prometheus-consumer_group-Variable-Query" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false }, { "current": { "selected": false, "text": "Prometheus", "value": "Prometheus" }, "hide": 0, "includeAll": false, "multi": false, "name": "data_source", "options": [], "query": "prometheus", "queryValue": "", "refresh": 1, "regex": "/^(?!default)/", "skipUrlSync": false, "type": "datasource" } ] }, "time": { "from": "now-2h", "to": "now" }, "timepicker": { "refresh_intervals": [ "10s", "30s", "1m", "5m" ], "time_options": [ "5m", "15m", "1h", "2h", "3h", "6h", "12h", "24h", "2d", "7d" ] }, "timezone": "", "title": "Kafka Consumers", "uid": null, "version": 1, "weekStart": "", "id": null }