{ "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "datasource", "uid": "grafana" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "description": "Kafka System Performance", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": null, "links": [ { "asDropdown": true, "icon": "external link", "includeVars": true, "keepTime": true, "tags": [ "kafka" ], "targetBlank": true, "title": "Related Dashboards", "type": "dashboards" } ], "liveNow": false, "panels": [ { "datasource": { "type": "prometheus", "uid": "$data_source" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 23763572007, "title": "Health", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "A producer produce 1 rps checking that the External Kafka listener is reachable Requires a synthetic canary producer exporting kafka_canary_producer_* metrics; adapt to your own canary tooling or remove this panel.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "0": { "color": "red", "index": 2, "text": "UNREACHABLE" } }, "type": "value" }, { "options": { "from": 0.8, "result": { "color": "green", "index": 0, "text": "OK" }, "to": 10 }, "type": "range" }, { "options": { "from": 0.1, "result": { "color": "orange", "index": 1, "text": "Intermittent Failures" }, "to": 0.8 }, "type": "range" }, { "options": { "match": "null+nan", "result": { "color": "red", "index": 3, "text": "NOT Monitored" } }, "type": "special" } ], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 4, "w": 5, "x": 0, "y": 1 }, "id": 23763572009, "options": { "displayMode": "gradient", "minVizHeight": 10, "minVizWidth": 0, "orientation": "horizontal", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "showUnfilled": true, "valueMode": "color" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "$data_source" }, "editorMode": "code", "exemplar": true, "expr": "sum(rate(kafka_canary_producer_record_send_total{cluster=\"$kafka_cluster\"}[1m])) by (type)", "interval": "", "legendFormat": "{{type}}", "range": true, "refId": "A" } ], "title": "Canary Producer Throughput", "transparent": true, "type": "bargauge" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "A producer produce 1 rps checking that the External Kafka listener has no errors Requires a synthetic canary producer exporting kafka_canary_producer_* metrics; adapt to your own canary tooling or remove this panel.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "0": { "color": "green", "index": 1, "text": "OK" } }, "type": "value" }, { "options": { "from": 0.01, "result": { "color": "red", "index": 0, "text": "ERRORS" }, "to": 1 }, "type": "range" }, { "options": { "match": "null+nan", "result": { "color": "red", "index": 2, "text": "NOT Monitored" } }, "type": "special" } ], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 4, "w": 5, "x": 5, "y": 1 }, "id": 23763572010, "options": { "displayMode": "gradient", "minVizHeight": 10, "minVizWidth": 0, "orientation": "horizontal", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "showUnfilled": true, "valueMode": "color" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "$data_source" }, "editorMode": "code", "exemplar": true, "expr": "sum(rate(kafka_canary_producer_record_error_total{cluster=\"$kafka_cluster\"}[1m])) by (type)", "interval": "", "legendFormat": "{{type}}", "range": true, "refId": "A" } ], "title": "Canary Producer Errors", "transparent": true, "type": "bargauge" }, { "collapsed": false, "datasource": { "type": "prometheus", "uid": "$data_source" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, "id": 106, "panels": [], "title": "Overview", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "Number of Brokers Online", "fieldConfig": { "defaults": { "mappings": [ { "id": 0, "op": "=", "text": "N/A", "type": 1, "value": "null" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "#d44a3a", "value": null }, { "color": "rgba(237, 129, 40, 0.89)", "value": 0 }, { "color": "semi-dark-green", "value": 2 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 0, "y": 6 }, "id": 46, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "fieldOptions": { "calcs": [ "lastNotNull" ] }, "graphMode": "none", "justifyMode": "center", "orientation": "vertical", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": true }, "text": {}, "textMode": "auto" }, "repeatDirection": "h", "targets": [ { "expr": "count by(broker_group) (kafka_server_app_info_version{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\"})", "instant": true, "interval": "", "legendFormat": "{{broker_group}}", "refId": "B" } ], "title": "Brokers Online", "transformations": [], "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "fieldConfig": { "defaults": { "mappings": [ { "$$hashKey": "object:224", "id": 0, "op": "=", "text": "N/A", "type": 1, "value": "null" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "semi-dark-blue", "value": null }, { "color": "#EAB839", "value": 1 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 3, "w": 2, "x": 3, "y": 6 }, "id": 23763571996, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "last" ], "fields": "/^version$/", "values": false }, "textMode": "auto" }, "targets": [ { "expr": "kafka_server_app_info_version{namespace=\"$kubernetes_namespace\"}", "format": "table", "instant": true, "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "title": "Cluster Version", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "Number of active controllers in the cluster.", "fieldConfig": { "defaults": { "mappings": [ { "id": 0, "op": "=", "text": "N/A", "type": 1, "value": "null" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "semi-dark-green", "value": null }, { "color": "#e5ac0e", "value": 2 }, { "color": "#bf1b00" } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 3, "w": 2, "x": 5, "y": 6 }, "id": 36, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "fieldOptions": { "calcs": [ "lastNotNull" ] }, "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "targets": [ { "expr": "sum(kafka_controller_kafkacontroller_activecontrollercount{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\"})", "format": "time_series", "hide": false, "instant": true, "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "title": "Active Controllers", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "Number of topics in cluster", "fieldConfig": { "defaults": { "mappings": [ { "id": 0, "op": "=", "text": "N/A", "type": 1, "value": "null" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "semi-dark-green", "value": null }, { "color": "rgba(237, 129, 40, 0.89)", "value": 100 }, { "color": "#bf1b00", "value": 1000 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 7, "y": 6 }, "id": 30, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "fieldOptions": { "calcs": [ "lastNotNull" ] }, "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "targets": [ { "expr": "count(sum(kafka:cluster:partition:replicascount{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",pod=~\"$cluster_name-($kafka_broker)\"}>0) by (topic))", "format": "time_series", "hide": false, "instant": true, "interval": "", "intervalFactor": 2, "legendFormat": "", "refId": "A" } ], "title": "Topics", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "Replicas that are online", "fieldConfig": { "defaults": { "mappings": [ { "id": 0, "op": "=", "text": "N/A", "type": 1, "value": "null" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "#d44a3a", "value": null }, { "color": "rgba(237, 129, 40, 0.89)", "value": 0 }, { "color": "semi-dark-green", "value": 0 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 10, "y": 6 }, "id": 136, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "fieldOptions": { "calcs": [ "lastNotNull" ] }, "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "targets": [ { "expr": "sum(irate(kafka_controller_controllerstats_uncleanleaderelections_total{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\"}[5m]))", "format": "time_series", "instant": true, "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "title": "Unclean Leader Election", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "Number of unhealthy kafka nodes in envoy proxy Only applicable if external access is exposed through an Envoy proxy in front of the brokers.", "fieldConfig": { "defaults": { "mappings": [ { "id": 0, "op": "=", "text": "N/A", "type": 1, "value": "null" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "semi-dark-green", "value": null }, { "color": "red", "value": 1 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 3, "w": 2, "x": 13, "y": 6 }, "id": 140, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "fieldOptions": { "calcs": [ "lastNotNull" ] }, "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "targets": [ { "expr": "(sum(envoy_cluster_membership_total{namespace=\"$kubernetes_namespace\"}) - sum(envoy_cluster_membership_healthy{namespace=\"$kubernetes_namespace\"}))", "instant": true, "interval": "", "legendFormat": "", "refId": "B" } ], "title": "Envoy Unhealthy Kafka", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "Number of partitions that don?t have an active leader and are hence not writable or readable.", "fieldConfig": { "defaults": { "mappings": [ { "id": 0, "op": "=", "text": "N/A", "type": 1, "value": "null" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "semi-dark-green", "value": null }, { "color": "#ef843c", "value": 1 }, { "color": "#bf1b00", "value": 1 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 15, "y": 6 }, "id": 32, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "fieldOptions": { "calcs": [ "lastNotNull" ] }, "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "targets": [ { "expr": "sum(kafka_controller_kafkacontroller_offlinepartitionscount{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\"})", "format": "time_series", "hide": false, "instant": true, "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "title": "Offline Partitions Count", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "Number of partitions which are at their minimum in sync replica count (| ISR | == | min.insync.replicas |).", "fieldConfig": { "defaults": { "mappings": [ { "id": 0, "op": "=", "text": "0", "type": 1, "value": "null" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "semi-dark-green", "value": null }, { "color": "#ef843c", "value": 1 }, { "color": "#bf1b00", "value": 1000 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 18, "y": 6 }, "id": 102, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "fieldOptions": { "calcs": [ "lastNotNull" ] }, "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "targets": [ { "expr": "sum(kafka:cluster:partition:atminisr{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\"})", "format": "time_series", "hide": false, "instant": true, "interval": "", "intervalFactor": 2, "legendFormat": "", "refId": "A" } ], "title": "Partitions at mininimum ISR", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "Number of partitions which are under their minimum in sync replica count (| ISR | < | min.insync.replicas |).", "fieldConfig": { "defaults": { "mappings": [ { "id": 0, "op": "=", "text": "0", "type": 1, "value": "null" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "semi-dark-green", "value": null }, { "color": "#ef843c", "value": 1 }, { "color": "#bf1b00", "value": 1 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 21, "y": 6 }, "id": 103, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "fieldOptions": { "calcs": [ "lastNotNull" ] }, "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "targets": [ { "expr": "sum(kafka:cluster:partition:underminisr{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\"})", "format": "time_series", "hide": false, "instant": true, "interval": "", "intervalFactor": 2, "legendFormat": "", "refId": "A" } ], "title": "Partitions under mininimum ISR", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": " Only applicable if external access is exposed through an Envoy proxy in front of the brokers.", "fieldConfig": { "defaults": { "mappings": [ { "id": 0, "op": "=", "text": "N/A", "type": 1, "value": "null" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "semi-dark-green", "value": null }, { "color": "#bf1b00" } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 0, "y": 9 }, "id": 139, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "orientation": "vertical", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "targets": [ { "expr": "sum by(broker_group) (envoy_server_live{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\"})", "instant": true, "interval": "", "legendFormat": "{{broker_group}}", "refId": "A" } ], "title": "Envoy Instances", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "Number of Zookeeper peers", "fieldConfig": { "defaults": { "mappings": [ { "id": 0, "op": "=", "text": "N/A", "type": 1, "value": "null" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "semi-dark-green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 3, "w": 2, "x": 3, "y": 9 }, "id": 137, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "fieldOptions": { "calcs": [ "lastNotNull" ] }, "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "targets": [ { "expr": "count(quorum_size{namespace=\"$kubernetes_namespace\", job=~\".+-headless\"})", "format": "time_series", "hide": false, "instant": true, "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "title": "ZK Quorum Size", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "Number of Zookeeper leaders in the cluster. It should always be 1", "fieldConfig": { "defaults": { "mappings": [ { "id": 0, "op": "=", "text": "N/A", "type": 1, "value": "null" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "semi-dark-green", "value": null }, { "color": "#C4162A", "value": 2 }, { "color": "#bf1b00" } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 3, "w": 2, "x": 5, "y": 9 }, "id": 138, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "fieldOptions": { "calcs": [ "lastNotNull" ] }, "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "targets": [ { "expr": "count(leader_uptime{namespace=\"$kubernetes_namespace\", job=~\".+-headless\"})", "format": "time_series", "hide": false, "instant": true, "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "title": "ZK Leaders", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "This should be 0 or No Data. If > 0 then a zk pod is out of sync with the cluster.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } }, "overrides": [] }, "gridPos": { "h": 3, "w": 2, "x": 7, "y": 9 }, "id": 23763571998, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "targets": [ { "exemplar": true, "expr": "digest_mismatches_count{namespace=\"$kubernetes_namespace\"} > 0", "interval": "", "legendFormat": "{{pod}}", "refId": "A" } ], "title": "ZK digest mismatches", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "Replicas that are online", "fieldConfig": { "defaults": { "mappings": [ { "id": 0, "op": "=", "text": "N/A", "type": 1, "value": "null" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "#d44a3a", "value": null }, { "color": "rgba(237, 129, 40, 0.89)", "value": 0 }, { "color": "semi-dark-green", "value": 0 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 9, "y": 9 }, "id": 135, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "fieldOptions": { "calcs": [ "lastNotNull" ] }, "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "targets": [ { "expr": "sum(kafka_server_replicamanager_partitioncount{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\"})", "format": "time_series", "instant": true, "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "title": "Online Replicas", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "Total remaining capacity in Kafka", "fieldConfig": { "defaults": { "mappings": [ { "id": 0, "op": "=", "text": "N/A", "type": 1, "value": "null" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 12, "y": 9 }, "id": 141, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "fieldOptions": { "calcs": [ "lastNotNull" ] }, "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "targets": [ { "expr": "sum(kubelet_volume_stats_available_bytes{namespace=\"$kubernetes_namespace\",persistentvolumeclaim=~\"$cluster_name.*\"})", "instant": true, "interval": "", "legendFormat": "Remaining", "refId": "A" }, { "expr": "sum by(cluster ) (label_replace(kubelet_volume_stats_capacity_bytes{namespace=\"$kubernetes_namespace\", persistentvolumeclaim=~\"$cluster_name.*\"}, \"cluster\", \"$1\", \"persistentvolumeclaimi\", \"(.+)-([0-9]+)-storage-.*\"))", "instant": true, "interval": "", "legendFormat": "Total", "refId": "B" } ], "title": "Kafka Disk Capacity", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "", "fieldConfig": { "defaults": { "mappings": [ { "id": 0, "op": "=", "text": "N/A", "type": 1, "value": "null" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "semi-dark-green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 15, "y": 9 }, "id": 142, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "fieldOptions": { "calcs": [ "lastNotNull" ] }, "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "targets": [ { "expr": "sum(kafka_server_replicamanager_reassigningpartitions{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",pod=~\"$cluster_name-($kafka_broker)\"})", "instant": true, "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Reassigning Partitions", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "CruiseControl unfixable goals requires manual intervention to Kafka. Check CC to see what goal fails to be self-healed", "fieldConfig": { "defaults": { "mappings": [ { "id": 0, "op": "=", "text": "N/A", "type": 1, "value": "null" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "semi-dark-green", "value": null }, { "color": "dark-red", "value": 1 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 18, "y": 9 }, "id": 143, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "fieldOptions": { "calcs": [ "lastNotNull" ] }, "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "targets": [ { "expr": "kafka_cruisecontrol_anomalydetector_goal_violation_has_unfixable_goals_value{namespace=\"$kubernetes_namespace\"}", "instant": true, "interval": "", "legendFormat": "Unfixable Goals", "refId": "B" } ], "title": "CC Unfixable Goals", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "Total number of consumer groups in cluster", "fieldConfig": { "defaults": { "mappings": [ { "options": { "match": "null", "result": { "text": "N/A" } }, "type": "special" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "semi-dark-green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 21, "y": 9 }, "id": 23763571993, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "fieldOptions": { "calcs": [ "lastNotNull" ] }, "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "targets": [ { "exemplar": true, "expr": "max(kminion_kafka_consumer_group_info_count{namespace=\"$kubernetes_namespace\"})", "instant": false, "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Consumer Groups", "type": "stat" }, { "collapsed": false, "datasource": { "type": "prometheus", "uid": "$data_source" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 12 }, "id": 151, "panels": [], "title": "Topic Utilization", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "", "fieldConfig": { "defaults": { "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "filterable": false, "inspect": false }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-green", "value": null }, { "color": "red", "value": 50 } ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "Value" }, "properties": [ { "id": "unit", "value": "short" }, { "id": "custom.cellOptions", "value": { "type": "color-text" } }, { "id": "decimals", "value": 1 }, { "id": "custom.cellOptions", "value": { "type": "color-text" } }, { "id": "custom.width", "value": 120 }, { "id": "custom.align", "value": "center" } ] }, { "matcher": { "id": "byName", "options": "topic" }, "properties": [ { "id": "links", "value": [ { "targetBlank": true, "title": "View topic details", "url": "/d/1d1d1d1d1/kafka-topic?${__url_time_range}&${kubernetes_namespace:queryparam}&var-kafka_topic=${__data.fields[topic]}" } ] }, { "id": "custom.width" } ] } ] }, "gridPos": { "h": 8, "w": 4, "x": 0, "y": 13 }, "id": 155, "links": [], "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "frameIndex": 0, "showHeader": true, "sortBy": [ { "desc": true, "displayName": "Value" } ] }, "targets": [ { "expr": "sum_over_time( sum by(topic) (increase(kafka_server_brokertopicmetrics_messagesin_total{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",topic=~\".*\",topic!=\"\",pod=~\"$cluster_name-($kafka_broker)\"}[3m])) [${__range_s}s:3m] ) > 0\n", "format": "table", "instant": true, "interval": "", "legendFormat": "{{topic}}", "refId": "B" } ], "title": "IN messages during interval", "transformations": [ { "id": "organize", "options": { "excludeByName": { "Time": true }, "indexByName": {}, "renameByName": {} } } ], "type": "table" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "", "fieldConfig": { "defaults": { "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "filterable": false, "inspect": false }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-green", "value": null }, { "color": "red", "value": 50 } ] }, "unit": "none" }, "overrides": [ { "matcher": { "id": "byName", "options": "Value" }, "properties": [ { "id": "unit", "value": "bytes" }, { "id": "custom.cellOptions", "value": { "type": "color-text" } }, { "id": "decimals", "value": 1 }, { "id": "custom.width", "value": 120 }, { "id": "custom.align", "value": "center" } ] }, { "matcher": { "id": "byName", "options": "topic" }, "properties": [ { "id": "links", "value": [ { "targetBlank": true, "title": "View topic details", "url": "/d/1d1d1d1d1/kafka-topic?${__url_time_range}&${kubernetes_namespace:queryparam}&var-kafka_topic=${__data.fields[topic]}" } ] }, { "id": "custom.width" } ] } ] }, "gridPos": { "h": 8, "w": 4, "x": 4, "y": 13 }, "hideTimeOverride": false, "id": 156, "links": [], "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "frameIndex": 0, "showHeader": true, "sortBy": [ { "desc": true, "displayName": "Value" } ] }, "targets": [ { "expr": "sum_over_time( sum by(topic) (increase(kafka_server_brokertopicmetrics_bytesin_total{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",topic=~\".*\",topic!=\"\",pod=~\"$cluster_name-($kafka_broker)\"}[90s])) [${__range_s}s:90s] ) > 0", "format": "table", "instant": true, "interval": "90s", "legendFormat": "{{topic}}", "refId": "B" } ], "title": "IN Size during interval", "transformations": [ { "id": "organize", "options": { "excludeByName": { "Time": true }, "indexByName": {}, "renameByName": {} } } ], "type": "table" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "", "fieldConfig": { "defaults": { "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "filterable": false, "inspect": false }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-green", "value": null }, { "color": "red", "value": 50 } ] }, "unit": "none" }, "overrides": [ { "matcher": { "id": "byName", "options": "Value" }, "properties": [ { "id": "unit", "value": "bytes" }, { "id": "custom.cellOptions", "value": { "type": "color-text" } }, { "id": "decimals", "value": 1 }, { "id": "custom.width", "value": 120 }, { "id": "custom.align", "value": "center" } ] }, { "matcher": { "id": "byName", "options": "topic" }, "properties": [ { "id": "links", "value": [ { "targetBlank": true, "title": "View topic details", "url": "/d/1d1d1d1d1/kafka-topic?${__url_time_range}&${kubernetes_namespace:queryparam}&var-kafka_topic=${__data.fields[topic]}" } ] }, { "id": "custom.width" } ] } ] }, "gridPos": { "h": 8, "w": 4, "x": 8, "y": 13 }, "hideTimeOverride": false, "id": 23763571994, "links": [], "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "frameIndex": 0, "showHeader": true, "sortBy": [ { "desc": true, "displayName": "Value" } ] }, "targets": [ { "expr": "sum_over_time( sum by(topic) (increase(kafka_server_brokertopicmetrics_bytesout_total{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",topic=~\".*\",topic!=\"\",pod=~\"$cluster_name-($kafka_broker)\"}[90s])) [${__range_s}s:90s] ) > 0", "format": "table", "instant": true, "interval": "90s", "legendFormat": "{{topic}}", "refId": "B" } ], "title": "OUT Size during interval", "transformations": [ { "id": "organize", "options": { "excludeByName": { "Time": true }, "indexByName": {}, "renameByName": {} } } ], "type": "table" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "", "fieldConfig": { "defaults": { "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "filterable": false, "inspect": false }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-green", "value": null }, { "color": "orange", "value": 50 }, { "color": "semi-dark-red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "Value" }, "properties": [ { "id": "unit", "value": "bytes" }, { "id": "custom.cellOptions", "value": { "type": "color-text" } }, { "id": "decimals", "value": 1 }, { "id": "custom.width", "value": 120 }, { "id": "custom.align", "value": "center" } ] }, { "matcher": { "id": "byName", "options": "topic" }, "properties": [ { "id": "links", "value": [ { "targetBlank": true, "title": "View topic details", "url": "/d/1d1d1d1d1/kafka-topic?${__url_time_range}&${kubernetes_namespace:queryparam}&var-kafka_topic=${__data.fields[topic]}" } ] } ] } ] }, "gridPos": { "h": 8, "w": 4, "x": 12, "y": 13 }, "id": 153, "links": [], "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [ { "desc": true, "displayName": "Value" } ] }, "targets": [ { "expr": "sum by (topic) (kafka:log:size:topic:broker{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\"}) > 0", "format": "table", "instant": true, "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "title": "Topics by size", "transformations": [ { "id": "organize", "options": { "excludeByName": { "Time": true }, "indexByName": {}, "renameByName": {} } } ], "type": "table" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "", "fieldConfig": { "defaults": { "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "filterable": false, "inspect": false }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-green", "value": null }, { "color": "orange", "value": 90 } ] }, "unit": "none" }, "overrides": [ { "matcher": { "id": "byName", "options": "Value" }, "properties": [ { "id": "unit", "value": "none" }, { "id": "custom.cellOptions", "value": { "type": "color-text" } }, { "id": "decimals", "value": 0 }, { "id": "custom.width", "value": 120 }, { "id": "custom.align", "value": "center" } ] }, { "matcher": { "id": "byName", "options": "topic" }, "properties": [ { "id": "links", "value": [ { "targetBlank": true, "title": "View topic details", "url": "/d/1d1d1d1d1/kafka-topic?${__url_time_range}&${kubernetes_namespace:queryparam}&var-kafka_topic=${__data.fields[topic]}" } ] } ] } ] }, "gridPos": { "h": 8, "w": 4, "x": 16, "y": 13 }, "id": 154, "links": [], "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "frameIndex": 0, "showHeader": true, "sortBy": [ { "desc": true, "displayName": "Value" } ] }, "targets": [ { "exemplar": true, "expr": "sum(kminion_kafka_topic_info_partitions_count{namespace=\"$kubernetes_namespace\"}) by (topic_name)", "format": "table", "hide": false, "instant": true, "interval": "", "legendFormat": "{{topic}}", "refId": "B" } ], "title": "Topics by no of partitions", "transformations": [ { "id": "organize", "options": { "excludeByName": { "Time": true }, "indexByName": {}, "renameByName": {} } } ], "type": "table" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "", "fieldConfig": { "defaults": { "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "filterable": false, "inspect": false }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 50 } ] }, "unit": "none" }, "overrides": [ { "matcher": { "id": "byName", "options": "Value" }, "properties": [ { "id": "unit", "value": "short" }, { "id": "custom.cellOptions", "value": { "type": "color-text" } }, { "id": "decimals", "value": 0 }, { "id": "custom.width", "value": 142 } ] }, { "matcher": { "id": "byName", "options": "topic" }, "properties": [ { "id": "links", "value": [ { "targetBlank": true, "title": "View topic details", "url": "/d/1d1d1d1d1/kafka-topic?${__url_time_range}&${kubernetes_namespace:queryparam}&var-kafka_topic=${__data.fields[topic]}" } ] } ] } ] }, "gridPos": { "h": 8, "w": 4, "x": 20, "y": 13 }, "id": 158, "links": [], "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "frameIndex": 0, "showHeader": true, "sortBy": [ { "desc": true, "displayName": "Value" } ] }, "targets": [ { "expr": "sum(kafka_server_fetcherlagmetrics_consumerlag{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",topic=~\".*\",topic!=\"\",pod=~\"$cluster_name-($kafka_broker)\"}) by (topic) > 0", "format": "table", "instant": true, "interval": "", "legendFormat": "{{topic}}", "refId": "B" } ], "title": "Topics by replication lag", "transformations": [ { "id": "organize", "options": { "excludeByName": { "Time": true }, "indexByName": {}, "renameByName": {} } } ], "type": "table" }, { "collapsed": false, "datasource": { "type": "prometheus", "uid": "$data_source" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 21 }, "id": 110, "panels": [], "title": "Cluster Utilization", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "Total Incoming Byte Rate", "fieldConfig": { "defaults": { "decimals": 1, "mappings": [ { "id": 0, "op": "=", "text": "0", "type": 1, "value": "null" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "#d44a3a", "value": null }, { "color": "rgba(237, 129, 40, 0.89)", "value": 0 }, { "color": "semi-dark-green", "value": 2 } ] }, "unit": "Bps" }, "overrides": [] }, "gridPos": { "h": 6, "w": 4, "x": 0, "y": 22 }, "id": 98, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "repeatDirection": "h", "targets": [ { "expr": "sum(irate(kafka_server_brokertopicmetrics_bytesin_total{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",topic=~\"$kafka_topic\",topic!=\"\",pod=~\"$cluster_name-($kafka_broker)\"}[90s]))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "title": "Total Incoming Byte Rate", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "", "fieldConfig": { "defaults": { "decimals": 1, "mappings": [ { "id": 0, "op": "=", "text": "0", "type": 1, "value": "null" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "#d44a3a", "value": null }, { "color": "rgba(237, 129, 40, 0.89)", "value": 0 }, { "color": "semi-dark-green", "value": 2 } ] }, "unit": "Bps" }, "overrides": [ { "matcher": { "id": "byName", "options": "Produce Message Rate" }, "properties": [ { "id": "unit", "value": "wps" } ] }, { "matcher": { "id": "byName", "options": "Produce Request Rate" }, "properties": [ { "id": "unit", "value": "wps" } ] } ] }, "gridPos": { "h": 6, "w": 4, "x": 4, "y": 22 }, "id": 100, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "repeatDirection": "h", "targets": [ { "expr": "sum(irate(kafka_server_brokertopicmetrics_messagesin_total{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",topic=~\"$kafka_topic\",topic!=\"\",pod=~\"$cluster_name-($kafka_broker)\"}[90s]))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "Produce Message Rate", "refId": "A" }, { "expr": "sum(irate(kafka_server_brokertopicmetrics_totalproducerequests_total{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",topic=~\"$kafka_topic\",topic!=\"\",pod=~\"$cluster_name-($kafka_broker)\"}[90s]))", "interval": "", "legendFormat": "Produce Request Rate", "refId": "B" } ], "title": "Total Incoming Rates", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "Replication IN + OUT across the entire cluster", "fieldConfig": { "defaults": { "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "#d44a3a", "value": null }, { "color": "rgba(237, 129, 40, 0.89)", "value": 0 }, { "color": "semi-dark-green", "value": 2 } ] }, "unit": "Bps" }, "overrides": [] }, "gridPos": { "h": 6, "w": 4, "x": 8, "y": 22 }, "id": 99, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "repeatDirection": "h", "targets": [ { "expr": "sum(rate(kafka_server_brokertopicmetrics_replicationbytesin_total{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",pod=~\"$cluster_name-($kafka_broker)\"}[90s])) + sum(rate(kafka_server_brokertopicmetrics_replicationbytesout_total{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",pod=~\"$cluster_name-($kafka_broker)\"}[90s]))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "title": "Total Cluster Replication Rate", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "Total Outgoind Byte Rate", "fieldConfig": { "defaults": { "decimals": 1, "mappings": [ { "id": 0, "op": "=", "text": "0", "type": 1, "value": "null" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "#d44a3a", "value": null }, { "color": "rgba(237, 129, 40, 0.89)", "value": 0 }, { "color": "semi-dark-green", "value": 2 } ] }, "unit": "Bps" }, "overrides": [] }, "gridPos": { "h": 6, "w": 4, "x": 12, "y": 22 }, "id": 112, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "repeatDirection": "h", "targets": [ { "expr": "sum(irate(kafka_server_brokertopicmetrics_bytesout_total{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",topic=~\"$kafka_topic\",topic!=\"\",pod=~\"$cluster_name-($kafka_broker)\"}[90s]))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "title": "Total Outgoing Byte Rate", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "Total Produce Request Rate", "fieldConfig": { "defaults": { "decimals": 1, "mappings": [ { "id": 0, "op": "=", "text": "0", "type": 1, "value": "null" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "#d44a3a", "value": null }, { "color": "rgba(237, 129, 40, 0.89)", "value": 0 }, { "color": "semi-dark-green", "value": 2 } ] }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 6, "w": 4, "x": 16, "y": 22 }, "id": 122, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "repeatDirection": "h", "targets": [ { "expr": "sum(irate(kafka_server_brokertopicmetrics_totalfetchrequests_total{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",topic=~\"$kafka_topic\",topic!=\"\",pod=~\"$cluster_name-($kafka_broker)\"}[90s]))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "title": "Total Fetch Request Rate", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "Message Conversations performed by broker due to message protocol mismatch on client side.", "fieldConfig": { "defaults": { "decimals": 1, "mappings": [ { "id": 0, "op": "=", "text": "0", "type": 1, "value": "0" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "semi-dark-green", "value": null }, { "color": "rgba(237, 129, 40, 0.89)", "value": 1 }, { "color": "#d44a3a", "value": 100 } ] }, "unit": "wps" }, "overrides": [] }, "gridPos": { "h": 6, "w": 4, "x": 20, "y": 22 }, "id": 123, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "repeatDirection": "h", "targets": [ { "expr": "sum(irate(kafka_server_brokertopicmetrics_producemessageconversions_total{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",pod=~\"$cluster_name-($kafka_broker)\"}[90s]))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "produce", "refId": "A" }, { "expr": "sum(irate(kafka_server_brokertopicmetrics_fetchmessageconversions_total{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",pod=~\"$cluster_name-($kafka_broker)\"}[90s]))", "interval": "", "legendFormat": "consume", "refId": "B" } ], "title": "Message Conversion Rate", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "Broker traffic excluding replication", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "Bps" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": "/out-.*/" }, "properties": [ { "id": "custom.transform", "value": "negative-Y" } ] }, { "matcher": { "id": "byName", "options": "IN-Total" }, "properties": [ { "id": "color", "value": { "fixedColor": "#56A64B", "mode": "fixed" } }, { "id": "custom.lineWidth", "value": 2 } ] }, { "matcher": { "id": "byName", "options": "OUT-Total" }, "properties": [ { "id": "color", "value": { "fixedColor": "#3274D9", "mode": "fixed" } }, { "id": "custom.lineWidth", "value": 2 } ] }, { "__systemRef": "hideSeriesFrom", "matcher": { "id": "byNames", "options": { "mode": "exclude", "names": [ "in-101", "in-102", "in-103", "in-104", "in-105", "in-106", "in-107", "in-108", "in-109", "in-110", "in-111", "in-112", "in-113", "in-114", "in-115", "in-116", "in-117", "in-118", "in-119", "in-120", "in-201", "in-202", "in-203", "in-204", "in-205", "in-206", "in-207", "in-208", "in-209", "in-210", "in-211", "in-212", "in-213", "in-214", "in-215", "in-216", "in-217", "in-218", "in-219", "in-220", "in-301", "in-302", "in-303", "in-304", "in-305", "in-306", "in-307", "in-308", "in-309", "in-310", "in-311", "in-312", "in-313", "in-314", "in-315", "in-316", "in-317", "in-318", "in-319", "in-320", "out-101", "out-102", "out-103", "out-104", "out-105", "out-106", "out-107", "out-108", "out-109", "out-110", "out-111", "out-112", "out-113", "out-114", "out-115", "out-116", "out-117", "out-118", "out-119", "out-120", "out-201", "out-202", "out-203", "out-204", "out-205", "out-206", "out-207", "out-208", "out-209", "out-210", "out-211", "out-212", "out-213", "out-214", "out-215", "out-216", "out-217", "out-218", "out-219", "out-220", "out-301", "out-302", "out-303", "out-304", "out-305", "out-306", "out-307", "out-308", "out-309", "out-310", "out-311", "out-312", "out-313", "out-314", "out-315", "out-316", "out-317", "out-318", "out-319", "out-320" ], "prefix": "All except:", "readOnly": true } }, "properties": [ { "id": "custom.hideFrom", "value": { "legend": false, "tooltip": false, "viz": true } } ] } ] }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 28 }, "id": 44, "links": [], "options": { "legend": { "calcs": [ "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "sum(irate(kafka_server_brokertopicmetrics_bytesin_total{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",pod=~\"$cluster_name-($kafka_broker)\",topic=~\"$kafka_topic\", topic!=\"\"}[1m])) by (brokerId)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "in-{{brokerId}}", "refId": "A" }, { "expr": "sum(irate(kafka_server_brokertopicmetrics_bytesout_total{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",pod=~\"$cluster_name-($kafka_broker)\",topic=~\"$kafka_topic\", topic!=\"\"}[1m])) by (brokerId)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "out-{{brokerId}}", "refId": "B" }, { "expr": "sum(irate(kafka_server_brokertopicmetrics_bytesin_total{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",topic=~\"$kafka_topic\",topic!=\"\"}[1m]))", "interval": "", "legendFormat": "IN-Total", "refId": "C" }, { "expr": "sum(irate(kafka_server_brokertopicmetrics_bytesout_total{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",topic=~\"$kafka_topic\",topic!=\"\"}[1m]))", "interval": "", "legendFormat": "OUT-Total", "refId": "D" } ], "title": "Byte Rate Per Second", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "wps" }, "overrides": [ { "matcher": { "id": "byName", "options": "Total" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Total" }, "properties": [ { "id": "custom.lineWidth", "value": 2 } ] }, { "__systemRef": "hideSeriesFrom", "matcher": { "id": "byNames", "options": { "mode": "exclude", "names": [ "101", "102", "103", "104", "105", "106", "107", "108", "109", "110", "111", "112", "113", "114", "115", "116", "117", "118", "119", "120", "201", "202", "203", "204", "205", "206", "207", "208", "209", "210", "211", "212", "213", "214", "215", "216", "217", "218", "219", "220", "301", "302", "303", "304", "305", "306", "307", "308", "309", "310", "311", "312", "313", "314", "315", "316", "317", "318", "319", "320" ], "prefix": "All except:", "readOnly": true } }, "properties": [ { "id": "custom.hideFrom", "value": { "legend": false, "tooltip": false, "viz": true } } ] } ] }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 28 }, "id": 58, "links": [], "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "sum(irate(kafka_server_brokertopicmetrics_messagesin_total{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",pod=~\"$cluster_name-($kafka_broker)\",topic=~\"$kafka_topic\",topic!=\"\"}[1m])) by (brokerId)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{brokerId}}", "refId": "D" }, { "expr": "sum(irate(kafka_server_brokertopicmetrics_messagesin_total{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",topic=~\"$kafka_topic\",topic!=\"\"}[1m])) ", "interval": "", "legendFormat": "Total", "refId": "A" } ], "title": "Messages In Per Second", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "stepBefore", "lineStyle": { "fill": "solid" }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 28 }, "id": 116, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "sum(kafka_server_fetcherlagmetrics_consumerlag{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",topic=~\"$kafka_topic\",topic!=\"\",pod=~\"$cluster_name-($kafka_broker)\", partition=~\"$kafka_partition\"}) by (brokerId,topic,partition) > 0", "interval": "", "legendFormat": "{{brokerId}}:{{topic}}:{{partition}}", "refId": "A" } ], "title": "Replication Lag ", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "Produce Request Rate.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "reqps" }, "overrides": [ { "matcher": { "id": "byName", "options": "Total" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Total" }, "properties": [ { "id": "custom.lineWidth", "value": 4 }, { "id": "custom.axisPlacement", "value": "right" }, { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } }, { "id": "unit", "value": "reqps" } ] } ] }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 35 }, "id": 50, "links": [], "options": { "legend": { "calcs": [ "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "$data_source" }, "expr": "sum(irate(kafka_server_brokertopicmetrics_totalproducerequests_total{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",pod=~\"$cluster_name-($kafka_broker)\",topic=~\"$kafka_topic\",topic!=\"\"}[1m])) by (brokerId)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{brokerId}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "expr": "sum(irate(kafka_server_brokertopicmetrics_failedproducerequests_total{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",pod=~\"$cluster_name-($kafka_broker)\",topic=~\"$kafka_topic\",topic!=\"\"}[1m])) by (brokerId) > 0", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "failed-{{brokerId}}", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "expr": "sum(irate(kafka_server_brokertopicmetrics_totalproducerequests_total{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",topic=~\"$kafka_topic\",topic!=\"\"}[1m]))", "hide": false, "interval": "", "legendFormat": "Total", "refId": "C" } ], "title": "Produce Request Rate", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "Fetch Request Rate", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "reqps" }, "overrides": [ { "matcher": { "id": "byName", "options": "Total" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "/failed-.*/i" }, "properties": [ { "id": "custom.transform", "value": "negative-Y" } ] }, { "matcher": { "id": "byName", "options": "Total" }, "properties": [ { "id": "custom.lineWidth", "value": 4 }, { "id": "unit", "value": "short" }, { "id": "decimals", "value": 1 }, { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } }, { "id": "unit", "value": "reqps" }, { "id": "custom.axisPlacement", "value": "right" } ] } ] }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 35 }, "id": 56, "links": [], "options": { "legend": { "calcs": [ "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "$data_source" }, "expr": "sum(irate(kafka_server_brokertopicmetrics_totalfetchrequests_total{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",pod=~\"$cluster_name-($kafka_broker)\",topic=~\"$kafka_topic\",topic!=\"\"}[1m])) by (brokerId)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{brokerId}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "expr": " sum(irate(kafka_server_brokertopicmetrics_failedfetchrequests_total{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",pod=~\"$cluster_name-($kafka_broker)\",topic=~\"$kafka_topic\",topic!=\"\"}[1m])) by (brokerId) > 0", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "failed-{{brokerId}}", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "expr": "sum(irate(kafka_server_brokertopicmetrics_totalfetchrequests_total{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",topic=~\"$kafka_topic\",topic!=\"\"}[1m]))", "interval": "", "legendFormat": "Total", "refId": "C" } ], "title": "Fetch Request Rate", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "left", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [ { "matcher": { "id": "byName", "options": "replicas_move_completed" }, "properties": [ { "id": "custom.drawStyle", "value": "bars" }, { "id": "custom.fillOpacity", "value": 83 }, { "id": "color", "value": { "fixedColor": "#96D98D", "mode": "fixed" } }, { "id": "custom.lineWidth", "value": 0 }, { "id": "custom.axisPlacement", "value": "right" }, { "id": "decimals" } ] }, { "matcher": { "id": "byName", "options": "leadership_move_completed" }, "properties": [ { "id": "custom.drawStyle", "value": "bars" }, { "id": "custom.fillOpacity", "value": 100 }, { "id": "color", "value": { "fixedColor": "#8AB8FF", "mode": "fixed" } }, { "id": "custom.lineWidth", "value": 0 }, { "id": "custom.pointSize", "value": 8 }, { "id": "decimals" } ] }, { "matcher": { "id": "byName", "options": "intra_broker_replica_move_completed" }, "properties": [ { "id": "custom.drawStyle", "value": "bars" }, { "id": "custom.fillOpacity", "value": 100 }, { "id": "color", "value": { "fixedColor": "#DEB6F2", "mode": "fixed" } }, { "id": "custom.lineWidth", "value": 0 } ] }, { "matcher": { "id": "byName", "options": "goal-violations" }, "properties": [ { "id": "color", "value": { "fixedColor": "#F2495C", "mode": "fixed" } }, { "id": "custom.lineWidth", "value": 0 }, { "id": "custom.pointSize", "value": 10 }, { "id": "custom.showPoints", "value": "always" } ] }, { "matcher": { "id": "byName", "options": "has_unfixable_goals" }, "properties": [ { "id": "color", "value": { "fixedColor": "#C4162A", "mode": "fixed" } }, { "id": "custom.lineWidth", "value": 0 }, { "id": "custom.showPoints", "value": "always" } ] }, { "matcher": { "id": "byName", "options": "broker-failure" }, "properties": [ { "id": "color", "value": { "fixedColor": "#FFB357", "mode": "fixed" } }, { "id": "custom.lineWidth", "value": 0 }, { "id": "custom.pointSize", "value": 8 }, { "id": "custom.showPoints", "value": "always" }, { "id": "custom.transform", "value": "negative-Y" } ] } ] }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 35 }, "id": 23763572003, "interval": "1m", "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "expr": "(max_over_time(kafka_cruisecontrol_executor_replica_action_completed_value{namespace=\"$kubernetes_namespace\",pod=~\"$cluster_name-cruisecontrol-.+\"}[$__interval])>0)", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "replicas_move_completed", "refId": "B" }, { "expr": "(max_over_time(kafka_cruisecontrol_executor_leadership_action_completed_value{namespace=\"$kubernetes_namespace\",pod=~\"$cluster_name-cruisecontrol-.+\"}[$__interval])>0)", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "leadership_move_completed", "refId": "D" }, { "expr": "(max_over_time(kafka_cruisecontrol_executor_intra_broker_replica_action_completed_value{namespace=\"$kubernetes_namespace\",pod=~\"$cluster_name-cruisecontrol-.+\"}[$__interval])>0)", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "intra_broker_replica_move_completed", "refId": "E" }, { "expr": "-1 * (kafka_cruisecontrol_anomalydetector_goal_violation_has_unfixable_goals_value{namespace=\"$kubernetes_namespace\",pod=~\"$cluster_name-cruisecontrol-.+\"} > 0)", "interval": "5m", "legendFormat": "has_unfixable_goals", "refId": "C" }, { "expr": "max_over_time ( sum by (namespace) (changes(kafka_cruisecontrol_anomalydetector_goal_violation_rate_count{namespace=\"$kubernetes_namespace\",pod=~\"$cluster_name-cruisecontrol-.+\"}[90s]))[10m:10s]) > 0\n", "hide": false, "interval": "5m", "intervalFactor": 1, "legendFormat": "goal-violations", "refId": "F" }, { "expr": "max_over_time ( sum by (namespace) (changes(kafka_cruisecontrol_anomalydetector_broker_failure_rate_count{namespace=\"$kubernetes_namespace\",pod=~\"$cluster_name-cruisecontrol-.+\"}[90s]))[10m:10s]) > 0\n", "interval": "5m", "legendFormat": "broker-failure", "refId": "A" } ], "title": "Cruise Control Auto Balance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "In the positive Y, number of active connections, in the negative Y, number of overflowing connections. The later should be 0. Only applicable if external access is exposed through an Envoy proxy in front of the brokers.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 5, "x": 0, "y": 43 }, "id": 23763572000, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "exemplar": true, "expr": "sum by (envoy_cluster_name)(envoy_cluster_upstream_cx_active{namespace=\"$kubernetes_namespace\", envoy_cluster_name=~\"broker.*\"})", "hide": false, "interval": "", "legendFormat": "active-{{envoy_cluster_name}}", "refId": "B" }, { "exemplar": true, "expr": "sum by (envoy_cluster_name)(envoy_cluster_upstream_cx_overflow{namespace=\"$kubernetes_namespace\",envoy_cluster_name=~\"broker.*\"}) * -1 < 0", "hide": false, "interval": "", "legendFormat": "overflow-{{envoy_cluster_name}}", "refId": "A" } ], "title": "Envoy connections", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "Shows the disk in cluster which are over-used - (size wise) compared to avg on that broker", "fieldConfig": { "defaults": { "custom": { "align": "left", "cellOptions": { "type": "color-text" }, "filterable": false, "inspect": false }, "mappings": [ { "from": "", "id": 0, "operator": "", "text": "", "to": "", "type": 1 } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "light-purple" } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "Value" }, "properties": [ { "id": "unit", "value": "bytes" }, { "id": "thresholds", "value": { "mode": "absolute", "steps": [ { "color": "semi-dark-orange" }, { "color": "light-green", "value": 361326587904 } ] } }, { "id": "displayName", "value": "Remaining" } ] }, { "matcher": { "id": "byName", "options": "broker" }, "properties": [ { "id": "custom.width", "value": 57 } ] }, { "matcher": { "id": "byName", "options": "persistentvolumeclaim" }, "properties": [ { "id": "custom.width", "value": 240 } ] }, { "matcher": { "id": "byName", "options": "Remaining" }, "properties": [ { "id": "custom.width", "value": 98 } ] } ] }, "gridPos": { "h": 7, "w": 5, "x": 5, "y": 43 }, "id": 147, "links": [], "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [] }, "targets": [ { "expr": "sum by(broker,persistentvolumeclaim) (label_replace ( (kubelet_volume_stats_available_bytes{namespace=\"$kubernetes_namespace\",persistentvolumeclaim=~\"$cluster_name.*\"}), \"broker\", \"$1\", \"persistentvolumeclaim\", \"$cluster_name-([0-9]+)-.+\"))\n\n < on (broker) group_left()\n\n (\n0.8 * avg by (broker)(sum by(broker,persistentvolumeclaim) (label_replace ( (kubelet_volume_stats_available_bytes{namespace=\"$kubernetes_namespace\",persistentvolumeclaim=~\"$cluster_name.*\"}), \"broker\", \"$1\", \"persistentvolumeclaim\", \"$cluster_name-([0-9]+)-.+\")))\n\n )", "format": "table", "hide": false, "instant": true, "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "title": "Unbalanced disks in kafka cluster", "transformations": [ { "id": "filterFieldsByName", "options": { "include": { "names": [ "broker", "persistentvolumeclaim", "Value" ] } } } ], "type": "table" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 0, "gradientMode": "none", "spanNulls": false, "showPoints": "never", "pointSize": 2, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto", "axisLabel": "", "lineInterpolation": "linear", "barAlignment": 0, "insertNulls": false, "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "thresholdsStyle": { "mode": "off" } } }, "overrides": [ { "matcher": { "id": "byRegexp", "options": "/fetch-failed.*/" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#F2495C" } }, { "id": "custom.axisPlacement", "value": "right" } ] }, { "matcher": { "id": "byRegexp", "options": "/req-errors.*/" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#FFA6B0" } }, { "id": "custom.drawStyle", "value": "bars" } ] } ] }, "gridPos": { "h": 7, "w": 6, "x": 10, "y": 43 }, "id": 117, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "max", "last" ], "sortBy": "max", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "sum by(brokerId, error) (rate(kafka_network_requestmetrics_errors_total{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",error!=\"NONE\"}[3m])) > 0", "hide": false, "interval": "", "legendFormat": "req-errors-{{brokerId}}-{{error}}", "refId": "B" }, { "expr": "sum by (brokerId,topic) (rate(kafka_server_brokertopicmetrics_failedfetchrequests_total{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",topic=~\"$kafka_topic\",topic!=\"\",pod=~\"$cluster_name-($kafka_broker)\"}[3m])) > 0", "interval": "", "legendFormat": "failed-fetches-{{brokerId}}:{{topic}}", "refId": "A" }, { "expr": "sum by(brokerId) (rate(kafka_server_replica_fetcher_metrics_connection_close_total{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\"}[3m])) > 0", "hide": false, "interval": "", "legendFormat": "fetch-failed-connections-{{brokerId}}", "refId": "C" } ], "title": "Errors", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": -1, "drawStyle": "points", "fillOpacity": 50, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 6, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 43 }, "id": 124, "links": [], "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "sum(kafka_server_replicamanager_reassigningpartitions{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",pod=~\"$cluster_name-($kafka_broker)\"}) by (brokerId) > 0", "format": "time_series", "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{brokerId}}", "refId": "A" } ], "title": "Reassigning Partitions", "type": "timeseries" }, { "collapsed": true, "datasource": { "type": "prometheus", "uid": "$data_source" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 50 }, "id": 23763572015, "panels": [ { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": " Requires a custom Kafka authorizer plugin that exports JMX metrics under the kafka_authorizer_* names used here; adjust the metric prefix/labels to match your authorizer, or remove this panel if you use Kafka's default authorizer.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "left", "cellOptions": { "type": "auto" }, "filterable": false, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 50 }, "hideTimeOverride": false, "id": 23763572017, "links": [], "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "frameIndex": 0, "showHeader": true, "sortBy": [ { "desc": true, "displayName": "Value" } ] }, "targets": [ { "datasource": { "type": "prometheus", "uid": "$data_source" }, "editorMode": "code", "exemplar": false, "expr": "sum_over_time(sum by (client_id) (increase(kafka_authorizer_valid_token_count{namespace=\"$kubernetes_namespace\", kafka_cr=\"$cluster_name\", pod=~\"$cluster_name-($kafka_broker)\"}[90s]))[${__range_s}s:90s] ) > 0", "format": "table", "hide": false, "instant": true, "interval": "90s", "legendFormat": "{{client_id}}", "refId": "B" } ], "title": "Authorizer Token Validations", "transformations": [ { "id": "sortBy", "options": { "fields": {}, "sort": [ { "desc": true, "field": "Value" } ] } }, { "id": "organize", "options": { "excludeByName": { "Time": true }, "indexByName": {}, "renameByName": { "Value": "Authentication count during interval", "client_id": "IMS client ID" } } } ], "type": "table" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": " Requires a custom Kafka authorizer plugin (see kafka_authorizer_* metrics); remove if not applicable.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 9, "w": 6, "x": 12, "y": 50 }, "hideTimeOverride": false, "id": 23763572019, "links": [], "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "frameIndex": 0, "showHeader": true, "sortBy": [ { "desc": true, "displayName": "Value" } ] }, "targets": [ { "datasource": { "type": "prometheus", "uid": "$data_source" }, "exemplar": false, "expr": "sum_over_time(sum by (principal,resourceType) (increase(kafka_authorizer_unauthorized_count{namespace=\"$kubernetes_namespace\", kafka_cr=\"$cluster_name\", pod=~\"$cluster_name-($kafka_broker)\"}[90s]))[${__range_s}s:90s] ) > 0", "format": "table", "hide": false, "instant": true, "interval": "90s", "legendFormat": "{{topic}}", "refId": "B" } ], "title": "Unauthorized requests per principal", "transformations": [ { "id": "organize", "options": { "excludeByName": { "Time": true }, "indexByName": {}, "renameByName": { "Time": "", "Value": "Count", "Value #A": "Unauthorized rate per min", "principal": "Principal", "resourceType": "Resource accessed" } } }, { "id": "sortBy", "options": { "fields": {}, "sort": [ { "desc": true, "field": "Count" } ] } } ], "type": "table" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": " Requires a custom Kafka authorizer plugin (see kafka_authorizer_* metrics); remove if not applicable.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 9, "w": 6, "x": 18, "y": 50 }, "id": 23763572021, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "$data_source" }, "editorMode": "code", "exemplar": true, "expr": "rate(kafka_authorizer_unauthorized_total_count{namespace=\"$kubernetes_namespace\", kafka_cr=\"$cluster_name\", pod=~\"$cluster_name-($kafka_broker)\"}[1m])>0", "format": "time_series", "hide": false, "instant": false, "interval": "", "legendFormat": "{{brokerId}}-{{resourceType}}", "refId": "A" } ], "title": "Unauthorized total requests", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": " Requires a custom Kafka authorizer plugin (see kafka_authorizer_* metrics); remove if not applicable.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "reqps" }, "overrides": [ { "matcher": { "id": "byName", "options": "total" }, "properties": [ { "id": "color", "value": { "fixedColor": "#E02F44", "mode": "fixed" } }, { "id": "custom.fillOpacity", "value": 0 }, { "id": "custom.lineWidth", "value": 3 }, { "id": "custom.axisPlacement", "value": "right" }, { "id": "custom.lineStyle", "value": { "dash": [ 10, 2 ], "fill": "dash" } } ] }, { "__systemRef": "hideSeriesFrom", "matcher": { "id": "byNames", "options": { "mode": "exclude", "names": [ "total" ], "prefix": "All except:", "readOnly": true } }, "properties": [ { "id": "custom.hideFrom", "value": { "legend": false, "tooltip": false, "viz": true } } ] } ] }, "gridPos": { "h": 10, "w": 6, "x": 0, "y": 59 }, "id": 23763572023, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "$data_source" }, "exemplar": true, "expr": "sum by (brokerId) (rate(kafka_authorizer_authorized_total_count{namespace=\"$kubernetes_namespace\", kafka_cr=\"$cluster_name\", pod=~\"$cluster_name-($kafka_broker)\"}[1m]))", "format": "time_series", "hide": false, "instant": false, "interval": "", "legendFormat": "{{brokerId}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "exemplar": true, "expr": "sum(rate(kafka_authorizer_authorized_total_count{namespace=\"$kubernetes_namespace\", kafka_cr=\"$cluster_name\", pod=~\"$cluster_name-($kafka_broker)\"}[1m]))", "format": "time_series", "hide": false, "instant": false, "interval": "", "legendFormat": "total", "refId": "B" } ], "title": "Authorized total requests", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": " Requires a custom Kafka authorizer plugin (see kafka_authorizer_* metrics); remove if not applicable.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 10, "w": 6, "x": 6, "y": 59 }, "id": 23763572025, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "$data_source" }, "exemplar": true, "expr": "sum by (brokerId) (rate(kafka_authorizer_authorized_service_count{resourceType=\"topic\",namespace=\"$kubernetes_namespace\", kafka_cr=\"$cluster_name\", pod=~\"$cluster_name-($kafka_broker)\"}[1m]))", "format": "time_series", "hide": false, "instant": false, "interval": "", "legendFormat": "service", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "exemplar": true, "expr": "sum by (brokerId) (rate(kafka_authorizer_authorized_anonymous_count{resourceType=\"topic\",namespace=\"$kubernetes_namespace\", kafka_cr=\"$cluster_name\", pod=~\"$cluster_name-($kafka_broker)\"}[1m]))", "format": "time_series", "hide": false, "instant": false, "interval": "", "legendFormat": "user-anonymous", "refId": "B" } ], "title": "Authorized topic requests", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": " Requires a custom Kafka authorizer plugin (see kafka_authorizer_* metrics); remove if not applicable.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 10, "w": 6, "x": 12, "y": 59 }, "id": 23763572027, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "$data_source" }, "exemplar": true, "expr": "sum by (brokerId) (rate(kafka_authorizer_authorized_service_count{resourceType=\"group\",namespace=\"$kubernetes_namespace\", kafka_cr=\"$cluster_name\", pod=~\"$cluster_name-($kafka_broker)\"}[1m]))", "format": "time_series", "hide": false, "instant": false, "interval": "", "legendFormat": "service", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "exemplar": true, "expr": "sum by (brokerId) (rate(kafka_authorizer_authorized_anonymous_count{resourceType=\"group\",namespace=\"$kubernetes_namespace\", kafka_cr=\"$cluster_name\", pod=~\"$cluster_name-($kafka_broker)\"}[1m]))", "format": "time_series", "hide": false, "instant": false, "interval": "", "legendFormat": "user-anonymous", "refId": "B" } ], "title": "Authorized consumer group requests", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": " Requires a custom Kafka authorizer plugin (see kafka_authorizer_* metrics); remove if not applicable.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 10, "w": 6, "x": 18, "y": 59 }, "id": 23763572029, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "$data_source" }, "exemplar": true, "expr": "sum by (brokerId) (rate(kafka_authorizer_authorized_service_count{resourceType=\"cluster\",namespace=\"$kubernetes_namespace\", kafka_cr=\"$cluster_name\", pod=~\"$cluster_name-($kafka_broker)\"}[1m]))", "format": "time_series", "hide": false, "instant": false, "interval": "", "legendFormat": "service", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "exemplar": true, "expr": "sum by (brokerId) (rate(kafka_authorizer_authorized_anonymous_count{resourceType=\"cluster\",namespace=\"$kubernetes_namespace\", kafka_cr=\"$cluster_name\", pod=~\"$cluster_name-($kafka_broker)\"}[1m]))", "format": "time_series", "hide": false, "instant": false, "interval": "", "legendFormat": "user-anonymous", "refId": "B" } ], "title": "Authorized cluster requests", "transformations": [], "type": "timeseries" } ], "title": "Authorization", "type": "row" }, { "collapsed": false, "datasource": { "type": "prometheus", "uid": "$data_source" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 51 }, "id": 28, "panels": [], "title": "Brokers Resources", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "Aggregated Kafka Broker Pods CPU Usage", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 7, "w": 5, "x": 0, "y": 52 }, "id": 81, "links": [], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "min(sum by(pod,container) (kube_pod_container_resource_limits{job=\"kube-state-metrics\", namespace=\"$kubernetes_namespace\",pod=~\"$cluster_name-($kafka_broker)\",container=\"kafka\", resource=\"cpu\"}))", "hide": false, "interval": "", "legendFormat": "kafka pod cpu limit", "refId": "B" }, { "expr": "irate(process_cpu_seconds_total{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\", pod=~\"$cluster_name-($kafka_broker)\"}[1m])", "hide": false, "interval": "", "legendFormat": "used-{{pod}}", "refId": "D" } ], "title": "CPU Usage", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "Kafka Broker Pods Memory Usage", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 7, "w": 5, "x": 5, "y": 52 }, "id": 82, "interval": "1m", "links": [], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "$data_source" }, "expr": "sum(container_memory_usage_bytes{namespace=\"$kubernetes_namespace\",pod=~\"$cluster_name-($kafka_broker)\",container=\"kafka\"}) by (pod)", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "used-{{pod}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "editorMode": "code", "expr": "min(sum (kube_pod_container_resource_limits{job=\"kube-state-metrics\", namespace=\"$kubernetes_namespace\", resource=\"memory\", pod=~\"$cluster_name-($kafka_broker)\", container=~\"kafka\"}) by(pod,container))", "legendFormat": "max-mem", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "expr": "process_resident_memory_bytes{namespace=\"$kubernetes_namespace\", pod=~\"$cluster_name-($kafka_broker)\", brokerId=~\".+\"}\t", "legendFormat": "rss-{{pod}}", "refId": "D" } ], "title": "Container Memory Usage", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "Kafka Broker Pods Available Disk Space", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 1, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": "/abs-available-./" }, "properties": [ { "id": "unit", "value": "bytes" } ] } ] }, "gridPos": { "h": 7, "w": 5, "x": 10, "y": 52 }, "id": 83, "interval": "1m", "links": [], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "sum by (persistentvolumeclaim) (kubelet_volume_stats_available_bytes{namespace=\"$kubernetes_namespace\",persistentvolumeclaim=~\".*kafka.*\"}) / (sum by (persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{namespace=\"$kubernetes_namespace\",persistentvolumeclaim=~\".*kafka.*\"}))", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{persistentvolumeclaim}}", "refId": "A" } ], "title": "Available Disk Space", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "Kafka Broker Pods Disk Usage (Aggregated by broker)", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 1, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 7, "w": 5, "x": 15, "y": 52 }, "id": 148, "links": [], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "sum by (broker_id) (label_replace(kubelet_volume_stats_used_bytes{namespace=\"$kubernetes_namespace\",persistentvolumeclaim=~\".*kafka.*\"}, \"broker_id\", \"$1\",\"persistentvolumeclaim\", \"(.+)-storage(.*)\"))", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{broker_id}}", "refId": "A" }, { "expr": "min(sum by (broker_id)\n(label_replace(\nkubelet_volume_stats_capacity_bytes{namespace=\"$kubernetes_namespace\", persistentvolumeclaim=~\"$cluster_name.*\"}\n,\"broker_id\", \"$1\",\"persistentvolumeclaim\", \"(.+)-storage(.*)\")))", "hide": false, "interval": "", "legendFormat": "Broker Total Storage", "refId": "B" } ], "title": "Disk Usage (Broker distribution)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "Zookeeper Pods Disk Usage", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 1, "mappings": [], "max": 1, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": "/abs-available-./" }, "properties": [ { "id": "unit", "value": "bytes" } ] } ] }, "gridPos": { "h": 7, "w": 4, "x": 20, "y": 52 }, "id": 149, "links": [], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "sum by (persistentvolumeclaim) (kubelet_volume_stats_available_bytes{namespace=\"$kubernetes_namespace\",persistentvolumeclaim=~\".*zookeeper.*\"}) / ( sum by (persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{namespace=\"$kubernetes_namespace\",persistentvolumeclaim=~\".*zookeeper.*\"}))", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{persistentvolumeclaim}}", "refId": "A" } ], "title": "Available Disk Space (Zookeeper)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "network and io/handlers pools usage", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "decimals": 1, "mappings": [], "max": 100, "min": -100, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": -100 }, { "color": "red", "value": 100 } ] }, "unit": "percent" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": "/handlers-.*/" }, "properties": [ { "id": "custom.transform", "value": "negative-Y" } ] } ] }, "gridPos": { "h": 7, "w": 5, "x": 0, "y": 59 }, "id": 159, "links": [], "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "100 - (kafka_network_socketserver_networkprocessoravgidle_percent{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",pod=~\"$cluster_name-($kafka_broker)\"}*100)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "network-{{brokerId}}", "refId": "A" }, { "expr": "100-irate(kafka_server_kafkarequesthandlerpool_requesthandleravgidlepercent_count{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",pod=~\"$cluster_name-($kafka_broker)\"}[1m])/10000000 > 0", "hide": false, "interval": "", "legendFormat": "handlers-{{brokerId}}", "refId": "C" } ], "title": "Network and Handlers Usage", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 7, "w": 5, "x": 5, "y": 59 }, "id": 93, "links": [], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "sum(jvm_memory_bytes_used{namespace=\"$kubernetes_namespace\",pod=~\"$cluster_name-($kafka_broker)\",kafka_cr=\"$cluster_name\"}) by (pod)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{pod}}", "refId": "A" } ], "title": "JVM Memory Usage", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 1, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 7, "w": 5, "x": 10, "y": 59 }, "id": 95, "links": [], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "sum(rate(jvm_gc_collection_seconds_sum{namespace=\"$kubernetes_namespace\",pod=~\"$cluster_name-($kafka_broker)\",kafka_cr=\"$cluster_name\"}[1m])) by (pod,gc) > 0", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{pod}} - {{gc}}", "refId": "A" } ], "title": "JVM GC Time", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 1, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 7, "w": 5, "x": 15, "y": 59 }, "id": 97, "links": [], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "sum(rate(jvm_gc_collection_seconds_count{namespace=\"$kubernetes_namespace\",pod=~\"$cluster_name-($kafka_broker)\",kafka_cr=\"$cluster_name\"}[1m])) by (pod,gc) > 0", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{pod}} - {{gc}}", "refId": "A" } ], "title": "JVM GC Rate (GC/s)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 1, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 7, "w": 4, "x": 20, "y": 59 }, "id": 160, "links": [], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "java_lang_operatingsystem_openfiledescriptorcount{namespace=\"$kubernetes_namespace\",pod=~\"$cluster_name-($kafka_broker)\",kafka_cr=\"$cluster_name\"}/java_lang_operatingsystem_maxfiledescriptorcount{namespace=\"$kubernetes_namespace\",pod=~\"$cluster_name-($kafka_broker)\",kafka_cr=\"$cluster_name\"}", "interval": "", "legendFormat": "{{app}}-{{brokerId}}", "refId": "B" } ], "title": "Open File Descriptors Pct", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "Kafka Broker Disk Write Throughput", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "Bps" }, "overrides": [] }, "gridPos": { "h": 7, "w": 5, "x": 0, "y": 66 }, "id": 23763572002, "links": [], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "$data_source" }, "expr": "irate(kafka_server_kafkaserver_linux_disk_write_bytes{namespace=\"$kubernetes_namespace\"}[3m])", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "broker-{{brokerId}}", "refId": "A" } ], "title": "Broker Disk Write Throughput", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "Kafka Broker Disk Read Throughput", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "Bps" }, "overrides": [] }, "gridPos": { "h": 7, "w": 5, "x": 5, "y": 66 }, "id": 23763572001, "interval": "1m", "links": [], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "$data_source" }, "expr": "irate(kafka_server_kafkaserver_linux_disk_read_bytes{namespace=\"$kubernetes_namespace\"}[3m])", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "broker-{{brokerId}}", "refId": "A" } ], "title": "Broker Disk Read Throughput", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "fieldConfig": { "defaults": { "unit": "bytes", "decimals": 2, "min": 0, "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 0, "gradientMode": "none", "spanNulls": false, "showPoints": "never", "pointSize": 5, "stacking": { "mode": "normal", "group": "A" }, "axisPlacement": "auto", "axisLabel": "", "lineInterpolation": "linear", "barAlignment": 0, "insertNulls": false, "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "thresholdsStyle": { "mode": "off" } } }, "overrides": [] }, "gridPos": { "h": 7, "w": 5, "x": 10, "y": 66 }, "id": 91, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "max", "last" ], "sortBy": "last", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "topk(30, kafka:log:size:topic:broker{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",topic=~\"$kafka_topic\"})", "interval": "", "legendFormat": "{{brokerId}}:{{topic}}", "refId": "B" } ], "title": "Topic Size per Broker", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "Top 10 topics wrt to number of segments per broker", "fieldConfig": { "defaults": { "unit": "none", "decimals": 0, "min": 0, "custom": { "drawStyle": "bars", "lineWidth": 1, "fillOpacity": 0, "gradientMode": "none", "spanNulls": false, "showPoints": "never", "pointSize": 5, "stacking": { "mode": "normal", "group": "A" }, "axisPlacement": "auto", "axisLabel": "", "lineInterpolation": "linear", "barAlignment": 0, "insertNulls": false, "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "thresholdsStyle": { "mode": "off" } } }, "overrides": [] }, "gridPos": { "h": 7, "w": 5, "x": 15, "y": 66 }, "id": 104, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "max", "last" ], "sortBy": "last", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "topk(30, kafka:log:numlogsegments{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",pod=~\"$cluster_name-($kafka_broker)\",topic=~\"$kafka_topic\"})", "format": "time_series", "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{brokerId}}::{{topic}}", "refId": "A" } ], "title": "Topic Segments #", "type": "timeseries" }, { "collapsed": false, "datasource": { "type": "prometheus", "uid": "$data_source" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 73 }, "id": 114, "panels": [], "title": "Performance", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 74 }, "id": 120, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "irate(kafka_network_requestmetrics_totaltimems_count{request=\"Produce\", namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",pod=~\"$cluster_name-($kafka_broker)\"}[1m])", "hide": true, "legendFormat": "{{brokerId}}", "refId": "A" }, { "expr": "kafka_network_requestmetrics_totaltimems{request=\"Produce\", namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",pod=~\"$cluster_name-($kafka_broker)\", quantile=\"0.999\"}", "interval": "", "legendFormat": "q{{brokerId}}", "refId": "B" } ], "title": "Broker Produce time ms (p.999)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 74 }, "id": 118, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "irate(kafka_network_requestmetrics_totaltimems_count{request=\"FetchConsumer\", namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",pod=~\"$cluster_name-($kafka_broker)\"}[1m])", "hide": true, "legendFormat": "{{brokerId}}", "refId": "A" }, { "expr": "kafka_network_requestmetrics_totaltimems{request=\"FetchConsumer\", namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",pod=~\"$cluster_name-($kafka_broker)\", quantile=\"0.999\"}", "interval": "", "legendFormat": "{{brokerId}}", "refId": "B" } ], "title": "Broker FetchConsumer time ms (p.999)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 74 }, "id": 119, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "irate(kafka_network_requestmetrics_totaltimems_count{request=\"FetchFollower\", namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",pod=~\"$cluster_name-($kafka_broker)\"}[1m])", "hide": true, "legendFormat": "{{brokerId}}", "refId": "A" }, { "expr": "kafka_network_requestmetrics_totaltimems{request=\"FetchFollower\", namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",pod=~\"$cluster_name-($kafka_broker)\", quantile=\"0.999\"}", "interval": "", "legendFormat": "q{{brokerId}}", "refId": "B" } ], "title": "Broker FetchFollower time ms (p.999)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "fieldConfig": { "defaults": { "unit": "ms", "custom": { "drawStyle": "bars", "lineWidth": 1, "fillOpacity": 0, "gradientMode": "none", "spanNulls": false, "showPoints": "never", "pointSize": 2, "stacking": { "mode": "normal", "group": "A" }, "axisPlacement": "auto", "axisLabel": "", "lineInterpolation": "linear", "barAlignment": 0, "insertNulls": false, "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "thresholdsStyle": { "mode": "off" } } }, "overrides": [ { "matcher": { "id": "byRegexp", "options": "/(\\d+)\\-req\\-queue/" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#73BF69" } } ] }, { "matcher": { "id": "byRegexp", "options": "/(\\d+)\\-req\\-local/" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#F2495C" } } ] }, { "matcher": { "id": "byRegexp", "options": "/(\\d+)\\-resp\\-send/" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#B877D9" } } ] }, { "matcher": { "id": "byRegexp", "options": "/(\\d+)\\-resp\\-queue/" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#5794F2" } } ] }, { "matcher": { "id": "byRegexp", "options": "/(\\d+)\\-req\\-remote/" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#FADE2A" } } ] } ] }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 81 }, "id": 125, "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "kafka_network_requestmetrics_requestqueuetimems{request=\"Produce\", namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",pod=~\"$cluster_name-($kafka_broker)\", quantile=\"0.999\"}>0", "legendFormat": "{{brokerId}}-req-queue", "refId": "B" }, { "expr": "kafka_network_requestmetrics_localtimems{request=\"Produce\", namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",pod=~\"$cluster_name-($kafka_broker)\", quantile=\"0.999\"}>0", "legendFormat": "{{brokerId}}-req-local", "refId": "C" }, { "expr": "kafka_network_requestmetrics_remotetimems{request=\"Produce\", namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",pod=~\"$cluster_name-($kafka_broker)\", quantile=\"0.999\"}>0", "legendFormat": "{{brokerId}}-req-remote", "refId": "A" }, { "expr": "kafka_network_requestmetrics_responsequeuetimems{request=\"Produce\", namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",pod=~\"$cluster_name-($kafka_broker)\", quantile=\"0.999\"}>0", "legendFormat": "{{brokerId}}-resp-queue", "refId": "D" }, { "expr": "kafka_network_requestmetrics_responsesendtimems{request=\"Produce\", namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",pod=~\"$cluster_name-($kafka_broker)\", quantile=\"0.999\"}>0", "legendFormat": "{{brokerId}}-resp-send", "refId": "E" } ], "title": "Broker Produce Request Time Breakdown (p.999)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "", "fieldConfig": { "defaults": { "unit": "ms", "custom": { "drawStyle": "bars", "lineWidth": 1, "fillOpacity": 0, "gradientMode": "none", "spanNulls": false, "showPoints": "never", "pointSize": 2, "stacking": { "mode": "normal", "group": "A" }, "axisPlacement": "auto", "axisLabel": "", "lineInterpolation": "linear", "barAlignment": 0, "insertNulls": false, "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "thresholdsStyle": { "mode": "off" } } }, "overrides": [ { "matcher": { "id": "byRegexp", "options": "/(\\d+)\\-req\\-queue/" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#73BF69" } } ] }, { "matcher": { "id": "byRegexp", "options": "/(\\d+)\\-req\\-local/" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#F2495C" } } ] }, { "matcher": { "id": "byRegexp", "options": "/(\\d+)\\-resp\\-send/" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#B877D9" } } ] }, { "matcher": { "id": "byRegexp", "options": "/(\\d+)\\-resp\\-queue/" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#5794F2" } } ] }, { "matcher": { "id": "byRegexp", "options": "/(\\d+)\\-req\\-remote/" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#FADE2A" } } ] } ] }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 81 }, "id": 126, "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "kafka_network_requestmetrics_requestqueuetimems{request=\"FetchConsumer\", namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",pod=~\"$cluster_name-($kafka_broker)\", quantile=\"0.999\"}>0", "legendFormat": "{{brokerId}}-req-queue", "refId": "B" }, { "expr": "kafka_network_requestmetrics_localtimems{request=\"FetchConsumer\", namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",pod=~\"$cluster_name-($kafka_broker)\", quantile=\"0.999\"}>0", "legendFormat": "{{brokerId}}-req-local", "refId": "C" }, { "expr": "kafka_network_requestmetrics_remotetimems{request=\"FetchConsumer\", namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",pod=~\"$cluster_name-($kafka_broker)\", quantile=\"0.999\"}>0", "legendFormat": "{{brokerId}}-req-remote", "refId": "A" }, { "expr": "kafka_network_requestmetrics_responsequeuetimems{request=\"FetchConsumer\", namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",pod=~\"$cluster_name-($kafka_broker)\", quantile=\"0.999\"}>0", "legendFormat": "{{brokerId}}-resp-queue", "refId": "D" }, { "expr": "kafka_network_requestmetrics_responsesendtimems{request=\"FetchConsumer\", namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",pod=~\"$cluster_name-($kafka_broker)\", quantile=\"0.999\"}>0", "legendFormat": "{{brokerId}}-resp-send", "refId": "E" } ], "title": "Broker Consume Request Time Breakdown (p.999)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "Replication Rate Per Second", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 1, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "Bps" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": "out-.+" }, "properties": [ { "id": "custom.axisPlacement", "value": "auto" }, { "id": "custom.transform", "value": "negative-Y" } ] } ] }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 81 }, "id": 111, "links": [], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "sum(irate(kafka_server_brokertopicmetrics_replicationbytesin_total{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",pod=~\"$cluster_name-($kafka_broker)\"}[1m])) by (brokerId)", "format": "time_series", "intervalFactor": 1, "legendFormat": "in-{{brokerId}}", "refId": "A" }, { "expr": "sum(irate(kafka_server_brokertopicmetrics_replicationbytesout_total{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",pod=~\"$cluster_name-($kafka_broker)\"}[1m])) by (brokerId)", "format": "time_series", "intervalFactor": 1, "legendFormat": "out-{{brokerId}}", "refId": "B" } ], "title": "Replication Rate Per Second", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "fieldConfig": { "defaults": { "unit": "short", "decimals": 0, "custom": { "drawStyle": "bars", "lineWidth": 1, "fillOpacity": 0, "gradientMode": "none", "spanNulls": false, "showPoints": "never", "pointSize": 0.5, "stacking": { "mode": "normal", "group": "A" }, "axisPlacement": "auto", "axisLabel": "", "lineInterpolation": "linear", "barAlignment": 0, "insertNulls": false, "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "thresholdsStyle": { "mode": "off" } } }, "overrides": [] }, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 88 }, "id": 121, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "max", "last" ], "sortBy": "last", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "sort_desc(sum(kafka:cluster:broker:topic:urp{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",kafka_cr=\"$cluster_name\", pod=~\"$cluster_name-($kafka_broker)\", topic=~\"$kafka_topic\"}) by (namespace,kafka_cr,brokerId)) ", "format": "time_series", "instant": false, "intervalFactor": 1, "legendFormat": "{{brokerId}}", "refId": "A" } ], "title": "Under Replicated Partitions", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 88 }, "id": 129, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "sum(kafka_log_logflushstats_logflushrateandtimems{quantile=\"0.999\", namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",pod=~\"$cluster_name-($kafka_broker)\"}) by (brokerId)", "legendFormat": "{{brokerId}}", "refId": "A" } ], "title": "Broker Flush Time", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "fieldConfig": { "defaults": { "unit": "short", "min": 0 }, "overrides": [ { "matcher": { "id": "byRegexp", "options": "/.+BLOCKED/i" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#F2495C" } } ] }, { "matcher": { "id": "byRegexp", "options": "/.+RUNNABLE/i" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#73BF69" } } ] }, { "matcher": { "id": "byRegexp", "options": "/.+-WAITING/i" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#FADE2A" } } ] }, { "matcher": { "id": "byRegexp", "options": "/.+-TIMED_WAITING/i" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#FF9830" } } ] } ] }, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 88 }, "id": 161, "options": { "orientation": "horizontal", "displayMode": "gradient", "showUnfilled": true, "valueMode": "color", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "targets": [ { "expr": "jvm_threads_state{namespace=\"$kubernetes_namespace\",pod=~\"$cluster_name-($kafka_broker)\",kafka_cr=\"$cluster_name\"}", "instant": true, "interval": "", "legendFormat": "{{app}}-{{brokerId}}-{{state}}", "refId": "B" } ], "title": "JVM Threads", "type": "bargauge" }, { "collapsed": true, "datasource": { "type": "prometheus", "uid": "$data_source" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 94 }, "id": 128, "panels": [ { "datasource": { "type": "prometheus", "uid": "$data_source" }, "fieldConfig": { "defaults": { "unit": "ms", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 0, "gradientMode": "none", "spanNulls": false, "showPoints": "never", "pointSize": 2, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto", "axisLabel": "", "lineInterpolation": "linear", "barAlignment": 0, "insertNulls": false, "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "thresholdsStyle": { "mode": "off" } } }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 94 }, "id": 131, "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "expr": "sum(kafka_server_zookeeperclientmetrics_zookeeperrequestlatencyms{quantile=\"0.999\", namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",pod=~\"$cluster_name-($kafka_broker)\"}) by (brokerId)", "legendFormat": "{{brokerId}}", "refId": "A" } ], "title": "Broker ZK Request Latency", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "fieldConfig": { "defaults": { "unit": "short", "decimals": 2 }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 94 }, "id": 134, "options": { "showHeader": true, "cellHeight": "sm" }, "targets": [ { "expr": "kafka_log_logmanager_logdirectoryoffline{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",pod=~\"$cluster_name-($kafka_broker)\"}>0", "format": "time_series", "instant": true, "legendFormat": "Broker:{{brokerId}} Disk:{{logDirectory}}", "refId": "A" } ], "title": "Offline broker disks", "type": "table", "transformations": [ { "id": "reduce", "options": { "reducers": [ "lastNotNull" ], "includeTimeField": false } } ] }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 0, "gradientMode": "none", "spanNulls": false, "showPoints": "never", "pointSize": 2, "stacking": { "mode": "normal", "group": "A" }, "axisPlacement": "auto", "axisLabel": "", "lineInterpolation": "linear", "barAlignment": 0, "insertNulls": false, "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "thresholdsStyle": { "mode": "off" } } }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 12, "y": 94 }, "id": 23763572004, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": false, "calcs": [ "last" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "exemplar": true, "expr": "kafka:topic_orphan_replicas_size{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\"}", "format": "time_series", "instant": false, "interval": "", "legendFormat": "{{topic}}", "refId": "A" } ], "title": "Kafka Topics With Orphan Replicas (Space Waste)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$data_source" }, "description": "", "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 0, "gradientMode": "none", "spanNulls": false, "showPoints": "never", "pointSize": 2, "stacking": { "mode": "normal", "group": "A" }, "axisPlacement": "auto", "axisLabel": "", "lineInterpolation": "linear", "barAlignment": 0, "insertNulls": false, "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "thresholdsStyle": { "mode": "off" } } }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 94 }, "id": 23763572005, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": false, "calcs": [ "last" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "exemplar": true, "expr": "kafka:topic_orphan_replicas_count{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\"}", "format": "time_series", "instant": false, "interval": "", "legendFormat": "{{topic}}", "refId": "A" } ], "title": "Kafka Topics With Orphan Replicas (Partition Count)", "type": "timeseries" } ], "title": "Errors", "type": "row" } ], "refresh": "1m", "schemaVersion": 41, "style": "dark", "tags": [ "kafka-operator", "kafka" ], "templating": { "list": [ { "current": { "selected": false, "text": "kafka", "value": "kafka" }, "datasource": { "type": "prometheus", "uid": "$data_source" }, "definition": "", "hide": 0, "includeAll": false, "label": "Namespace", "multi": false, "name": "kubernetes_namespace", "options": [], "query": { "query": "query_result(kafka_server_app_info_version)", "refId": "Prometheus-kubernetes_namespace-Variable-Query" }, "refresh": 1, "regex": "/.*namespace=\"([^\"]*).*/", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false }, { "current": { "selected": false, "text": "kafka", "value": "kafka" }, "datasource": { "type": "prometheus", "uid": "$data_source" }, "definition": "query_result(kafka_server_app_info_version{namespace=\"$kubernetes_namespace\"})", "hide": 0, "includeAll": false, "label": "Cluster Name", "multi": false, "name": "cluster_name", "options": [], "query": { "query": "query_result(kafka_server_app_info_version{namespace=\"$kubernetes_namespace\"})", "refId": "Prometheus-cluster_name-Variable-Query" }, "refresh": 1, "regex": "/.*kafka_cr=\"([^\"]*).*/", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false }, { "current": { "selected": false, "text": "cluster-1", "value": "cluster-1" }, "datasource": { "type": "prometheus", "uid": "$data_source" }, "definition": "label_values(kafka_server_replicamanager_leadercount{namespace=\\\"$kubernetes_namespace\\\"}, canary_cluster)", "hide": 2, "includeAll": false, "label": "Kafka cluster", "multi": false, "name": "kafka_cluster", "options": [], "query": { "query": "label_values(kafka_server_replicamanager_leadercount{namespace=\"$kubernetes_namespace\"}, canary_cluster)", "refId": "Prometheus-kafka_cluster-Variable-Query" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": ".*", "current": { "selected": false, "text": "All", "value": "$__all" }, "datasource": { "type": "prometheus", "uid": "$data_source" }, "definition": "query_result(max_over_time(kafka_server_app_info_version{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\"}[${__range_s}s]))", "hide": 0, "includeAll": true, "label": "Broker", "multi": true, "name": "kafka_broker", "options": [], "query": { "query": "query_result(max_over_time(kafka_server_app_info_version{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\"}[${__range_s}s]))", "refId": "Prometheus-kafka_broker-Variable-Query" }, "refresh": 1, "regex": "/.*pod=\"$cluster_name-([^\"]*).*/", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": ".+", "current": { "selected": false, "text": "All", "value": "$__all" }, "datasource": { "type": "prometheus", "uid": "$data_source" }, "definition": "query_result(sort_desc((sum_over_time( sum by(topic) (increase(kafka_server_brokertopicmetrics_bytesin_total{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",topic=~\".*\",topic!=\"\"}[90s])) [${__range_s}s:90s] ) )+(sum_over_time( sum by(topic) (increase(kafka_server_brokertopicmetrics_bytesout_total{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",topic=~\".*\",topic!=\"\"}[90s])) [${__range_s}s:90s] ) ) OR (sum_over_time( sum by(topic) (increase(kafka_server_brokertopicmetrics_bytesin_total{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",topic=~\".*\",topic!=\"\"}[90s])) [${__range_s}s:90s] )) OR (sum_over_time( sum by(topic) (increase(kafka_server_brokertopicmetrics_bytesout_total{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",topic=~\".*\",topic!=\"\"}[90s])) [${__range_s}s:90s] ) ) ))", "hide": 0, "includeAll": true, "label": "Topic", "multi": true, "name": "kafka_topic", "options": [], "query": { "query": "query_result(sort_desc((sum_over_time( sum by(topic) (increase(kafka_server_brokertopicmetrics_bytesin_total{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",topic=~\".*\",topic!=\"\"}[90s])) [${__range_s}s:90s] ) )+(sum_over_time( sum by(topic) (increase(kafka_server_brokertopicmetrics_bytesout_total{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",topic=~\".*\",topic!=\"\"}[90s])) [${__range_s}s:90s] ) ) OR (sum_over_time( sum by(topic) (increase(kafka_server_brokertopicmetrics_bytesin_total{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",topic=~\".*\",topic!=\"\"}[90s])) [${__range_s}s:90s] )) OR (sum_over_time( sum by(topic) (increase(kafka_server_brokertopicmetrics_bytesout_total{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",topic=~\".*\",topic!=\"\"}[90s])) [${__range_s}s:90s] ) ) ))", "refId": "Prometheus-kafka_topic-Variable-Query" }, "refresh": 1, "regex": "/.*topic=\"([^\"]*).*/", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": ".*", "current": { "selected": true, "text": [ "All" ], "value": [ "$__all" ] }, "datasource": { "type": "prometheus", "uid": "$data_source" }, "definition": "query_result(kafka:topic_partition_size:count{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",topic=~\"$kafka_topic\"})", "hide": 0, "includeAll": true, "label": "Partition", "multi": true, "name": "kafka_partition", "options": [], "query": { "query": "query_result(kafka:topic_partition_size:count{namespace=\"$kubernetes_namespace\",kafka_cr=\"$cluster_name\",topic=~\"$kafka_topic\"})", "refId": "Prometheus-kafka_partition-Variable-Query" }, "refresh": 1, "regex": "/.*partition=\"([^\"]*).*/", "skipUrlSync": false, "sort": 3, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false }, { "current": { "selected": false, "text": "Prometheus", "value": "Prometheus" }, "hide": 0, "includeAll": false, "multi": false, "name": "data_source", "options": [], "query": "prometheus", "queryValue": "", "refresh": 1, "regex": "/^(?!default)/", "skipUrlSync": false, "type": "datasource" } ] }, "time": { "from": "now-3h", "to": "now" }, "timepicker": { "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ] }, "timezone": "", "title": "Kafka Looking Glass", "uid": null, "version": 1, "weekStart": "" }