<- Back to Templates
Kafka
By KloudMate
Kafka Monitoring using OpenTelemetry (OTel) provides unified observability into Kafka producers, consumers, brokers, and the data pipelines that depend on them
Template JSON
{
"name": "Kafka",
"description": "",
"data": {
"tz": "local",
"links": [],
"panels": [
{
"id": "dg0lflmca4",
"tz": "local",
"axis": {
"x": {
"date_style": "auto"
},
"y": {
"format": "number",
"suffix": "{brokers}",
"decimals": 3
}
},
"STATS": {},
"title": "Kafka Brokers",
"panelType": "STATS",
"timeRange": "1h",
"statConfig": {
"color": {
"scheme": "fixed",
"fixedColor": "#4a4a4a"
},
"statView": "grid",
"baseColor": "#4CBB17",
"colorMode": "",
"calculation": "last"
},
"timeseries": {
"legend": {
"view": "list",
"values": []
}
},
"description": "Number of Kafka Brokers",
"node_configs": [
{
"page": 1,
"stat": "",
"type": "query",
"hidden": false,
"metric": "",
"nodeId": "A",
"region": "",
"source": "kloudmate",
"dataset": "metrics",
"filters": [],
"groupBy": [],
"orderBy": [],
"interval": "",
"namespace": "",
"dimensions": [],
"metricType": "",
"time_range": {
"to": 0,
"from": 600
},
"aggregation": {
"stat": "AVG",
"type": "",
"unit": "{brokers}",
"field": "",
"value": "kafka_brokers",
"metric": "kafka_brokers",
"metricType": "Sum",
"temporality": "Cumulative"
},
"datasourceId": "kloudmate",
"responseType": "series"
}
],
"time_options": {
"override": false,
"time_shift": ""
}
},
{
"id": "b6fl0lyvdrg",
"tz": "local",
"axis": {
"x": {
"date_style": "auto"
},
"y": {
"format": "duration",
"decimals": 3,
"multiplyBy": 1000
}
},
"STATS": {},
"table": {
"columns": [
{
"key": "A:AVG()",
"format": {
"format": "duration",
"decimals": 1,
"multiplyBy": 1000
},
"rename": "Period"
},
{
"key": "broker",
"rename": ""
}
]
},
"title": "Broker Logs Retention Period",
"tooltip": {
"mode": "all"
},
"panelType": "table",
"timeRange": "8h",
"TimeSeries": {},
"statConfig": {
"format": "duration_nano",
"statView": "grid",
"baseColor": "#4CBB17",
"calculation": "last"
},
"timeseries": {
"legend": {
"view": "table",
"values": [
{
"label": "Max",
"value": "max"
}
]
}
},
"description": "Log retention period of Broker",
"node_configs": [
{
"page": 1,
"stat": "",
"type": "query",
"hidden": false,
"metric": "",
"nodeId": "A",
"region": "",
"source": "kloudmate",
"dataset": "metrics",
"filters": [],
"groupBy": [
{
"type": "string",
"field": "broker"
}
],
"orderBy": [],
"interval": "",
"namespace": "",
"dimensions": [],
"metricType": "",
"time_range": {
"to": 0,
"from": 600
},
"aggregation": {
"stat": "AVG",
"type": "",
"unit": "s",
"field": "",
"value": "kafka_broker_log_retention_period",
"metric": "kafka_broker_log_retention_period",
"metricType": "Gauge",
"temporality": "Cumulative"
},
"datasourceId": "kloudmate",
"responseType": "number"
}
],
"time_options": {
"override": false,
"time_shift": ""
}
},
{
"id": "djze5kgybyr",
"tz": "local",
"axis": {
"x": {
"date_style": "auto"
},
"y": {
"format": "number",
"decimals": 1
}
},
"table": {
"columns": [
{
"key": "A:LAST()",
"format": {
"format": "number",
"decimals": 2
},
"rename": "Value"
},
{
"key": "topic",
"rename": ""
},
{
"key": "group",
"rename": ""
}
]
},
"title": "Consumer Group Offset",
"panelType": "TimeSeries",
"timeRange": "1h",
"TimeSeries": {},
"statConfig": {
"statView": "grid",
"baseColor": "#4CBB17",
"calculation": "last"
},
"timeseries": {
"legend": {
"view": "table",
"values": [
{
"label": "Max",
"value": "max"
},
{
"label": "Last",
"value": "last"
}
]
}
},
"description": "The sum of latest committed offsets for consumer group by Topic",
"node_configs": [
{
"page": 1,
"stat": "",
"type": "query",
"hidden": false,
"metric": "",
"nodeId": "A",
"region": "",
"source": "kloudmate",
"dataset": "metrics",
"filters": [
{
"type": "string",
"field": "topic",
"value": "$Topic",
"operator": "="
}
],
"groupBy": [
{
"type": "string",
"field": "topic"
},
{
"type": "string",
"field": "group"
}
],
"orderBy": [],
"interval": "",
"namespace": "",
"dimensions": [],
"metricType": "",
"time_range": {
"to": 0,
"from": 600
},
"aggregation": {
"stat": "LAST",
"type": "",
"unit": "1",
"field": "",
"value": "kafka_consumer_group_offset_sum",
"metric": "kafka_consumer_group_offset_sum",
"metricType": "Gauge",
"temporality": "Cumulative"
},
"datasourceId": "kloudmate",
"responseType": "series"
}
],
"time_options": {
"override": false,
"time_shift": ""
}
},
{
"id": "fo9r7tw7mgo",
"tz": "local",
"axis": {
"x": {
"date_style": "auto"
},
"y": {
"format": "number",
"decimals": 2
}
},
"table": {
"columns": [
{
"key": "A:AVG()",
"format": {
"decimals": 2
},
"rename": "count"
},
{
"key": "topic",
"rename": ""
},
{
"key": "group",
"rename": ""
}
]
},
"title": "Consumer Group Lag",
"panelType": "TimeSeries",
"timeRange": "1d",
"TimeSeries": {},
"statConfig": {
"statView": "grid",
"baseColor": "#4CBB17",
"calculation": "last"
},
"timeseries": {
"legend": {
"view": "table",
"values": [
{
"label": "Max",
"value": "max"
},
{
"label": "Last",
"value": "last"
}
]
}
},
"description": "Represents how many messages a consumer group is behind the latest data in the topic",
"node_configs": [
{
"page": 1,
"stat": "",
"type": "query",
"hidden": false,
"metric": "",
"nodeId": "A",
"region": "",
"source": "kloudmate",
"dataset": "metrics",
"filters": [
{
"type": "string",
"field": "topic",
"value": "$Topic",
"operator": "="
}
],
"groupBy": [
{
"type": "string",
"field": "topic"
},
{
"type": "string",
"field": "group"
}
],
"orderBy": [],
"interval": "",
"namespace": "",
"dimensions": [],
"metricType": "",
"time_range": {
"to": 0,
"from": 600
},
"aggregation": {
"stat": "SUM",
"type": "",
"unit": "1",
"field": "",
"value": "kafka_consumer_group_lag",
"metric": "kafka_consumer_group_lag",
"metricType": "Gauge",
"temporality": "Cumulative"
},
"datasourceId": "kloudmate",
"responseType": "series"
}
],
"time_options": {
"override": false,
"time_shift": ""
}
},
{
"id": "67b09a20mxm",
"tz": "local",
"axis": {
"x": {
"date_style": "auto"
},
"y": {
"format": "number",
"suffix": "{partitions}",
"decimals": 3
}
},
"STATS": {},
"title": "Topic Partitions",
"panelType": "STATS",
"timeRange": "1h",
"TimeSeries": {},
"statConfig": {
"color": {
"scheme": "fixed",
"fixedColor": "#dd223a"
},
"decimals": 0,
"statView": "grid",
"baseColor": "#4CBB17",
"colorMode": "",
"calculation": {
"label": "Last",
"value": "last"
}
},
"timeseries": {
"legend": {
"view": "list",
"values": []
}
},
"description": "Number of partitions in a in the Topic",
"node_configs": [
{
"page": 1,
"stat": "",
"type": "query",
"hidden": false,
"metric": "",
"nodeId": "A",
"region": "",
"source": "kloudmate",
"dataset": "metrics",
"filters": [
{
"type": "string",
"field": "topic",
"value": "$Topic",
"operator": "="
}
],
"groupBy": [
{
"type": "string",
"field": "topic"
}
],
"orderBy": [],
"interval": "",
"namespace": "",
"dimensions": [],
"metricType": "",
"time_range": {
"to": 0,
"from": 600
},
"aggregation": {
"stat": "SUM",
"type": "",
"unit": "{partitions}",
"field": "",
"value": "kafka_topic_partitions",
"metric": "kafka_topic_partitions",
"metricType": "Sum",
"temporality": "Cumulative"
},
"datasourceId": "kloudmate",
"responseType": "series"
}
],
"time_options": {
"override": false,
"time_shift": ""
}
},
{
"id": "2iv8u1d4w09",
"tz": "local",
"axis": {
"x": {
"date_style": "auto"
},
"y": {
"format": "number",
"decimals": 3
}
},
"STATS": {},
"title": "Topic Replication Factor",
"panelType": "TimeSeries",
"timeRange": "6h",
"TimeSeries": {},
"statConfig": {
"statView": "grid",
"baseColor": "#4CBB17",
"calculation": "last"
},
"timeseries": {
"legend": {
"view": "list",
"values": []
}
},
"description": "Indicates how many copies of each partition Kafka keeps",
"node_configs": [
{
"page": 1,
"stat": "",
"type": "query",
"hidden": false,
"metric": "",
"nodeId": "A",
"region": "",
"source": "kloudmate",
"dataset": "metrics",
"filters": [
{
"type": "string",
"field": "topic",
"value": "$Topic",
"operator": "="
}
],
"groupBy": [
{
"type": "string",
"field": "topic"
}
],
"orderBy": [],
"interval": "",
"namespace": "",
"dimensions": [],
"metricType": "",
"time_range": {
"to": 0,
"from": 600
},
"aggregation": {
"stat": "AVG",
"type": "",
"unit": "1",
"field": "",
"value": "kafka_topic_replication_factor",
"metric": "kafka_topic_replication_factor",
"metricType": "Gauge",
"temporality": "Cumulative"
},
"datasourceId": "kloudmate",
"responseType": "series"
}
],
"time_options": {
"override": false,
"time_shift": ""
}
},
{
"id": "l7fi2p70wms",
"tz": "local",
"axis": {
"x": {
"date_style": "auto"
},
"y": {
"format": "bytes",
"decimals": 3
}
},
"STATS": {},
"title": "Topic Log Retention Size",
"panelType": "STATS",
"timeRange": "6h",
"TimeSeries": {},
"statConfig": {
"color": {
"scheme": "fixed",
"fixedColor": "#4688d6"
},
"format": "bytes",
"decimals": 2,
"statView": "grid",
"baseColor": "#4CBB17",
"colorMode": "",
"calculation": "last"
},
"timeseries": {
"legend": {
"view": "list",
"values": []
}
},
"description": "Log retention size of the Topic",
"node_configs": [
{
"page": 1,
"stat": "",
"type": "query",
"hidden": false,
"metric": "",
"nodeId": "A",
"region": "",
"source": "kloudmate",
"dataset": "metrics",
"filters": [
{
"type": "string",
"field": "topic",
"value": "$Topic",
"operator": "="
}
],
"groupBy": [],
"orderBy": [],
"interval": "",
"namespace": "",
"dimensions": [],
"metricType": "",
"time_range": {
"to": 0,
"from": 600
},
"aggregation": {
"stat": "MAX",
"type": "",
"unit": "By",
"field": "",
"value": "kafka_topic_log_retention_size",
"metric": "kafka_topic_log_retention_size",
"metricType": "Gauge",
"temporality": "Cumulative"
},
"datasourceId": "kloudmate",
"responseType": "series"
}
],
"time_options": {
"override": false,
"time_shift": ""
}
},
{
"id": "9n8mdfkac2n",
"tz": "local",
"axis": {
"x": {
"date_style": "auto"
},
"y": {
"format": "number",
"suffix": "{replicas}",
"decimals": 3
}
},
"title": "Topic Min In-Sync Replicas",
"tooltip": {
"mode": "all"
},
"panelType": "TimeSeries",
"timeRange": "1h",
"statConfig": {
"statView": "grid",
"baseColor": "#4CBB17",
"calculation": "last"
},
"timeseries": {
"legend": {
"view": "list",
"values": []
}
},
"description": "the minimum number of in-sync replicas (ISR) that must acknowledge a write for it to be considered successful.",
"node_configs": [
{
"page": 1,
"stat": "",
"type": "query",
"hidden": false,
"metric": "",
"nodeId": "A",
"region": "",
"source": "kloudmate",
"dataset": "metrics",
"filters": [
{
"type": "string",
"field": "topic",
"value": "$Topic",
"operator": "="
}
],
"groupBy": [
{
"type": "string",
"field": "topic"
}
],
"orderBy": [],
"interval": "",
"namespace": "",
"dimensions": [],
"metricType": "",
"time_range": {
"to": 0,
"from": 600
},
"aggregation": {
"stat": "SUM",
"type": "",
"unit": "{replicas}",
"field": "",
"value": "kafka_topic_min_insync_replicas",
"metric": "kafka_topic_min_insync_replicas",
"metricType": "Gauge",
"temporality": "Cumulative"
},
"datasourceId": "kloudmate"
}
],
"time_options": {
"override": false,
"time_shift": ""
}
},
{
"id": "aawt6s7nkwd",
"tz": "local",
"axis": {
"x": {
"date_style": "auto"
},
"y": {
"format": "number",
"suffix": "{replicas}",
"decimals": 3
}
},
"table": {
"columns": [
{
"key": "A:AVG()",
"rename": "replicas"
},
{
"key": "topic",
"rename": ""
},
{
"key": "partition",
"rename": ""
}
]
},
"title": "Partitions Replicas",
"tooltip": {
"mode": "all"
},
"panelType": "table",
"timeRange": "2h",
"statConfig": {
"statView": "grid",
"baseColor": "#4CBB17",
"calculation": "last"
},
"timeseries": {
"legend": {
"view": "table",
"values": [
{
"label": "Mean",
"value": "mean"
},
{
"label": "Last",
"value": "last"
}
]
}
},
"description": "Number of replicas for partition of topic",
"node_configs": [
{
"page": 1,
"stat": "",
"type": "query",
"hidden": false,
"metric": "",
"nodeId": "A",
"region": "",
"source": "kloudmate",
"dataset": "metrics",
"filters": [
{
"type": "string",
"field": "topic",
"value": "$Topic",
"operator": "="
}
],
"groupBy": [
{
"type": "string",
"field": "topic"
},
{
"type": "string",
"field": "partition"
}
],
"orderBy": [],
"interval": "",
"namespace": "",
"dimensions": [],
"metricType": "",
"time_range": {
"to": 0,
"from": 600
},
"aggregation": {
"stat": "AVG",
"type": "",
"unit": "{replicas}",
"field": "",
"value": "kafka_partition_replicas",
"metric": "kafka_partition_replicas",
"metricType": "Sum",
"temporality": "Cumulative"
},
"datasourceId": "kloudmate",
"responseType": "number"
}
],
"time_options": {
"override": false,
"time_shift": ""
}
},
{
"id": "2hy2n01d3yg",
"tz": "local",
"axis": {
"x": {
"date_style": "auto"
},
"y": {
"format": "number",
"suffix": "{replicas}",
"decimals": 3
}
},
"STATS": {},
"title": "Partition Replica IN Sync",
"panelType": "STATS",
"timeRange": "2h",
"statConfig": {
"color": {
"scheme": "fixed",
"fixedColor": "#76c029"
},
"statView": "grid",
"baseColor": "#4CBB17",
"colorMode": "",
"calculation": "last"
},
"timeseries": {
"legend": {
"view": "list",
"values": []
}
},
"description": "Partition Replica IN Sync by Topic",
"node_configs": [
{
"page": 1,
"stat": "",
"type": "query",
"hidden": false,
"metric": "",
"nodeId": "A",
"region": "",
"source": "kloudmate",
"dataset": "metrics",
"filters": [
{
"type": "string",
"field": "topic",
"value": "$Topic",
"operator": "="
}
],
"groupBy": [
{
"type": "string",
"field": "topic"
}
],
"orderBy": [],
"interval": "",
"namespace": "",
"dimensions": [],
"metricType": "",
"time_range": {
"to": 0,
"from": 600
},
"aggregation": {
"stat": "SUM",
"type": "",
"unit": "{replicas}",
"field": "",
"value": "kafka_partition_replicas_in_sync",
"metric": "kafka_partition_replicas_in_sync",
"metricType": "Sum",
"temporality": "Cumulative"
},
"datasourceId": "kloudmate",
"responseType": "series"
}
],
"time_options": {
"override": false,
"time_shift": ""
}
},
{
"id": "26x5the7n9e",
"tz": "local",
"axis": {
"x": {
"date_style": "auto"
},
"y": {
"format": "number",
"suffix": "{members}",
"decimals": 1
}
},
"STATS": {},
"title": "Partition Current Offset",
"tooltip": {
"mode": "all"
},
"panelType": "TimeSeries",
"timeRange": "6h",
"TimeSeries": {},
"statConfig": {
"format": "number",
"decimals": 3,
"statView": "grid",
"baseColor": "#4CBB17",
"calculation": "last"
},
"timeseries": {
"legend": {
"view": "table",
"values": [
{
"label": "Mean",
"value": "mean"
},
{
"label": "Last",
"value": "last"
}
]
}
},
"description": "latest (newest) offset in a partition",
"node_configs": [
{
"page": 1,
"stat": "",
"type": "query",
"hidden": false,
"metric": "",
"nodeId": "A",
"region": "",
"source": "kloudmate",
"dataset": "metrics",
"filters": [
{
"type": "string",
"field": "topic",
"value": "$Topic",
"operator": "="
}
],
"groupBy": [
{
"type": "string",
"field": "partition"
},
{
"type": "string",
"field": "topic"
}
],
"orderBy": [],
"interval": "",
"namespace": "",
"dimensions": [],
"metricType": "",
"time_range": {
"to": 0,
"from": 600
},
"aggregation": {
"stat": "AVG",
"type": "",
"unit": "1",
"field": "",
"value": "kafka_partition_current_offset",
"metric": "kafka_partition_current_offset",
"metricType": "Gauge",
"temporality": "Cumulative"
},
"datasourceId": "kloudmate",
"responseType": "series"
}
],
"time_options": {
"override": false,
"time_shift": ""
}
},
{
"id": "768zl6z2t6",
"tz": "local",
"axis": {
"x": {
"date_style": "auto"
},
"y": {
"format": "number",
"decimals": 1
}
},
"title": "Partition Oldest Offset",
"panelType": "TimeSeries",
"timeRange": "1d",
"statConfig": {
"statView": "grid",
"baseColor": "#4CBB17",
"calculation": "last"
},
"timeseries": {
"legend": {
"view": "table",
"values": [
{
"label": "Mean",
"value": "mean"
},
{
"label": "Last",
"value": "last"
}
]
}
},
"description": "The earliest offset still retained in the Kafka partition.",
"node_configs": [
{
"page": 1,
"stat": "",
"type": "query",
"hidden": false,
"metric": "",
"nodeId": "A",
"region": "",
"source": "kloudmate",
"dataset": "metrics",
"filters": [
{
"type": "string",
"field": "topic",
"value": "$Topic",
"operator": "="
}
],
"groupBy": [
{
"type": "string",
"field": "partition"
},
{
"type": "string",
"field": "topic"
}
],
"orderBy": [],
"interval": "",
"namespace": "",
"dimensions": [],
"metricType": "",
"time_range": {
"to": 0,
"from": 600
},
"aggregation": {
"stat": "AVG",
"type": "",
"unit": "1",
"field": "",
"value": "kafka_partition_oldest_offset",
"metric": "kafka_partition_oldest_offset",
"metricType": "Gauge",
"temporality": "Cumulative"
},
"datasourceId": "kloudmate"
}
],
"time_options": {
"override": false,
"time_shift": ""
}
},
{
"id": "1beothrhdn5",
"tz": "local",
"axis": {
"x": {
"date_style": "auto"
},
"y": {
"format": "number",
"decimals": 1
}
},
"STATS": {},
"table": {
"columns": [
{
"key": "A:COUNT()",
"rename": ""
},
{
"key": "group",
"rename": ""
}
]
},
"title": "Consumer Groups",
"panelType": "STATS",
"timeRange": "1h",
"statConfig": {
"color": {
"scheme": "fixed",
"fixedColor": "#f0a834"
},
"statView": "grid",
"baseColor": "#4CBB17",
"colorMode": "",
"calculation": "last"
},
"timeseries": {
"legend": {
"view": "list",
"values": []
}
},
"description": "Number of consumer Groups in the Topic",
"node_configs": [
{
"page": 1,
"stat": "",
"type": "query",
"hidden": false,
"metric": "",
"nodeId": "A",
"region": "",
"source": "kloudmate",
"dataset": "metrics",
"filters": [
{
"type": "string",
"field": "topic",
"value": "$Topic",
"operator": "="
}
],
"groupBy": [],
"orderBy": [],
"interval": "",
"namespace": "",
"dimensions": [],
"metricType": "",
"time_range": {
"to": 0,
"from": 600
},
"aggregation": {
"stat": "COUNT",
"type": "",
"unit": "1",
"field": "",
"value": "kafka_consumer_group_offset_sum",
"metric": "kafka_consumer_group_offset_sum",
"metricType": "Gauge",
"temporality": "Cumulative"
},
"datasourceId": "kloudmate",
"responseType": "series"
}
],
"time_options": {
"override": false,
"time_shift": ""
}
}
],
"timeRange": "1h",
"variables": [
{
"name": "Topic",
"type": "query",
"label": "Topic",
"config": {
"otel": {
"key": {
"type": "string",
"field": "topic"
},
"dataset": "metrics"
},
"source": "cloudwatch",
"queryType": "otel.values"
},
"is_multi": false,
"description": "",
"usedInPanels": [
{
"id": "9n8mdfkac2n",
"name": "Topic Min In-Sync Replicas"
}
],
"usesVariables": [],
"show_all_option": false,
"usedInVariables": []
}
],
"panel_layouts": [
{
"h": 6,
"i": "dg0lflmca4",
"w": 4,
"x": 20,
"y": 0,
"minH": 1,
"minW": 1,
"moved": false,
"static": false
},
{
"h": 10,
"i": "b6fl0lyvdrg",
"w": 8,
"x": 0,
"y": 16,
"minH": 1,
"minW": 1,
"moved": false,
"static": false
},
{
"h": 10,
"i": "djze5kgybyr",
"w": 8,
"x": 8,
"y": 6,
"minH": 1,
"minW": 1,
"moved": false,
"static": false
},
{
"h": 10,
"i": "fo9r7tw7mgo",
"w": 8,
"x": 16,
"y": 6,
"minH": 1,
"minW": 1,
"moved": false,
"static": false
},
{
"h": 6,
"i": "67b09a20mxm",
"w": 5,
"x": 5,
"y": 0,
"minH": 1,
"minW": 1,
"moved": false,
"static": false
},
{
"h": 10,
"i": "2iv8u1d4w09",
"w": 8,
"x": 8,
"y": 16,
"minH": 1,
"minW": 1,
"moved": false,
"static": false
},
{
"h": 6,
"i": "l7fi2p70wms",
"w": 5,
"x": 15,
"y": 0,
"minH": 1,
"minW": 1,
"moved": false,
"static": false
},
{
"h": 10,
"i": "9n8mdfkac2n",
"w": 8,
"x": 16,
"y": 16,
"minH": 1,
"minW": 1,
"moved": false,
"static": false
},
{
"h": 10,
"i": "aawt6s7nkwd",
"w": 8,
"x": 0,
"y": 6,
"minH": 1,
"minW": 1,
"moved": false,
"static": false
},
{
"h": 6,
"i": "2hy2n01d3yg",
"w": 5,
"x": 10,
"y": 0,
"minH": 1,
"minW": 1,
"moved": false,
"static": false
},
{
"h": 10,
"i": "26x5the7n9e",
"w": 12,
"x": 12,
"y": 26,
"minH": 1,
"minW": 1,
"moved": false,
"static": false
},
{
"h": 10,
"i": "768zl6z2t6",
"w": 12,
"x": 0,
"y": 26,
"minH": 1,
"minW": 1,
"moved": false,
"static": false
},
{
"h": 6,
"i": "1beothrhdn5",
"w": 5,
"x": 0,
"y": 0,
"minH": 1,
"minW": 1,
"moved": false,
"static": false
}
],
"default_region": "us-east-1"
},
"tags": []
}