resource "aws_vpc" "vpc" {
cidr_block = "192.168.0.0/22"
}
data "aws_availability_zones" "azs" {
state = "available"
}
resource "aws_subnet" "subnet_az1" {
availability_zone = data.aws_availability_zones.azs.names[0]
cidr_block = "192.168.0.0/24"
vpc_id = aws_vpc.vpc.id
}
resource "aws_subnet" "subnet_az2" {
availability_zone = data.aws_availability_zones.azs.names[1]
cidr_block = "192.168.1.0/24"
vpc_id = aws_vpc.vpc.id
}
resource "aws_subnet" "subnet_az3" {
availability_zone = data.aws_availability_zones.azs.names[2]
cidr_block = "192.168.2.0/24"
vpc_id = aws_vpc.vpc.id
}
resource "aws_security_group" "sg" {
vpc_id = aws_vpc.vpc.id
}
resource "aws_kms_key" "kms" {
description = "example"
}
resource "aws_cloudwatch_log_group" "test" {
name = "msk_broker_logs"
}
resource "aws_s3_bucket" "bucket" {
bucket = "msk-broker-logs-bucket"
}
resource "aws_s3_bucket_acl" "bucket_acl" {
bucket = aws_s3_bucket.bucket.id
acl = "private"
}
resource "aws_iam_role" "firehose_role" {
name = "firehose_test_role"
assume_role_policy = <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Action": "sts:AssumeRole",
"Principal": {
"Service": "firehose.amazonaws.com"
},
"Effect": "Allow",
"Sid": ""
}
]
}
EOF
}
resource "aws_kinesis_firehose_delivery_stream" "test_stream" {
name = "terraform-kinesis-firehose-msk-broker-logs-stream"
destination = "s3"
s3_configuration {
role_arn = aws_iam_role.firehose_role.arn
bucket_arn = aws_s3_bucket.bucket.arn
}
tags = {
LogDeliveryEnabled = "placeholder"
}
lifecycle {
ignore_changes = [
tags["LogDeliveryEnabled"],
]
}
}
resource "aws_msk_cluster" "example" {
cluster_name = "example"
kafka_version = "3.2.0"
number_of_broker_nodes = 3
broker_node_group_info {
instance_type = "kafka.m5.large"
client_subnets = [
aws_subnet.subnet_az1.id,
aws_subnet.subnet_az2.id,
aws_subnet.subnet_az3.id,
]
storage_info {
ebs_storage_info {
volume_size = 1000
}
}
security_groups = [aws_security_group.sg.id]
}
encryption_info {
encryption_at_rest_kms_key_arn = aws_kms_key.kms.arn
}
open_monitoring {
prometheus {
jmx_exporter {
enabled_in_broker = true
}
node_exporter {
enabled_in_broker = true
}
}
}
logging_info {
broker_logs {
cloudwatch_logs {
enabled = true
log_group = aws_cloudwatch_log_group.test.name
}
firehose {
enabled = true
delivery_stream = aws_kinesis_firehose_delivery_stream.test_stream.name
}
s3 {
enabled = true
bucket = aws_s3_bucket.bucket.id
prefix = "logs/msk-"
}
}
}
tags = {
foo = "bar"
}
}
output "zookeeper_connect_string" {
value = aws_msk_cluster.example.zookeeper_connect_string
}
output "bootstrap_brokers_tls" {
description = "TLS connection host:port pairs"
value = aws_msk_cluster.example.bootstrap_brokers_tls
}
create-cluster
--broker-node-group-info <value>
[--client-authentication <value>]
--cluster-name <value>
[--configuration-info <value>]
[--encryption-info <value>]
[--enhanced-monitoring <value>]
[--open-monitoring <value>]
--kafka-version <value>
[--logging-info <value>]
--number-of-broker-nodes <value>
[--tags <value>]
[--cli-input-json | --cli-input-yaml]
[--generate-cli-skeleton <value>]
[--debug]
[--endpoint-url <value>]
[--no-verify-ssl]
[--no-paginate]
[--output <value>]
[--query <value>]
[--profile <value>]
[--region <value>]
[--version <value>]
[--color <value>]
[--no-sign-request]
[--ca-bundle <value>]
[--cli-read-timeout <value>]
[--cli-connect-timeout <value>]
[--cli-binary-format <value>]
[--no-cli-pager]
[--cli-auto-prompt]
[--no-cli-auto-prompt]
aws kafka create-cluster \
--cluster-name "MessagingCluster" \
--broker-node-group-info file://brokernodegroupinfo.json \
--kafka-version "2.2.1" \
--number-of-broker-nodes 3
<Region>-Kafka.Storage.GP2
<Region>-Kafka.<Instance_Type>
DataTransfer-Regional-Bytes
<Region>-S3-Egress-Bytes
Categorized by Availability, Security & Compliance and Cost
