Amazon MSKのマルチAZ設計入門｜aws kafkaコマンドで実装するブローカー分散配置とVPCサブネット設計

この記事のポイント

・MSKブローカーの分散配置はサブネット指定で決まる——AZが異なる3つのサブネットを必ず渡す
・aws kafka create-cluster-v2 の --broker-node-group-info でサブネットIDをAZバラバラに3つ指定する
・レプリケーション係数3＋min.insync.replicas=2でAZ1つ消失してもデータ損失ゼロを実現できる
・ブローカーエンドポイントはTLSポート9094が推奨——sg設定とDNS疎通の確認で接続問題の大半は解消する

# 3つのAZのプライベートサブネットを指定——これがマルチAZ配置の核心
subnet-0aaa...  # ap-northeast-1a
subnet-0bbb...  # ap-northeast-1c
subnet-0ccc...  # ap-northeast-1d

# VPC作成（例：10.0.0.0/16）
$ aws ec2 create-vpc --cidr-block 10.0.0.0/16     --tag-specifications 'ResourceType=vpc,Tags=[{Key=Name,Value=msk-vpc}]'

# MSK用プライベートサブネット——AZごとに1つずつ
$ aws ec2 create-subnet --vpc-id vpc-0123456789abcdef0     --cidr-block 10.0.10.0/24 --availability-zone ap-northeast-1a     --tag-specifications 'ResourceType=subnet,Tags=[{Key=Name,Value=msk-private-1a}]'

$ aws ec2 create-subnet --vpc-id vpc-0123456789abcdef0     --cidr-block 10.0.11.0/24 --availability-zone ap-northeast-1c     --tag-specifications 'ResourceType=subnet,Tags=[{Key=Name,Value=msk-private-1c}]'

$ aws ec2 create-subnet --vpc-id vpc-0123456789abcdef0     --cidr-block 10.0.12.0/24 --availability-zone ap-northeast-1d     --tag-specifications 'ResourceType=subnet,Tags=[{Key=Name,Value=msk-private-1d}]'

# MSK用セキュリティグループ作成
$ aws ec2 create-security-group     --group-name msk-broker-sg     --description "MSK Broker Security Group"     --vpc-id vpc-0123456789abcdef0

# プレーンテキスト（非推奨。開発環境のみ）
$ aws ec2 authorize-security-group-ingress     --group-id sg-0msk123     --protocol tcp --port 9092 --source-group sg-0client456

# TLS接続（本番推奨）
$ aws ec2 authorize-security-group-ingress     --group-id sg-0msk123     --protocol tcp --port 9094 --source-group sg-0client456

# IAM認証＋TLS
$ aws ec2 authorize-security-group-ingress     --group-id sg-0msk123     --protocol tcp --port 9098 --source-group sg-0client456

# KRaftモード（ZooKeeperなし）の場合は2181/2182は不要

# broker-config.json
{
  "BrokerAZDistribution": "DEFAULT",
  "ClientSubnets": [
    "subnet-0aaa111222333444a",  # ap-northeast-1a
    "subnet-0bbb111222333444b",  # ap-northeast-1c
    "subnet-0ccc111222333444c"   # ap-northeast-1d
  ],
  "InstanceType": "kafka.m5.large",
  "SecurityGroups": ["sg-0msk123abc456def"],
  "StorageInfo": {
    "EbsStorageInfo": {
      "VolumeSize": 100
    }
  }
}

$ aws kafka create-cluster-v2     --cluster-name msk-prod     --provisioned     --kafka-version "3.7.x"     --number-of-broker-nodes 3     --broker-node-group-info file://broker-config.json     --client-authentication '{"Sasl":{"Iam":{"Enabled":true}},"Unauthenticated":{"Enabled":false}}'     --encryption-info '{"EncryptionInTransit":{"ClientBroker":"TLS","InCluster":true}}'     --region ap-northeast-1

{
    "ClusterArn": "arn:aws:kafka:ap-northeast-1:123456789012:cluster/msk-prod/abc123...",
    "ClusterName": "msk-prod",
    "State": "CREATING"
}

$ aws kafka describe-cluster-v2     --cluster-arn "arn:aws:kafka:ap-northeast-1:123456789012:cluster/msk-prod/abc123..."     --region ap-northeast-1     --query "ClusterInfo.State"

"ACTIVE"

# ブローカーのエンドポイントを取得する
$ aws kafka get-bootstrap-brokers     --cluster-arn "arn:aws:kafka:ap-northeast-1:123456789012:cluster/msk-prod/abc123..."     --region ap-northeast-1

{
    "BootstrapBrokerStringTls": "b-1.msk-prod.abc123.c2.kafka.ap-northeast-1.amazonaws.com:9094,b-2.msk-prod.abc123.c2.kafka.ap-northeast-1.amazonaws.com:9094,b-3.msk-prod.abc123.c2.kafka.ap-northeast-1.amazonaws.com:9094",
    "BootstrapBrokerStringSaslIam": "b-1.msk-prod.abc123.c2.kafka.ap-northeast-1.amazonaws.com:9098,b-2.msk-prod.abc123.c2.kafka.ap-northeast-1.amazonaws.com:9098,b-3.msk-prod.abc123.c2.kafka.ap-northeast-1.amazonaws.com:9098"
}

# kafka-topicsコマンドでトピックを作成（EC2またはECSクライアントから実行）
$ kafka-topics.sh     --bootstrap-server b-1.msk-prod.abc123.c2.kafka.ap-northeast-1.amazonaws.com:9098     --command-config client.properties     --create     --topic app-events     --partitions 12     --replication-factor 3

Created topic app-events.

# レプリカの配置を確認する
$ kafka-topics.sh     --bootstrap-server b-1.msk-prod.abc123.c2.kafka.ap-northeast-1.amazonaws.com:9098     --command-config client.properties     --describe     --topic app-events

Topic: app-events  PartitionCount: 12  ReplicationFactor: 3
Topic: app-events  Partition: 0  Leader: 1  Replicas: 1,2,3  Isr: 1,2,3
Topic: app-events  Partition: 1  Leader: 2  Replicas: 2,3,1  Isr: 2,3,1
Topic: app-events  Partition: 2  Leader: 3  Replicas: 3,1,2  Isr: 3,1,2

# クライアントEC2またはECSタスク内から実行
$ nslookup b-1.msk-prod.abc123.c2.kafka.ap-northeast-1.amazonaws.com

Server:  169.254.169.253
Address: 169.254.169.253#53

Non-authoritative answer:
Name:   b-1.msk-prod.abc123.c2.kafka.ap-northeast-1.amazonaws.com
Address: 10.0.10.45

# client.properties（IAM認証設定）
security.protocol=SASL_SSL
sasl.mechanism=AWS_MSK_IAM
sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;
sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler

# テストメッセージの送信
$ kafka-console-producer.sh     --bootstrap-server b-1.msk-prod.abc123.c2.kafka.ap-northeast-1.amazonaws.com:9098     --topic app-events     --producer.config client.properties
>test-message-1
>test-message-2

# under-replicated partitions を確認する
$ kafka-topics.sh     --bootstrap-server b-1.msk-prod.abc123.c2.kafka.ap-northeast-1.amazonaws.com:9098     --command-config client.properties     --describe     --under-replicated-partitions

Topic: app-events  Partition: 3  Leader: 2  Replicas: 2,3,1  Isr: 2,1

# Isr に 3 が含まれていない = ブローカー3番がISRから外れている