Post

카프카핵심가이드 2장 - 카프카 설치하기

카프카 핵심가이드 책을 읽은 내용 정리

Posted
Kafka. The Definitive Guide. Real-Time Data and Stream Processing at.
Kafka. The Definitive Guide. Real-Time Data and Stream Processing at.
By ydj515
views 5 min read

카프카 설치

2장은 카프카 설치를 다루고 있다. 이 내용을 docker-compose로 대체한다.
기존 kafka에서는 zookeper를 사용하였으나 최신버전 kfaka 부터는 zookeeper를 사용하지않고 kraft를 사용한다.

아파치 카프카 3.7 버전이 주키퍼 모드를 지원하는 마지막 버전이고, 이후 카프카 4.0 버전의 경우는 KRaft 모드로만 사용해야 합니다. (참고자료)

standalone kafka with zookeeper

아래 docker-comspoe 파일은 kafka 3.3.0 기준으로 작성되었다.

  • docker-compose.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82

version: '3.8'
services:
  zookeeper:
    image: confluentinc/cp-zookeeper:7.3.0
    hostname: zookeeper
    container_name: zookeeper
    ports:
      - "2181:2181"
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
    networks:
      - jpasample-network
  kafka:
    image: confluentinc/cp-kafka:7.3.0
    hostname: kafka
    container_name: kafka
    depends_on:
      - zookeeper
    ports:
      - "29092:29092"
      - "9092:9092"
      - "9101:9101"
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://127.0.0.1:9092
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_JMX_PORT: 9101
      KAFKA_JMX_HOSTNAME: localhost
    networks:
      - jpasample-network
#  kafka-connect:
#    image: confluentinc/cp-kafka-connect:7.0.1
#    ports:
#      - "8083:8083"
#    container_name: kafka-connect
#    environment:
#      CONNECT_BOOTSTRAP_SERVERS: kafka:29092
#      CONNECT_REST_PORT: 8083
#      CONNECT_GROUP_ID: "quickstart-avro"
#      CONNECT_CONFIG_STORAGE_TOPIC: "quickstart-avro-config"
#      CONNECT_OFFSET_STORAGE_TOPIC: "quickstart-avro-offsets"
#      CONNECT_STATUS_STORAGE_TOPIC: "quickstart-avro-status"
#      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
#      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
#      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
#      CONNECT_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
#      CONNECT_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
#      CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
#      CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
#      CONNECT_REST_ADVERTISED_HOST_NAME: "localhost"
#      CONNECT_LOG4J_ROOT_LOGLEVEL: WARN
#      CONNECT_PLUGIN_PATH: "/usr/share/java,/etc/kafka-connect/jars"
#    depends_on:
#      - kafka
#    volumes:
#      - ./docker-data-files/lib:/etc/kafka-connect/jars # https://www.confluent.io/hub/confluentinc/kafka-connect-jdbc에서 다운 받고 lib/하위의 jar 파일들을 lib폴더하위로 복사
#    networks:
#      - jpasample-network
  kafka-ui:
    image: provectuslabs/kafka-ui:latest
    container_name: kafka-ui
    ports:
      - "8080:8080"
    environment:
      KAFKA_CLUSTERS_0_NAME: "local-kafka"
      KAFKA_CLUSTERS_0_BOOTSTRAP_SERVERS: "kafka:29092"
      KAFKA_CLUSTERS_0_ZOOKEEPER_CONNECT: "zookeeper:2181"
    depends_on:
      - kafka
      - zookeeper
    networks:
      - jpasample-network

networks:
  jpasample-network:
    driver: bridge

kafka cluster with zookeeper

아래 docker-comspoe 파일은 kafka 3.3.0 기준으로 작성되었다.

  • docker-compose.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106

version: '3.8'
services:
  zookeeper:
    image: confluentinc/cp-zookeeper:7.3.0
    hostname: zookeeper
    container_name: zookeeper
    ports:
      - "2181:2181"
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
    networks:
      - jpasample-network

  kafka-1:
    image: confluentinc/cp-kafka:7.3.0
    hostname: kafka-1
    container_name: kafka-1
    depends_on:
      - zookeeper
    ports:
      - "29091:29091"
      - "9091:9091"
      - "9101:9101"
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-1:29091,PLAINTEXT_HOST://127.0.0.1:9091
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_JMX_PORT: 9101
      KAFKA_JMX_HOSTNAME: localhost
    networks:
      - jpasample-network

  kafka-2:
    image: confluentinc/cp-kafka:7.3.0
    hostname: kafka-2
    container_name: kafka-2
    depends_on:
      - zookeeper
    ports:
      - "29092:29092"
      - "9092:9092"
      - "9102:9102"
    environment:
      KAFKA_BROKER_ID: 2
      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-2:29092,PLAINTEXT_HOST://127.0.0.1:9092
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_JMX_PORT: 9102
      KAFKA_JMX_HOSTNAME: localhost
    networks:
      - jpasample-network

  kafka-3:
    image: confluentinc/cp-kafka:7.3.0
    hostname: kafka-3
    container_name: kafka-3
    depends_on:
      - zookeeper
    ports:
      - "29093:29093"
      - "9093:9093"
      - "9103:9103"
    environment:
      KAFKA_BROKER_ID: 3
      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-3:29093,PLAINTEXT_HOST://127.0.0.1:9093
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_JMX_PORT: 9103
      KAFKA_JMX_HOSTNAME: localhost
    networks:
      - jpasample-network

  kafka-ui:
    image: provectuslabs/kafka-ui:latest
    container_name: kafka-ui
    ports:
      - "8080:8080"
    environment:
      KAFKA_CLUSTERS_0_NAME: "local-kafka"
      KAFKA_CLUSTERS_0_BOOTSTRAP_SERVERS: "kafka-1:29091,kafka-2:29092,kafka-3:29093"
      KAFKA_CLUSTERS_0_ZOOKEEPER_CONNECT: "zookeeper:2181"
    depends_on:
      - kafka-1
      - kafka-2
      - kafka-3
      - zookeeper
    networks:
      - jpasample-network

networks:
  jpasample-network:
    driver: bridge

kafka cluster with KRaft

  • docker-compose.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82

version: '3.8'

services:
  kafka-1:
    container_name: kafka-1
    image: confluentinc/cp-kafka:7.5.3
    ports:
      - "${KAFKA_1_PORT}:9092"
#    volumes:
#      - ./data/${KAFKA_DIR}/kafka-1:/var/lib/kafka/data
    environment:
      KAFKA_NODE_ID: 1
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://:29092,EXTERNAL://${DOCKER_HOST_IP}:${KAFKA_1_PORT}
      KAFKA_LISTENERS: INTERNAL://:29092,CONTROLLER://:29093,EXTERNAL://0.0.0.0:${KAFKA_1_PORT}
      KAFKA_PROCESS_ROLES: 'broker,controller'
      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka-1:29093,2@kafka-2:29093,3@kafka-3:29093
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
      CLUSTER_ID: ${KAFKA_CLUSTER_ID}
    networks:
      - jpasample-network

  kafka-2:
    container_name: kafka-2
    image: confluentinc/cp-kafka:7.5.3
    ports:
      - "${KAFKA_2_PORT}:9093"
#    volumes:
#      - ./data/${KAFKA_DIR}/kafka-2:/var/lib/kafka/data
    environment:
      KAFKA_NODE_ID: 2
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://:29092,EXTERNAL://${DOCKER_HOST_IP}:${KAFKA_2_PORT}
      KAFKA_LISTENERS: INTERNAL://:29092,CONTROLLER://kafka-2:29093,EXTERNAL://0.0.0.0:${KAFKA_2_PORT}
      KAFKA_PROCESS_ROLES: 'broker,controller'
      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka-1:29093,2@kafka-2:29093,3@kafka-3:29093
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
      CLUSTER_ID: ${KAFKA_CLUSTER_ID}
    networks:
      - jpasample-network

  kafka-3:
    container_name: kafka-3
    image: confluentinc/cp-kafka:7.5.3
    ports:
      - "${KAFKA_3_PORT}:9094"
#    volumes:
#      - ./data/${KAFKA_DIR}/kafka-3:/var/lib/kafka/data
    environment:
      KAFKA_NODE_ID: 3
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://:29092,EXTERNAL://${DOCKER_HOST_IP}:${KAFKA_3_PORT}
      KAFKA_LISTENERS: INTERNAL://:29092,CONTROLLER://kafka-3:29093,EXTERNAL://0.0.0.0:${KAFKA_3_PORT}
      KAFKA_PROCESS_ROLES: 'broker,controller'
      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka-1:29093,2@kafka-2:29093,3@kafka-3:29093
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
      CLUSTER_ID: ${KAFKA_CLUSTER_ID}
    networks:
      - jpasample-network

  kafka-ui:
    image: provectuslabs/kafka-ui
    container_name: kafka-ui
    ports:
      - "${KAFKA_UI_PORT}:8080"
    restart: always
    environment:
      KAFKA_CLUSTERS_0_NAME: ${PROFILE:-local}
      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka-1:29092,kafka-2:29092,kafka-3:29092
    depends_on:
      - kafka-1
      - kafka-2
      - kafka-3
    networks:
      - jpasample-network

networks:
  jpasample-network:
    driver: bridge
books, kafka
kafka
This post is licensed under CC BY 4.0 by the author.