Docker Compose Multi Node Broker with FilePulse, Debezium and Schema Registry
11-11-2022version: "3.2" services: zookeeper: image: bitnami/zookeeper:3 ports: - 2181:2181 environment: ALLOW_ANONYMOUS_LOGIN: "yes" kafka-0: image: bitnami/kafka:2 ports: - 9092:9092 environment: KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181 ALLOW_PLAINTEXT_LISTENER: "yes" KAFKA_LISTENERS: >- INTERNAL://:29092,EXTERNAL://:9092 KAFKA_ADVERTISED_LISTENERS: >- INTERNAL://kafka-0:29092,EXTERNAL://localhost:9092 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: >- INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: "INTERNAL" depends_on: - zookeeper kafka-1: image: bitnami/kafka:2 ports: - 9093:9093 environment: KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181 ALLOW_PLAINTEXT_LISTENER: "yes" KAFKA_LISTENERS: >- INTERNAL://:29092,EXTERNAL://:9093 KAFKA_ADVERTISED_LISTENERS: >- INTERNAL://kafka-1:29092,EXTERNAL://localhost:9093 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: >- INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: "INTERNAL" depends_on: - zookeeper kafka-2: image: bitnami/kafka:2 ports: - 9094:9094 environment: KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181 ALLOW_PLAINTEXT_LISTENER: "yes" KAFKA_LISTENERS: >- INTERNAL://:29092,EXTERNAL://:9094 KAFKA_ADVERTISED_LISTENERS: >- INTERNAL://kafka-2:29092,EXTERNAL://localhost:9094 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: >- INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: "INTERNAL" depends_on: - zookeeper schema-registry: image: docker.io/bitnami/schema-registry:7.2 hostname: schema-registry container_name: schema-registry depends_on: - kafka-0 - kafka-1 - kafka-2 ports: - "8081:8081" environment: - SCHEMA_REGISTRY_LISTENERS=http://0.0.0.0:8081 - SCHEMA_REGISTRY_KAFKA_BROKERS=PLAINTEXT://kafka-0:29092,PLAINTEXT://kafka-1:29092,PLAINTEXT://kafka-2:29092 - SCHEMA_REGISTRY_HOST_NAME=schema-registry connect-file-pulse: image: streamthoughts/kafka-connect-file-pulse:2.5.0 container_name: connect depends_on: - kafka-0 - kafka-1 - kafka-2 - schema-registry ports: - "8083:8083" - "8000:8000" environment: CONNECT_BOOTSTRAP_SERVERS: 'kafka-0:29092,kafka-1:29092,kafka-2:29092' CONNECT_REST_ADVERTISED_HOST_NAME: connect CONNECT_REST_PORT: 8083 CONNECT_GROUP_ID: compose-connect-group CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 3 CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000 CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 3 CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 3 CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081 CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" CONNECT_ZOOKEEPER_CONNECT: 'zookeeper:2181' # kafka-connect-file-pulse is installed as a confluent-hub component CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components/" CONNECT_LOG4J_ROOT_LOGLEVEL: "INFO" # CONNECT_LOG4J_LOGGERS: io.streamthoughts=DEBUG,org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR # Uncomment for debugging the connector during development cycle # KAFKA_OPTS: "-agentlib:jdwp=transport=dt_socket,server=y,address=8000,suspend=n" kafdrop: image: obsidiandynamics/kafdrop:latest ports: - 9000:9000 environment: KAFKA_BROKERCONNECT: >- kafka-0:29092,kafka-1:29092,kafka-2:29092 depends_on: - kafka-0 - kafka-1 - kafka-2
Second Example For Debezium Connector
version: "3.2" services: zookeeper: image: bitnami/zookeeper:3 ports: - 2181:2181 environment: ALLOW_ANONYMOUS_LOGIN: "yes" kafka-0: image: bitnami/kafka:2 ports: - 9092:9092 environment: KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181 ALLOW_PLAINTEXT_LISTENER: "yes" KAFKA_LISTENERS: >- INTERNAL://:29092,EXTERNAL://:9092 KAFKA_ADVERTISED_LISTENERS: >- INTERNAL://kafka-0:29092,EXTERNAL://localhost:9092 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: >- INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: "INTERNAL" depends_on: - zookeeper kafka-1: image: bitnami/kafka:2 ports: - 9093:9093 environment: KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181 ALLOW_PLAINTEXT_LISTENER: "yes" KAFKA_LISTENERS: >- INTERNAL://:29092,EXTERNAL://:9093 KAFKA_ADVERTISED_LISTENERS: >- INTERNAL://kafka-1:29092,EXTERNAL://localhost:9093 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: >- INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: "INTERNAL" depends_on: - zookeeper kafka-2: image: bitnami/kafka:2 ports: - 9094:9094 environment: KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181 ALLOW_PLAINTEXT_LISTENER: "yes" KAFKA_LISTENERS: >- INTERNAL://:29092,EXTERNAL://:9094 KAFKA_ADVERTISED_LISTENERS: >- INTERNAL://kafka-2:29092,EXTERNAL://localhost:9094 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: >- INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: "INTERNAL" depends_on: - zookeeper schema-registry: image: docker.io/bitnami/schema-registry:7.2 hostname: schema-registry container_name: schema-registry depends_on: - kafka-0 - kafka-1 - kafka-2 ports: - "8081:8081" environment: - SCHEMA_REGISTRY_LISTENERS=http://0.0.0.0:8081 - SCHEMA_REGISTRY_KAFKA_BROKERS=PLAINTEXT://kafka-0:29092,PLAINTEXT://kafka-1:29092,PLAINTEXT://kafka-2:29092 - SCHEMA_REGISTRY_HOST_NAME=schema-registry apicurio: image: apicurio/apicurio-registry-mem:2.2.5.Final hostname: apicurio container_name: apicurio depends_on: - kafka-0 - kafka-1 - kafka-2 ports: - "8080:8080" environment: - SCHEMA_REGISTRY_LISTENERS=http://0.0.0.0:8080 - SCHEMA_REGISTRY_KAFKA_BROKERS=PLAINTEXT://kafka-0:29092,PLAINTEXT://kafka-1:29092,PLAINTEXT://kafka-2:29092 - SCHEMA_REGISTRY_HOST_NAME=apicurio connect: image: debezium/connect:latest hostname: connect container_name: connect depends_on: - kafka-0 - kafka-1 - kafka-2 - schema-registry - apicurio ports: - "8083:8083" environment: BOOTSTRAP_SERVERS: "kafka-0:29092,kafka-1:29092,kafka-2:29092" ADVERTISED_PORT: 8083 ADVERTISED_HOST_NAME: "kafka-connect" GROUP_ID: compose-connect-group CONFIG_STORAGE_TOPIC: docker-connect-configs OFFSET_STORAGE_TOPIC: docker-connect-offsets STATUS_STORAGE_TOPIC: docker-connect-status ENABLE_APICURIO_CONVERTERS: "true" KEY_CONVERTER: io.apicurio.registry.utils.converter.AvroConverter VALUE_CONVERTER: io.apicurio.registry.utils.converter.AvroConverter INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" CONNECT_KEY_CONVERTER: "io.apicurio.registry.utils.converter.AvroConverter" CONNECT_KEY_CONVERTER_APICURIO.REGISTRY_URL: "http://apicurio:8080/apis/registry/v2" CONNECT_KEY_CONVERTER_APICURIO_REGISTRY_AUTO-REGISTER: "true" CONNECT_KEY_CONVERTER_APICURIO_REGISTRY_FIND-LATEST: "true" CONNECT_VALUE_CONVERTER: "io.apicurio.registry.utils.converter.AvroConverter" CONNECT_VALUE_CONVERTER_APICURIO_REGISTRY_URL: "http://apicurio:8080/apis/registry/v2" CONNECT_VALUE_CONVERTER_APICURIO_REGISTRY_AUTO-REGISTER: "true" CONNECT_VALUE_CONVERTER_APICURIO_REGISTRY_FIND-LATEST: "true" CONNECT_SCHEMA_NAME_ADJUSTMENT_MODE: "avro" LOG4J_ROOT_LOGLEVEL: "INFO" LOG4J_LOGGERS: "org.apache.kafka.connect.runtime.rest=WARN,org.reflections=ERROR" CONFIG_STORAGE_REPLICATION_FACTOR: "3" OFFSET_STORAGE_REPLICATION_FACTOR: "3" STATUS_STORAGE_REPLICATION_FACTOR: "3" OFFSET_FLUSH_INTERVAL_MS: 60000 PLUGIN_PATH: '/usr/share/java' volumes: - ./connect/postgres-source.json:/connect/postgres-source.json kafdrop: image: obsidiandynamics/kafdrop:latest ports: - 9000:9000 environment: KAFKA_BROKERCONNECT: >- kafka-0:29092,kafka-1:29092,kafka-2:29092 depends_on: - kafka-0 - kafka-1 - kafka-2