docker-compose.yml

## based on https://github.com/gschmutz/stream-processing-workshop/blob/master/01-environment/docker/docker-compose.yml
# does not include: Connect, Schema Registry, KSQL
# does include KafkaCat (https://docs.confluent.io/platform/current/app-development/kafkacat-usage.html and https://docs.confluent.io/platform/current/tutorials/examples/clients/docs/kafkacat.html#client-examples-kafkacat) 
# and Kafka HQ (https://akhq.io/) available at http://localhost:28042
# relies on environment variable PUBLIC_IP 
# Prepare Environment Variables
# set PUBLIC_IP to the IP address on which the local host machine connects to the Docker host
# - this can be 127.0.0.1 when you run Docker Compose locally or it can be the IP address of the VM inside which you have Docker Compose running or even the Public IP address of the Compute Instance in which you run Docker Compose


# =====================================================
# Platform: default
# Platform Stack:          derived from trivadis/platys-modern-data-platform
# Platform Stack Version:  derived from 1.10.0
# =====================================================
version: '3.5'
# enforce some dependencies
networks:
  default:
    name: streaming-platform
services:
  #  ================================== Zookeeper ========================================== #}
  #  latest:  image: confluentinc/cp-zookeeper:7.3.3
  zookeeper-1:
    image: confluentinc/cp-zookeeper:6.0.1 
    container_name: zookeeper-1
    hostname: zookeeper-1
    ports:
      - 2181:2181
    environment:
      ZOOKEEPER_SERVER_ID: 1
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
      ZOOKEEPER_INIT_LIMIT: 5
      ZOOKEEPER_SYNC_LIMIT: 2
    restart: unless-stopped
  #  ================================== Kafka ========================================== #}
  #  latest: confluentinc/cp-server:7.3.3
  kafka-1:
    image: confluentinc/cp-kafka:6.0.1
    container_name: kafka-1
    hostname: kafka-1
    depends_on:
      - zookeeper-1
    ports:
      - 9092:9092
      - 29092:29092
      - 9992:9992
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_BROKER_RACK: r1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181
      KAFKA_LISTENERS: LISTENER_INTERNAL://kafka-1:19092,LISTENER_DOCKERHOST://kafka-1:29092,LISTENER_EXTERNAL://kafka-1:9092
      KAFKA_ADVERTISED_LISTENERS: LISTENER_INTERNAL://kafka-1:19092,LISTENER_DOCKERHOST://localhost:29092,LISTENER_EXTERNAL://${PUBLIC_IP:-127.0.0.1}:9092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_INTERNAL:PLAINTEXT,LISTENER_DOCKERHOST:PLAINTEXT,LISTENER_EXTERNAL:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_INTERNAL
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
      KAFKA_MESSAGE_TIMESTAMP_TYPE: CreateTime
      KAFKA_MIN_INSYNC_REPLICAS: 1
      KAFKA_DELETE_TOPIC_ENABLE: 'True'
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'False'
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 100
      KAFKA_JMX_PORT: 9992
      KAFKA_JMX_OPTS: -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.rmi.port=9992
      KAFKA_JMX_HOSTNAME: ${PUBLIC_IP:-127.0.0.1}
    volumes:
      - ./data-transfer:/data-transfer
    restart: unless-stopped
  kafka-2:
    image: confluentinc/cp-kafka:6.0.1
    container_name: kafka-2
    hostname: kafka-2
    depends_on:
      - zookeeper-1
    ports:
      - 9093:9093
      - 29093:29093
      - 9993:9993
    environment:
      KAFKA_BROKER_ID: 2
      KAFKA_BROKER_RACK: r2
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181
      KAFKA_LISTENERS: LISTENER_INTERNAL://kafka-2:19093,LISTENER_DOCKERHOST://kafka-2:29093,LISTENER_EXTERNAL://kafka-2:9093
      KAFKA_ADVERTISED_LISTENERS: LISTENER_INTERNAL://kafka-2:19093,LISTENER_DOCKERHOST://localhost:29093,LISTENER_EXTERNAL://${PUBLIC_IP:-127.0.0.1}:9093
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_INTERNAL:PLAINTEXT,LISTENER_DOCKERHOST:PLAINTEXT,LISTENER_EXTERNAL:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_INTERNAL
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
      KAFKA_MESSAGE_TIMESTAMP_TYPE: CreateTime
      KAFKA_MIN_INSYNC_REPLICAS: 1
      KAFKA_DELETE_TOPIC_ENABLE: 'True'
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'False'
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 100
      KAFKA_JMX_PORT: 9993
      KAFKA_JMX_OPTS: -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.rmi.port=9993
      KAFKA_JMX_HOSTNAME: ${PUBLIC_IP:-127.0.0.1}
    volumes:
      - ./data-transfer:/data-transfer
    restart: unless-stopped
  kafka-3:
    image: confluentinc/cp-kafka:6.0.1
    container_name: kafka-3
    hostname: kafka-3
    depends_on:
      - zookeeper-1
    ports:
      - 9094:9094
      - 29094:29094
      - 9994:9994
    environment:
      KAFKA_BROKER_ID: 3
      KAFKA_BROKER_RACK: r3
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181
      KAFKA_LISTENERS: LISTENER_INTERNAL://kafka-3:19094,LISTENER_DOCKERHOST://kafka-3:29094,LISTENER_EXTERNAL://kafka-3:9094
      KAFKA_ADVERTISED_LISTENERS: LISTENER_INTERNAL://kafka-3:19094,LISTENER_DOCKERHOST://localhost:29094,LISTENER_EXTERNAL://${PUBLIC_IP:-127.0.0.1}:9094
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_INTERNAL:PLAINTEXT,LISTENER_DOCKERHOST:PLAINTEXT,LISTENER_EXTERNAL:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_INTERNAL
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
      KAFKA_MESSAGE_TIMESTAMP_TYPE: CreateTime
      KAFKA_MIN_INSYNC_REPLICAS: 1
      KAFKA_DELETE_TOPIC_ENABLE: 'True'
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'False'
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 100
      KAFKA_JMX_PORT: 9994
      KAFKA_JMX_OPTS: -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.rmi.port=9994
      KAFKA_JMX_HOSTNAME: ${PUBLIC_IP:-127.0.0.1}
    volumes:
      - ./data-transfer:/data-transfer
    restart: unless-stopped
 #  ================================== Kcat  ========================================== #
  kcat:
    image: edenhill/kcat:1.7.1
    # latest images is edenhill/kcat:1.7.1 - see https://github.com/edenhill/kcat
    container_name: kcat
    hostname: kcat
    volumes:
      - ./data-transfer:/data-transfer
    entrypoint:
      - /bin/sh
      - -c
      - |
        apk add jq;
        while [ 1 -eq 1 ];do sleep 60;done
    restart: unless-stopped
  #  ================================== Apache Kafka HQ (AKHQ) ========================================== #
  akhq:
    image: tchiotludo/akhq:latest
    container_name: akhq
    hostname: akhq
    labels:
      com.platys.webui.title: Apache Kafka AQ UI
      com.platys.webui.url: http://${PUBLIC_IP}:28042
    ports:
      - 28042:8080
    environment:
      AKHQ_CONFIGURATION: |
        akhq:
          connections:
            docker-kafka-server:
              properties:
                bootstrap.servers: 'kafka-1:19092,kafka-2:19093,kafka-3:19094'
              # schema-registry:
              #   url: "http://schema-registry-1:8081"
              # connect:
              #   - name: "kafka-connect-1"
              #     url: "http://kafka-connect-1:8083"
              #   - name: "kafka-connect-2"
              #     url: "http://kafka-connect-1:8084"
    depends_on:
      - kafka-1
    volumes:
      - ./data-transfer:/data-transfer
    restart: unless-stopped


  schema-registry:
    image: confluentinc/cp-schema-registry:7.3.3
    hostname: schema-registry
    container_name: schema-registry
    depends_on:
      - kafka-1
      - kafka-2
      - kafka-3
    ports:
      - "8081:8081"
    environment:
      SCHEMA_REGISTRY_HOST_NAME: schema-registry
      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'kafka-1:19092,kafka-2:19093,kafka-3:19094'
      SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081  

  rest-proxy:
    image: confluentinc/cp-kafka-rest:7.3.3
    depends_on:
      - kafka-1
      - kafka-2
      - kafka-3
      - schema-registry
    ports:
      - 8082:8082
    hostname: rest-proxy
    container_name: rest-proxy
    environment:
      KAFKA_REST_HOST_NAME: rest-proxy
      KAFKA_REST_BOOTSTRAP_SERVERS: 'kafka-1:19092,kafka-2:19093,kafka-3:19094'
      KAFKA_REST_LISTENERS: "http://0.0.0.0:8082"
      KAFKA_REST_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081'

  # adding Kafka Magic (https://www.kafkamagic.com/); note:L without License Key, functionality is limited to browsing topcis and partitions
  # To connect to a Kafka Cluster In the UI click ‘Register New’ button and define your Kafka cluster with these parameters:
  # for Cluster Name enter a value you like for your cluster - such as *My Local Kafka Cluster*  
  # for Bootstrap Servers enter: kafka-1:19092,kafka-2:19093,kafka-3:19094
  # for Schema Registry URL enter: http://schema-registry:8081
  # after entering these values, click on the *Verify* link to see if the connection can be created. When successful, click on *Register Connection* to create the connection

  # magic:
  #   image: digitsy/kafka-magic
  #   ports:
  #     - "8080:80"
  #   environment:
  #     KMAGIC_ALLOW_TOPIC_DELETE: "true"
  #     KMAGIC_ALLOW_SCHEMA_DELETE: "true"