docker-compose.yml

services:
  # Invoice API Service (Existing)
  invoice-api:
    build: ./api
    hostname: invoice-api
    container_name: invoice-api
    environment:
      - API_HOST=0.0.0.0
      - API_PORT=8000
      - KAFKA_BOOTSTRAP_SERVERS=broker:29092
    ports:
      - "8000:8000"
    deploy:
      resources:
        limits:
          memory: 2G
          cpus: '1.0'
        reservations:
          memory: 1G
          cpus: '0.5'
    depends_on:
      broker:
        condition: service_healthy
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 10s
      timeout: 5s
      retries: 5
      start_period: 30s
    networks:
      - invoice-network
    restart: unless-stopped


  # Invoice Generator Service (Existing)
  invoice-generator:
    build: ./generator
    hostname: invoice-generator
    container_name: invoice-generator
    depends_on:
      - invoice-api
    environment:
      - API_ENDPOINT=http://invoice-api:8000/invoices
      - GENERATION_RATE=50
      - GENERATION_DURATION=0  # 0 = unlimited
    deploy:
      resources:
        limits:
          memory: 1G
          cpus: '0.5'
    networks:
      - invoice-network
    restart: unless-stopped
    command: ["python", "invoice_generator.py", "--rate=50", "--api-endpoint=http://invoice-api:8000/invoices"]

  # Zookeeper for Kafka
  zookeeper:
    image: confluentinc/cp-zookeeper:7.4.0
    hostname: zookeeper
    container_name: invoice-zookeeper
    ports:
      - "2181:2181"
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
    volumes:
      - zookeeper-data:/var/lib/zookeeper/data
      - zookeeper-logs:/var/lib/zookeeper/log
    healthcheck:
      test: ["CMD", "bash", "-c", "echo 'ruok' | nc localhost 2181"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 30s
    restart: unless-stopped
    deploy:
      resources:
        limits:
          memory: 1G
          cpus: '0.5'
    networks:
      - invoice-network

  # Kafka Broker
  broker:
    image: confluentinc/cp-server:7.4.0
    hostname: broker
    container_name: invoice-broker
    depends_on:
      zookeeper:
        condition: service_healthy
    ports:
      - "9092:9092"
      - "9101:9101"
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
      KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_CONFLUENT_BALANCER_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_JMX_PORT: 9101
      KAFKA_JMX_HOSTNAME: localhost
      KAFKA_CONFLUENT_SCHEMA_REGISTRY_URL: http://schema-registry:8081
      CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: broker:29092
      CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
      CONFLUENT_METRICS_ENABLE: 'true'
      CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous'
      # Performance tuning for invoice streaming
      KAFKA_NUM_PARTITIONS: 6
      KAFKA_DEFAULT_REPLICATION_FACTOR: 1
      KAFKA_LOG_RETENTION_HOURS: 168  # 7 days
      KAFKA_LOG_SEGMENT_BYTES: 1073741824  # 1GB
    volumes:
      - kafka-data:/var/lib/kafka/data
    healthcheck:
      test: ["CMD", "kafka-topics", "--bootstrap-server", "localhost:9092", "--list"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 60s
    restart: unless-stopped
    deploy:
      resources:
        limits:
          memory: 4G
          cpus: '1.0'
    networks:
      - invoice-network

  # Initialize Kafka topics for Invoice Streaming
  kafka-setup:
    image: confluentinc/cp-kafka:7.4.0
    container_name: invoice-kafka-setup
    depends_on:
      broker:
        condition: service_healthy
    command: |
      bash -c "
      echo 'Creating simple invoice streaming topic...';
      kafka-topics --create --if-not-exists --bootstrap-server broker:29092 --topic invoice-stream --partitions 6 --replication-factor 1;
      echo 'Single topic created: invoice-stream';
      kafka-topics --list --bootstrap-server broker:29092;
      "
    networks:
      - invoice-network

  # Schema Registry for data governance
  schema-registry:
    image: confluentinc/cp-schema-registry:7.4.0
    hostname: schema-registry
    container_name: invoice-schema-registry
    depends_on:
      broker:
        condition: service_healthy
    ports:
      - "8081:8081"
    environment:
      SCHEMA_REGISTRY_HOST_NAME: schema-registry
      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'broker:29092'
      SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8081/subjects"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 40s
    restart: unless-stopped
    deploy:
      resources:
        limits:
          memory: 1G
          cpus: '0.5'
    networks:
      - invoice-network

  # Kafka UI for monitoring
  kafka-ui:
    image: provectuslabs/kafka-ui:v0.7.2
    container_name: invoice-kafka-ui
    depends_on:
      broker:
        condition: service_healthy
    ports:
      - "8080:8080"
    environment:
      KAFKA_CLUSTERS_0_NAME: invoice-cluster
      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: broker:29092
      KAFKA_CLUSTERS_0_JMXPORT: 9101
      KAFKA_CLUSTERS_0_SCHEMAREGISTRY: http://schema-registry:8081
      DYNAMIC_CONFIG_ENABLED: true
    restart: unless-stopped
    deploy:
      resources:
        limits:
          memory: 1G
          cpus: '0.5'
    networks:
      - invoice-network
  
  kafka-bridge:
    build: ./kafka_bridge
    hostname: kafka-bridge
    container_name: kafka-bridge
    depends_on:
      invoice-api:
        condition: service_healthy
      broker:
        condition: service_healthy
    environment:
      - API_URL=http://invoice-api:8000/invoices
      - KAFKA_SERVERS=broker:29092
      - KAFKA_TOPIC=invoice-stream
      - POLL_INTERVAL=5
    deploy:
      resources:
        limits:
          memory: 512M
          cpus: '0.5'
    networks:
      - invoice-network
    restart: unless-stopped

  # MongoDB Database
  mongodb:
    image: mongo:7.0
    hostname: mongodb
    container_name: mongodb
    environment:
      MONGO_INITDB_ROOT_USERNAME: admin
      MONGO_INITDB_ROOT_PASSWORD: password123
      MONGO_INITDB_DATABASE: invoices
    ports:
      - "27017:27017"
    volumes:
      - mongodb_data:/data/db
      - ./config/init-mongo.js:/docker-entrypoint-initdb.d/init-mongo.js:ro
    deploy:
      resources:
        limits:
          memory: 4G
          cpus: '1.0'
    networks:
      - invoice-network
    restart: unless-stopped

  # MongoDB Web UI (Simple and Reliable)
  mongo-express:
    image: mongo-express:1.0.2
    hostname: mongo-express
    container_name: mongo-express
    depends_on:
      - mongodb
    environment:
      ME_CONFIG_MONGODB_ADMINUSERNAME: admin
      ME_CONFIG_MONGODB_ADMINPASSWORD: password123
      ME_CONFIG_MONGODB_URL: mongodb://admin:password123@mongodb:27017/
      ME_CONFIG_BASICAUTH_USERNAME: mongouser
      ME_CONFIG_BASICAUTH_PASSWORD: mongopass
      ME_CONFIG_MONGODB_SERVER: mongodb
    ports:
      - "8084:8081"
    networks:
      - invoice-network
    restart: unless-stopped

  # Spark Master - for Invoice Processing
  spark-master:
    build:
      context: .
      dockerfile: Dockerfile.spark
    hostname: spark-master
    container_name: invoice-spark-master
    ports:
      - "8090:8080"    # Spark Master Web UI
      - "7077:7077"    # Spark Master port
      - "6066:6066"    # Spark Master REST port
    environment:
      - SPARK_MODE=master
      - SPARK_RPC_AUTHENTICATION_ENABLED=no
      - SPARK_RPC_ENCRYPTION_ENABLED=no
      - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
      - SPARK_SSL_ENABLED=no
      - SPARK_USER=spark
      - SPARK_MASTER_OPTS=-Dspark.deploy.defaultCores=2
    volumes:
      - ./spark-apps:/opt/spark-apps
      - ./spark-data:/opt/spark-data
      - ./spark-logs:/opt/spark/logs
      - ./jars:/opt/spark/jars/extra
    healthcheck:
      test: ["CMD-SHELL", "pgrep -f 'org.apache.spark.deploy.master.Master' || exit 1"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 30s
    networks:
      - invoice-network
    restart: unless-stopped

  # Spark Worker 1
  spark-worker-1:
    build:
      context: .
      dockerfile: Dockerfile.spark
    hostname: spark-worker-1
    container_name: invoice-spark-worker-1
    ports:
      - "8091:8081"    # Worker Web UI
    environment:
      - SPARK_MODE=worker
      - SPARK_MASTER_URL=spark://spark-master:7077
      - SPARK_WORKER_MEMORY=4G
      - SPARK_WORKER_CORES=2
      - SPARK_RPC_AUTHENTICATION_ENABLED=no
      - SPARK_RPC_ENCRYPTION_ENABLED=no
      - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
      - SPARK_SSL_ENABLED=no
      - SPARK_USER=spark
      - SPARK_WORKER_OPTS=-Dspark.worker.cleanup.enabled=true
    volumes:
      - ./spark-apps:/opt/spark-apps
      - ./spark-data:/opt/spark-data
      - ./spark-logs:/opt/spark/logs
      - ./jars:/opt/spark/jars/extra
    depends_on:
      spark-master:
        condition: service_healthy
    networks:
      - invoice-network
    restart: unless-stopped

  # Spark Streaming Application - Runs Continuously
  spark-streaming:
    build:
      context: .
      dockerfile: Dockerfile.spark
    hostname: spark-streaming
    container_name: invoice-spark-streaming
    depends_on:
      spark-master:
        condition: service_healthy
      mongodb:
        condition: service_started
      broker:
        condition: service_healthy
    environment:
      - SPARK_MASTER_URL=spark://spark-master:7077
      - SPARK_APPLICATION_PYTHON_LOCATION=/opt/spark-apps/invoice_processor.py
    volumes:
      - ./spark-apps:/opt/spark-apps
      - ./spark-data:/opt/spark-data
      - ./spark-logs:/opt/spark/logs
      - ./jars:/opt/spark/jars/extra
    command: >
      spark-submit
      --master spark://spark-master:7077
      --packages org.mongodb.spark:mongo-spark-connector_2.12:10.3.0,org.apache.spark:spark-sql-kafka-0-10_2.12:3.5.0
      --driver-memory 1g
      --executor-memory 1g
      /opt/spark-apps/invoice_processor.py
    deploy:
      resources:
        limits:
          memory: 2G
          cpus: '1.0'
    networks:
      - invoice-network
    restart: unless-stopped

  powerbi-api:
    build: 
      context: ./powerbi-api
      dockerfile: Dockerfile
    ports:
      - "5000:5000"
    depends_on:
      - mongodb
    networks:
      - invoice-network
    environment:
      - MONGO_HOST=mongodb
    
volumes:
  zookeeper-data:
  zookeeper-logs:
  kafka-data:
  mongodb_data: 

networks:
  invoice-network:
    driver: bridge