analytics-pipeline/docker-compose.yml at main · raptor7197/analytics-pipeline · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
services:
  zookeeper:
    image: confluentinc/cp-zookeeper:7.6.0
    container_name: zookeeper
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
    ports:
      - "2181:2181"
    volumes:
      - zookeeper_data:/var/lib/zookeeper/data
      - zookeeper_logs:/var/lib/zookeeper/log
    healthcheck:
      test: ["CMD", "echo", "ruok", "|", "nc", "localhost", "2181"]
      interval: 10s
      timeout: 5s
      retries: 5
    networks:
      - event-network

  # broker
  kafka:
    image: confluentinc/cp-kafka:7.6.0
    container_name: kafka
    depends_on:
      zookeeper:
        condition: service_healthy
    ports:
      - "9092:9092"
      - "29092:29092"
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      # Two listeners: one for internal docker traffic, one for host access
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
      KAFKA_NUM_PARTITIONS: 3
      KAFKA_LOG_RETENTION_HOURS: 168 # 7 days
    volumes:
      - kafka_data:/var/lib/kafka/data
    healthcheck:
      test:
        [
          "CMD",
          "kafka-broker-api-versions",
          "--bootstrap-server",
          "localhost:9092",
        ]
      interval: 10s
      timeout: 10s
      retries: 10
      start_period: 30s
    networks:
      - event-network

  # clickHouse db
  clickhouse:
    image: clickhouse/clickhouse-server:24.3-alpine
    container_name: clickhouse
    ports:
      - "8123:8123" # HTTP interface
      - "9000:9000" # Native TCP interface
    environment:
      CLICKHOUSE_DB: event_analytics
      CLICKHOUSE_USER: default
      CLICKHOUSE_PASSWORD: ""
      CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
    volumes:
      - clickhouse_data:/var/lib/clickhouse
      - clickhouse_logs:/var/log/clickhouse-server
    ulimits:
      nofile:
        soft: 262144
        hard: 262144
    healthcheck:
      test: ["CMD", "clickhouse-client", "--query", "SELECT 1"]
      interval: 5s
      timeout: 5s
      retries: 10
      start_period: 10s
    networks:
      - event-network

  # API Server
  api:
    build:
      context: .
      dockerfile: Dockerfile.api
    container_name: event-api
    ports:
      - "8080:8080"
    environment:
      API_PORT: "8080"
      CLICKHOUSE_HOST: clickhouse
      CLICKHOUSE_PORT: "9000"
      CLICKHOUSE_DATABASE: event_analytics
      CLICKHOUSE_USER: default
      CLICKHOUSE_PASSWORD: ""
      KAFKA_BROKERS: kafka:29092
      KAFKA_TOPIC: events
    depends_on:
      kafka:
        condition: service_healthy
      clickhouse:
        condition: service_healthy
    restart: on-failure
    networks:
      - event-network

  # consumer
  consumer:
    build:
      context: .
      dockerfile: Dockerfile.consumer
    container_name: event-consumer
    environment:
      CLICKHOUSE_HOST: clickhouse
      CLICKHOUSE_PORT: "9000"
      CLICKHOUSE_DATABASE: event_analytics
      CLICKHOUSE_USER: default
      CLICKHOUSE_PASSWORD: ""
      KAFKA_BROKERS: kafka:29092
      KAFKA_TOPIC: events
      KAFKA_GROUP_ID: event-consumer-group
      CONSUMER_BATCH_SIZE: "100"
      CONSUMER_FLUSH_INTERVAL_MS: "200"
    depends_on:
      kafka:
        condition: service_healthy
      clickhouse:
        condition: service_healthy
    restart: on-failure
    networks:
      - event-network

volumes:
  zookeeper_data:
  zookeeper_logs:
  kafka_data:
  clickhouse_data:
  clickhouse_logs:

networks:
  event-network:
    driver: bridge