compose-flex.yml

x-common-flink-config: &flink_image_pull_policy_config
  image: fh-flink-1.20.1${FLINK_SUFFIX:-} # ${FLINK_SUFFIX} is either unset (blank) or -py
  build:
    context: ./resources/flink/
    dockerfile: Dockerfile${FLINK_SUFFIX:-}
  pull_policy: never

x-common-environment: &flink_common_env_vars
  AWS_REGION: us-east-1
  HADOOP_CONF_DIR: /opt/flink/conf
  HIVE_CONF_DIR: /opt/flink/conf
  CUSTOM_JARS_DIRS: "/tmp/hadoop;/tmp/hive;/tmp/iceberg;/tmp/parquet"

x-common-flink-volumes: &flink_common_volumes
  - ./resources/flink/bin/config.sh:/opt/flink/bin/config.sh:ro
  - ./resources/flink/bin/flink-console.sh:/opt/flink/bin/flink-console.sh:ro
  - ./resources/flink/bin/flink-daemon.sh:/opt/flink/bin/flink-daemon.sh:ro
  - ./resources/flink/bin/sql-client.sh:/opt/flink/bin/sql-client.sh:ro
  - ./resources/flink/flink-conf.yaml:/opt/flink/conf/flink-conf.yaml:ro
  - ./resources/flink/hive-site.xml:/opt/flink/conf/hive-site.xml
  - ./resources/flink/core-site.xml:/opt/flink/conf/core-site.xml
  - ./resources/flink/init-catalogs.sql:/opt/flink/conf/init-catalogs.sql
  - ./resources/deps/hadoop:/tmp/hadoop
  - ./resources/deps/flink/hive/flink-sql-connector-hive-3.1.3_2.12-1.20.1.jar:/tmp/hive/flink-sql-connector-hive-3.1.3_2.12-1.20.1.jar
  - ./resources/deps/flink/hive/antlr-runtime-3.5.2.jar:/tmp/hive/antlr-runtime-3.5.2.jar
  - ./resources/deps/flink/iceberg:/tmp/iceberg
  - ./resources/deps/flink/parquet:/tmp/parquet
  - ./resources/deps/flink/connector:/tmp/connector
  - ./resources/deps/postgres:/tmp/postgres

services:
  flex:
    image: factorhouse/flex${FLEX_SUFFIX:-}:latest
    container_name: flex${FLEX_SUFFIX:-}
    pull_policy: always
    restart: always
    ports:
      - "3001:3000"
    networks:
      - factorhouse
    depends_on:
      jobmanager:
        condition: service_healthy
    env_file:
      - resources/flex/config/setup.env
      - ${FLEX_LICENSE:-resources/flex/config/license.env}
    mem_limit: 2G
    volumes:
      - ./resources/flex/jaas:/etc/flex/jaas
      - ./resources/flex/rbac:/etc/flex/rbac

  jobmanager:
    <<: *flink_image_pull_policy_config
    container_name: jobmanager
    command: jobmanager
    ports:
      - "8082:8081"
    networks:
      - factorhouse
    depends_on:
      hive-metastore:
        condition: service_healthy
    environment:
      <<: *flink_common_env_vars
    volumes: *flink_common_volumes
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8081/config"]
      interval: 5s
      timeout: 5s
      retries: 5

  taskmanager-1:
    <<: *flink_image_pull_policy_config
    container_name: taskmanager-1
    command: taskmanager
    networks:
      - factorhouse
    depends_on:
      jobmanager:
        condition: service_healthy
    environment:
      <<: *flink_common_env_vars
    volumes: *flink_common_volumes

  taskmanager-2:
    <<: *flink_image_pull_policy_config
    container_name: taskmanager-2
    command: taskmanager
    networks:
      - factorhouse
    depends_on:
      jobmanager:
        condition: service_healthy
    environment:
      <<: *flink_common_env_vars
    volumes: *flink_common_volumes

  taskmanager-3:
    <<: *flink_image_pull_policy_config
    container_name: taskmanager-3
    command: taskmanager
    networks:
      - factorhouse
    depends_on:
      jobmanager:
        condition: service_healthy
    environment:
      <<: *flink_common_env_vars
    volumes: *flink_common_volumes

  sql-gateway:
    <<: *flink_image_pull_policy_config
    container_name: sql-gateway
    command: >
      bin/sql-gateway.sh
      start-foreground
      -Dsql-gateway.endpoint.rest.address=0.0.0.0
      -Dsql-gateway.endpoint.rest.port=9090
      -Djobmanager.rpc.address=jobmanager
      -Drest.address=jobmanager
    ports:
      - "9090:9090"
    networks:
      - factorhouse
    depends_on:
      jobmanager:
        condition: service_healthy
    environment:
      FLINK_SQL_GATEWAY_DEFAULT_CATALOG_NAME: default_catalog
      FLINK_SQL_GATEWAY_DEFAULT_DATABASE_NAME: default_database
      <<: *flink_common_env_vars
    volumes: *flink_common_volumes

  minio:
    image: minio/minio
    container_name: minio
    environment:
      - MINIO_ROOT_USER=admin
      - MINIO_ROOT_PASSWORD=password
      - MINIO_DOMAIN=minio
    ports:
      - 9001:9001
      - 9000:9000
    networks:
      - factorhouse
    command: ["server", "/data", "--console-address", ":9001"]

  mc:
    image: minio/mc
    container_name: mc
    networks:
      - factorhouse
    environment:
      - AWS_ACCESS_KEY_ID=admin
      - AWS_SECRET_ACCESS_KEY=password
      - AWS_REGION=us-east-1
    entrypoint: |
      /bin/sh -c "
      until (/usr/bin/mc alias set minio http://minio:9000 admin password) do
        echo '...waiting for minio...' && sleep 1;
      done;
      BUCKETS_TO_CREATE='warehouse fh-dev-bucket flink-checkpoints flink-savepoints'
      for bucket in $$BUCKETS_TO_CREATE; do
        echo \"--- Configuring bucket: $$bucket ---\"
        /usr/bin/mc rm -r --force minio/$$bucket;
        /usr/bin/mc mb minio/$$bucket;
        /usr/bin/mc policy set public minio/$$bucket;
      done;
      echo '--- Minio setup complete, container will remain active ---'
      tail -f /dev/null
      "
    depends_on:
      - minio

  postgres:
    image: postgres:17
    container_name: postgres
    command:
      - "postgres"
      - "-c"
      - "wal_level=logical"
      - "-c"
      - "shared_preload_libraries=pg_stat_statements" # For OMT PostgreSQL connector
      - "-c"
      - "pg_stat_statements.track=all" # For OMT stored procedure lineage
      - "-c"
      - "log_statement=all" # For OMT detailed logging
    ports:
      - 5432:5432
    networks:
      - factorhouse
    volumes:
      - ./resources/postgres:/docker-entrypoint-initdb.d
    environment:
      POSTGRES_DB: fh_dev
      POSTGRES_USER: db_user
      POSTGRES_PASSWORD: db_password
      TZ: UTC # change if necessary eg) Australia/Melbourne
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U db_user -d metastore"]
      interval: 2s
      timeout: 2s
      retries: 10
      start_period: 3s

  hive-metastore:
    image: apache/hive:3.1.3
    container_name: hive-metastore
    ports:
      - "9083:9083"
    networks:
      - factorhouse
    depends_on:
      postgres:
        condition: service_healthy
    environment:
      SERVICE_NAME: metastore
      DB_DRIVER: postgres
      METASTORE_PORT: 9083
    volumes:
      - ./resources/hms/hive-site.xml:/opt/hive/conf/hive-site.xml
      - ./resources/hms/core-site.xml:/opt/hive/conf/core-site.xml
      - ./resources/deps/postgres/postgresql-42.7.3.jar:/opt/hive/lib/postgresql-42.7.3.jar
      - ./resources/deps/hadoop/hadoop-common-3.3.6.jar:/opt/hive/lib/hadoop-common-3.3.6.jar
      - ./resources/deps/hadoop/hadoop-auth-3.3.6.jar:/opt/hive/lib/hadoop-auth-3.3.6.jar
      - ./resources/deps/hadoop/hadoop-aws-3.3.6.jar:/opt/hive/lib/hadoop-aws-3.3.6.jar
      - ./resources/deps/hadoop/aws-java-sdk-bundle-1.11.1026.jar:/opt/hive/lib/aws-java-sdk-bundle-1.11.1026.jar
      - ./resources/deps/hadoop/hadoop-shaded-guava-1.1.1.jar:/opt/hive/lib/hadoop-shaded-guava-1.1.1.jar
    healthcheck:
      test: ["CMD", "bash", "-c", "exec 3<>/dev/tcp/localhost/9083"]
      interval: 2s
      timeout: 2s
      retries: 10
      start_period: 3s

  spark:
    image: apache/spark:3.5.5-java17-python3
    container_name: spark-iceberg
    command: >
      bash -c "
      mkdir -p /tmp/spark-events &&
      /opt/spark/bin/spark-class org.apache.spark.deploy.history.HistoryServer \
        --properties-file /tmp/spark-defaults-history.conf"
    ports:
      - "4040:4040" # Spark Web UI
      - "18080:18080" # Spark History Server
    networks:
      - factorhouse
    depends_on:
      hive-metastore:
        condition: service_healthy
    volumes:
      - ./resources/spark/spark-defaults.conf:/opt/spark/conf/spark-defaults.conf:ro
      - ./resources/spark/spark-defaults-history.conf:/tmp/spark-defaults-history.conf:ro
      - ./resources/spark/log4j2.properties:/opt/spark/conf/log4j2.properties:ro
      - ./resources/spark/hive-site.xml:/opt/spark/conf/hive-site.xml
      - ./resources/spark/core-site.xml:/opt/spark/conf/core-site.xml
      # Iceberg, Hadoop dependencies
      - ./resources/deps/spark/iceberg/iceberg-spark-runtime-3.5_2.12-1.8.1.jar:/opt/spark/jars/iceberg-spark-runtime-3.5_2.12-1.8.1.jar
      - ./resources/deps/spark/iceberg/iceberg-aws-bundle-1.8.1.jar:/opt/spark/jars/iceberg-aws-bundle-1.8.1.jar
      - ./resources/deps/hadoop/hadoop-aws-3.3.6.jar:/opt/spark/jars/hadoop-aws-3.3.6.jar
      - ./resources/deps/hadoop/aws-java-sdk-bundle-1.11.1026.jar:/opt/spark/jars/aws-java-sdk-bundle-1.11.1026.jar
      - ./resources/deps/hadoop/hadoop-common-3.3.6.jar:/opt/spark/jars/hadoop-common-3.3.6.jar
      # Lineage dependencies
      - ./resources/deps/spark/lineage/openlineage-spark_2.12-1.37.0.jar:/tmp/jars/openlineage-spark_2.12-1.37.0.jar
      - ./resources/deps/spark/lineage/openmetadata-spark-agent-1.1.jar:/tmp/jars/openmetadata-spark-agent-1.1.jar
      - ./resources/deps/postgres/postgresql-42.7.3.jar:/tmp/jars/postgresql-42.7.3.jar
    environment:
      - SPARK_NO_DAEMONIZE=true
      - AWS_ACCESS_KEY_ID=admin
      - AWS_SECRET_ACCESS_KEY=password
      - AWS_REGION=us-east-1

  redis:
    image: redis:8
    container_name: redis
    command: redis-server --appendonly yes --loglevel warning --requirepass redis-pass
    restart: always
    ports:
      - "6379:6379"
    networks:
      - factorhouse

networks:
  factorhouse:
    external: ${USE_EXT:-true}
    name: factorhouse