spark-minio-delta-lakehouse-docker/docker-compose.yaml at main · ion-bostanica/spark-minio-delta-lakehouse-docker · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
version: '3.7'
services:

  minio:
    hostname: minio
    container_name: minio
    image: 'minio/minio:RELEASE.2024-01-13T07-53-03Z'
    ports:
      - '9000:9000'
      - '9001:9001'
    volumes:
      - ./minio-data:/data
    environment:
      MINIO_ROOT_USER: accesskey
      MINIO_ROOT_PASSWORD: secretkey
    command: server /data --console-address ":9001"
    networks:
      - wba-network

  createbucket:
    hostname: createbucket
    container_name: createbucket
    image: minio/mc:RELEASE.2024-01-13T08-44-48Z
    depends_on:
      - minio
    entrypoint: >
      /bin/sh -c "
      /usr/bin/mc alias set myminio http://minio:9000 accesskey secretkey;
      /usr/bin/mc mb myminio/wba;
      /usr/bin/mc anonymous set public myminio/wba;
      "
    networks:
      - wba-network

  postgres:
    hostname: postgres
    container_name: postgres
    image: postgres:10-alpine
    ports:
      - 5432:5432
    environment:
      POSTGRES_USER: admin
      POSTGRES_PASSWORD: admin
      POSTGRES_DB: metastore_db
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U admin -d metastore_db"]
      interval: 1s
      timeout: 10s
      retries: 10
    networks:
      - wba-network

  hive-metastore:
    hostname: hive-metastore
    container_name: hive-metastore
    build: .
    ports:
      - 9083:9083
    environment:
      SERVICE_NAME: metastore
      DB_DRIVER: postgres
      HIVE_CUSTOM_CONF_DIR: /opt/hive/conf
    volumes:
      - ./hive-config:/opt/hive/conf
      - ./hadoop-libs/hadoop-aws-3.1.0.jar:/opt/hive/lib/hadoop-aws-3.1.0.jar
      - ./hadoop-libs/aws-java-sdk-bundle-1.11.271.jar:/opt/hive/lib/aws-java-sdk-bundle-1.11.271.jar
    depends_on:
      postgres:
        condition: service_healthy
    networks:
      - wba-network

  hive-server:
    hostname: hive-server
    container_name: hive-server
    image: apache/hive:3.1.3
    ports:
      - 10000:10000
      - 10002:10002
    environment:
      SERVICE_NAME: hiveserver2
      IS_RESUME: "true"
      HIVE_CUSTOM_CONF_DIR: /opt/hive/conf
    volumes:
      - ./hive-config:/opt/hive/conf
      - ./hadoop-libs/hadoop-aws-3.1.0.jar:/opt/hive/lib/hadoop-aws-3.1.0.jar
      - ./hadoop-libs/aws-java-sdk-bundle-1.11.271.jar:/opt/hive/lib/aws-java-sdk-bundle-1.11.271.jar
    depends_on:
      postgres:
        condition: service_healthy
    networks:
      - wba-network

  spark-master:
    hostname: spark-master
    container_name: spark-master
    image: bitnami/spark:3.4.1
    command: bin/spark-class org.apache.spark.deploy.master.Master
    volumes:
      - ./spark-apps:/opt/spark-apps
      - ./spark-config/spark-defaults.conf:/opt/bitnami/spark/conf/spark-defaults.conf
      - ./spark-config/hive-site.xml:/opt/bitnami/spark/conf/hive-site.xml
    ports:
      - "8080:8080"
      - "7077:7077"
    networks:
      - wba-network

  spark-worker-1:
    hostname: spark-worker-1
    container_name: spark-worker-1
    image: bitnami/spark:3.4.1
    command: bin/spark-class org.apache.spark.deploy.worker.Worker spark://spark-master:7077
    depends_on:
      - spark-master
    environment:
      SPARK_MODE: worker
      SPARK_WORKER_CORES: 2
      SPARK_WORKER_MEMORY: 5g
      SPARK_MASTER_URL: spark://spark-master:7077
    networks:
      - wba-network

  spark-worker-2:
    hostname: spark-worker-2
    container_name: spark-worker-2
    image: bitnami/spark:3.4.1
    command: bin/spark-class org.apache.spark.deploy.worker.Worker spark://spark-master:7077
    depends_on:
      - spark-master
    environment:
      SPARK_MODE: worker
      SPARK_WORKER_CORES: 2
      SPARK_WORKER_MEMORY: 5g
      SPARK_MASTER_URL: spark://spark-master:7077
    networks:
      - wba-network

  kyuubi:
    hostname: kyuubi
    container_name: kyuubi
    image: apache/kyuubi:1.8.0-spark
    volumes:
      - ./kyuubi-config/kyuubi-defaults.conf:/opt/kyuubi/conf/kyuubi-defaults.conf
    ports:
      - "10009:10009"
      - "10099:10099"
    depends_on:
      - spark-master
    networks:
      - wba-network

networks:
  wba-network:
    driver: bridge