Skip to content
The Chat Archive
gpt-4o-2024-08-06· OpenAI WildChat-4.8M (AI2)

version: '3.9' x-base-workers: &base-workers build: context: . dockerfile:…

  1. User

    version: '3.9'

    x-base-workers: &base-workers build: context: . dockerfile: ./docker/workers_api/Dockerfile networks: - tw-net env_file: - ./.envs/.env

    x-vars-workers: &vars-workers <<: *base-workers depends_on: db_init: condition: service_completed_successfully kafka-init: condition: service_completed_successfully kafka103: condition: service_started kafka102: condition: service_started kafka101: condition: service_started postgres_wallet: condition: service_started

    x-vars-kafka-kraft: &vars-kafka-kraft CLUSTER_ID: 'xtzWWN4bTjitpL3kfd9s5g' KAFKA_CONTROLLER_QUORUM_VOTERS: '101@kafka101:29093,102@kafka102:29093,103@kafka103:29093' KAFKA_PROCESS_ROLES: 'broker,controller' KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT' KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 2 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 2 CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1 CONFLUENT_METRICS_ENABLE: 'false' KAFKA_HEAP_OPTS: "-XX:MaxRAMPercentage=70.0"

    services:

    logstash:

    build:

    context: ./

    dockerfile: ./docker/logstash/Dockerfile

    env_file:

    - .envs/.env

    networks:

    - tw-net

    ports:

    - "5044:5044"

    kafka-init: image: "confluentinc/cp-kafka" container_name: kafka-init networks: - tw-net profiles: [ 'kafka' ] depends_on: - kafka101 entrypoint: [ '/bin/sh', '-c' ] command: | " sleep 10 # blocks until kafka is reachable kafka-topics --bootstrap-server kafka101:29092 --list

      echo -e 'Creating kafka topics'
      kafka-topics --bootstrap-server kafka101:29092,kafka102:29092,kafka103:29092 --create --if-not-exists --topic notification_topic --replication-factor 2 --partitions 6
      kafka-topics --bootstrap-server kafka101:29092,kafka102:29092,kafka103:29092 --create --if-not-exists --topic income_transactions --replication-factor 2 --partitions 6
      kafka-topics --bootstrap-server kafka101:29092,kafka102:29092,kafka103:29092 --create --if-not-exists --topic withdraw_request --replication-factor 2 --partitions 6
      kafka-topics --bootstrap-server kafka101:29092,kafka102:29092,kafka103:29092 --create --if-not-exists --topic withdraw_transaction --replication-factor 2 --partitions 6
      kafka-topics --bootstrap-server kafka101:29092,kafka102:29092,kafka103:29092 --create --if-not-exists --topic wallet_create_ipn_request --replication-factor 2 --partitions 6
      kafka-topics --bootstrap-server kafka101:29092,kafka102:29092,kafka103:29092 --create --if-not-exists --topic wallet_create_ipn_response --replication-factor 2 --partitions 6
      kafka-topics --bootstrap-server kafka101:29092,kafka102:29092,kafka103:29092 --create --if-not-exists --topic user_event --replication-factor 2 --partitions 6
      kafka-topics --bootstrap-server kafka101:29092,kafka102:29092,kafka103:29092 --create --if-not-exists --topic withdraw_status_from_ipn --replication-factor 2 --partitions 6
      kafka-topics --bootstrap-server kafka101:29092,kafka102:29092,kafka103:29092 --create --if-not-exists --topic internal_transfers --replication-factor 2 --partitions 6
      kafka-topics --bootstrap-server kafka101:29092,kafka102:29092,kafka103:29092 --create --if-not-exists --topic exc_withdraw_requests --replication-factor 2 --partitions 6
      kafka-topics --bootstrap-server kafka101:29092,kafka102:29092,kafka103:29092 --create --if-not-exists --topic exc_withdraw_requests --replication-factor 2 --partitions 6
      kafka-topics --bootstrap-server kafka101:29092,kafka102:29092,kafka103:29092 --create --if-not-exists --topic withdraw_commission_request --replication-factor 2 --partitions 6
      kafka-topics --bootstrap-server kafka101:29092,kafka102:29092,kafka103:29092 --create --if-not-exists --topic withdraw_commission_response --replication-factor 2 --partitions 6
    
      echo -e 'Successfully created the following topics:'
      kafka-topics --bootstrap-server kafka102:29092 --list
      sleep 10
      "
    

    kafka101: image: "confluentinc/cp-kafka" ports: - "19092:9092" - "9991:9991" container_name: kafka101_prod profiles: [ 'kafka' ] networks: - tw-net environment: KAFKA_BROKER_ID: 101 # WARN: node.id must be set to the same value as broker.id KAFKA_NODE_ID: 101 # random cluster ID used for formatting LOG_DIR for KRaft <<: *vars-kafka-kraft KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka101:29092,PLAINTEXT_HOST://localhost:19092' KAFKA_LISTENERS: 'PLAINTEXT://kafka101:29092,PLAINTEXT_HOST://kafka101:9092,CONTROLLER://kafka101:29093' CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka101:29092 deploy: resources: limits: memory: 512M # Limits the memory usage to 512 megabytes # Optionally, you can also set soft limits with reservation reservations: memory: 256M # Soft limit of 256 megabytes

    kafka102: image: "confluentinc/cp-kafka" ports: - "29092:9092" - "9992:9992" container_name: kafka102_prod profiles: [ 'kafka' ] networks: - tw-net environment: KAFKA_BROKER_ID: 102 # WARN: node.id must be set to the same value as broker.id KAFKA_NODE_ID: 102 # random cluster ID used for formatting LOG_DIR for KRaft <<: *vars-kafka-kraft KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka102:29092,PLAINTEXT_HOST://localhost:29092' KAFKA_LISTENERS: 'PLAINTEXT://kafka102:29092,PLAINTEXT_HOST://kafka102:9092,CONTROLLER://kafka102:29093' CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka102:29092 deploy: resources: limits: memory: 512M # Limits the memory usage to 512 megabytes # Optionally, you can also set soft limits with reservation reservations: memory: 256M # Soft limit of 256 megabytes

    kafka103: image: "confluentinc/cp-kafka" ports: - "39092:9092" - "9993:9993" container_name: kafka103_prod profiles: [ 'kafka' ] networks: - tw-net environment: KAFKA_BROKER_ID: 103 # WARN: node.id must be set to the same value as broker.id KAFKA_NODE_ID: 103 # random cluster ID used for formatting LOG_DIR for KRaft <<: *vars-kafka-kraft KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka103:29092,PLAINTEXT_HOST://localhost:39092' KAFKA_LISTENERS: 'PLAINTEXT://kafka103:29092,PLAINTEXT_HOST://kafka103:9092,CONTROLLER://kafka103:29093' CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka103:29092 deploy: resources: limits: memory: 512M # Limits the memory usage to 512 megabytes # Optionally, you can also set soft limits with reservation reservations: memory: 256M # Soft limit of 256 megabytes

    kafka_ui: image: provectuslabs/kafka-ui container_name: kafka-ui profiles: [ 'kafka' ] ports: - "8080:8080" environment: KAFKA_CLUSTERS_0_BOOTSTRAP_SERVERS: "kafka101:29092,kafka102:29092,kafka103:29092" KAFKA_CLUSTERS_0_NAME: "touchwallet_ms" networks: - tw-net

    mock_http: build: context: ./ dockerfile: docker/mock_api/Dockerfile hostname: mock_api profiles: [ 'mock_api' ] ports: - "1080:443" networks: - tw-net

    wallet_api: build: context: . dockerfile: ./docker/workers_api/Dockerfile entrypoint: /entrypoints/api.sh container_name: api hostname: api profiles: [ 'api' ] ports: - "8000:8000" env_file: - .envs/.env depends_on: - kafka103 - kafka102 - kafka101 - postgres_wallet - db_init networks: - tw-net

    postgres_wallet: build: context: . dockerfile: ./docker/postgres/Dockerfile restart: always hostname: postgres_wallet profiles: [ 'db' ] healthcheck: test: [ "CMD-SHELL", "pg_isready -U $$POSTGRES_USER" ] interval: 20s timeout: 5s retries: 5 env_file: - .envs/.env networks: - tw-net ports: - "5432:5432" volumes: - postgres_data_write:/bitnami/postgresql

    db_init: build: context: . dockerfile: ./docker/db_init/Dockerfile env_file: - .envs/.env networks: - tw-net profiles: [ 'db' ] depends_on: postgres_wallet: condition: service_healthy

    redis: image: redis hostname: redis profiles: [ 'redis' ] ports: - "6379:6379" # restart: always command: redis-server --save 20 1 --loglevel warning networks: - tw-net volumes: - cache:/data

    worker_income: <<: *vars-workers entrypoint: /entrypoints/income.sh profiles: [ 'workers' ]

    worker_withdraw: <<: *vars-workers entrypoint: /entrypoints/withdraw.sh profiles: [ 'workers' ]

    worker_exc_withdraw: <<: *vars-workers entrypoint: /entrypoints/exc_withdraw.sh profiles: [ 'workers' ]

    worker_withdraw_status: <<: *vars-workers entrypoint: /entrypoints/withdraw_status.sh profiles: [ 'workers' ]

    worker_wallet_status: <<: *vars-workers entrypoint: /entrypoints/wallet_status.sh profiles: [ 'workers' ]

    worker_user_event: <<: *vars-workers entrypoint: /entrypoints/user_event.sh profiles: [ 'workers' ]

    worker_touchcoin: <<: *vars-workers entrypoint: /entrypoints/touchcoin.sh profiles: [ 'workers' ]

    worker_int_transfer: <<: *vars-workers entrypoint: /entrypoints/internal_transfer.sh profiles: [ 'workers' ]

    worker_cache_api: <<: *base-workers entrypoint: /entrypoints/cache_api.sh profiles: [ 'workers' ]

    worker_system_balance: <<: *vars-workers entrypoint: /entrypoints/system_balance.sh profiles: [ 'workers' ]

    worker_income_s1: build: context: ./ dockerfile: docker/tests_stress/income/t1/Dockerfile <<: *vars-workers entrypoint: /entrypoints/income.sh profiles: [ 'tests_stress' ]

    worker_income_s2: build: context: ./ dockerfile: docker/tests_stress/income/t2/Dockerfile <<: *vars-workers entrypoint: /entrypoints/income.sh profiles: [ 'tests_stress' ]

    worker_withdraw_s1: build: context: ./ dockerfile: docker/tests_stress/withdraw/t1/Dockerfile <<: *vars-workers entrypoint: /entrypoints/withdraw.sh profiles: [ 'tests_stress' ]

    worker_withdraw_s2: build: context: ./ dockerfile: docker/tests_stress/withdraw/t2/Dockerfile <<: *vars-workers entrypoint: /entrypoints/withdraw.sh profiles: [ 'tests_stress' ]

    tests: build: context: ./ dockerfile: docker/tests/Dockerfile env_file: - .envs/.env profiles: [ 'tests' ] depends_on: postgres_wallet: condition: service_healthy db_init: condition: service_completed_successfully kafka-init: condition: service_completed_successfully worker_income: condition: service_started worker_touchcoin: condition: service_started worker_user_event: condition: service_started worker_wallet_status: condition: service_started worker_withdraw_status: condition: service_started worker_int_transfer: condition: service_started worker_withdraw: condition: service_started networks: - tw-net

    tests_local: build: context: ./ dockerfile: docker/tests_local/Dockerfile env_file: - .envs/.env volumes: - /var/run/docker.sock:/var/run/docker.sock profiles: [ 'tests_local' ] depends_on: postgres_wallet: condition: service_healthy db_init: condition: service_completed_successfully kafka-init: condition: service_completed_successfully worker_income: condition: service_started worker_touchcoin: condition: service_started worker_user_event: condition: service_started worker_wallet_status: condition: service_started worker_withdraw_status: condition: service_started worker_int_transfer: condition: service_started worker_withdraw: condition: service_started networks: - tw-net

    tests_stress: build: context: ./ dockerfile: docker/tests_stress/tests/Dockerfile env_file: - .envs/.test.env profiles: [ 'tests_stress' ] depends_on: postgres_wallet: condition: service_healthy db_init: condition: service_completed_successfully kafka-init: condition: service_completed_successfully worker_income_s1: condition: service_started worker_touchcoin: condition: service_started worker_user_event: condition: service_started worker_wallet_status: condition: service_started worker_withdraw_status: condition: service_started worker_withdraw_s1: condition: service_started networks: - tw-net

    volumes: postgres_data_write: { } cache: { }

    networks: tw-net: name: tw_net external: true

    перепиши для подключения worker локальными дирикториями

  2. Assistant

    To update your Docker Compose file so that workers can be connected using local directories (volumes), you'll need to modify the services to include volume mounts. This will allow you to connect your local files or directories to the containers.

    Here is your updated docker-compose.yaml with local directory mounts for the worker services. Make sure that you have these directories set up locally and adjust paths as needed:

    version: '3.9'
    
    x-base-workers: &base-workers
      build:
        context: .
        dockerfile: ./docker/workers_api/Dockerfile
      networks:
        - tw-net
      env_file:
        - ./.envs/.env
      volumes:
        - ./local/entrypoints:/entrypoints # Local directory for entrypoints
    
    x-vars-workers: &vars-workers
      <<: *base-workers
      depends_on:
        db_init:
          condition: service_completed_successfully
        kafka-init:
          condition: service_completed_successfully
        kafka103:
          condition: service_started
        kafka102:
          condition: service_started
        kafka101:
          condition: service_started
        postgres_wallet:
          condition: service_started
    
    # (Rest of your services and configurations here...)
    
      worker_income:
        <<: *vars-workers
        entrypoint: /entrypoints/income.sh
        profiles: [ 'workers' ]
        volumes:
          - ./local/entrypoints/income.sh:/entrypoints/income.sh # Specific entrypoint script
    
      worker_withdraw:
        <<: *vars-workers
        entrypoint: /entrypoints/withdraw.sh
        profiles: [ 'workers' ]
        volumes:
          - ./local/entrypoints/withdraw.sh:/entrypoints/withdraw.sh # Specific entrypoint script
    
      worker_exc_withdraw:
        <<: *vars-workers
        entrypoint: /entrypoints/exc_withdraw.sh
        profiles: [ 'workers' ]
        volumes:
          - ./local/entrypoints/exc_withdraw.sh:/entrypoints/exc_withdraw.sh # Specific entrypoint script
    
      worker_withdraw_status:
        <<: *vars-workers
        entrypoint: /entrypoints/withdraw_status.sh
        profiles: [ 'workers' ]
        volumes:
          - ./local/entrypoints/withdraw_status.sh:/entrypoints/withdraw_status.sh # Specific entrypoint script
    
      worker_wallet_status:
        <<: *vars-workers
        entrypoint: /entrypoints/wallet_status.sh
        profiles: [ 'workers' ]
        volumes:
          - ./local/entrypoints/wallet_status.sh:/entrypoints/wallet_status.sh # Specific entrypoint script
    
      worker_user_event:
        <<: *vars-workers
        entrypoint: /entrypoints/user_event.sh
        profiles: [ 'workers' ]
        volumes:
          - ./local/entrypoints/user_event.sh:/entrypoints/user_event.sh # Specific entrypoint script
    
      worker_touchcoin:
        <<: *vars-workers
        entrypoint: /entrypoints/touchcoin.sh
        profiles: [ 'workers' ]
        volumes:
          - ./local/entrypoints/touchcoin.sh:/entrypoints/touchcoin.sh # Specific entrypoint script
    
      worker_int_transfer:
        <<: *vars-workers
        entrypoint: /entrypoints/internal_transfer.sh
        profiles: [ 'workers' ]
        volumes:
          - ./local/entrypoints/internal_transfer.sh:/entrypoints/internal_transfer.sh # Specific entrypoint script
    
      worker_cache_api:
        <<: *base-workers
        entrypoint: /entrypoints/cache_api.sh
        profiles: [ 'workers' ]
        volumes:
          - ./local/entrypoints/cache_api.sh:/entrypoints/cache_api.sh # Specific entrypoint script
    
      worker_system_balance:
        <<: *vars-workers
        entrypoint: /entrypoints/system_balance.sh
        profiles: [ 'workers' ]
        volumes:
          - ./local/entrypoints/system_balance.sh:/entrypoints/system_balance.sh # Specific entrypoint script
    
    # (Rest of your service configurations here...)
    

    Key Notes:

    • Ensure you have a local/entrypoints directory in your project root that contains all the entrypoint scripts for each worker (income.sh, withdraw.sh, etc.).
    • Map these scripts using Docker's volume feature, adjusting the paths as needed based on where your local scripts are located.

    This way, you'll be able to modify the scripts locally, and those changes will be reflected in the running Docker containers.

  3. User

    и еще добавь к ним каталог app

  4. Assistant

    Certainly! To include an additional directory, app, into each of the worker services, you'll need to mount it in each service using Docker volumes. Make sure that this directory exists in your local setup.

    Here is how you can modify your docker-compose.yaml to include the app directory:

    version: '3.9'
    
    x-base-workers: &base-workers
      build:
        context: .
        dockerfile: ./docker/workers_api/Dockerfile
      networks:
        - tw-net
      env_file:
        - ./.envs/.env
      volumes:
        - ./local/entrypoints:/entrypoints # Local directory for entrypoints
        - ./local/app:/app # Local directory for app
    
    x-vars-workers: &vars-workers
      <<: *base-workers
      depends_on:
        db_init:
          condition: service_completed_successfully
        kafka-init:
          condition: service_completed_successfully
        kafka103:
          condition: service_started
        kafka102:
          condition: service_started
        kafka101:
          condition: service_started
        postgres_wallet:
          condition: service_started
    
    # (Rest of your services and configurations here...)
    
      worker_income:
        <<: *vars-workers
        entrypoint: /entrypoints/income.sh
        profiles: [ 'workers' ]
        volumes:
          - ./local/entrypoints/income.sh:/entrypoints/income.sh # Specific entrypoint script
          - ./local/app:/app # Local app directory
    
      worker_withdraw:
        <<: *vars-workers
        entrypoint: /entrypoints/withdraw.sh
        profiles: [ 'workers' ]
        volumes:
          - ./local/entrypoints/withdraw.sh:/entrypoints/withdraw.sh # Specific entrypoint script
          - ./local/app:/app # Local app directory
    
      worker_exc_withdraw:
        <<: *vars-workers
        entrypoint: /entrypoints/exc_withdraw.sh
        profiles: [ 'workers' ]
        volumes:
          - ./local/entrypoints/exc_withdraw.sh:/entrypoints/exc_withdraw.sh # Specific entrypoint script
          - ./local/app:/app # Local app directory
    
      worker_withdraw_status:
        <<: *vars-workers
        entrypoint: /entrypoints/withdraw_status.sh
        profiles: [ 'workers' ]
        volumes:
          - ./local/entrypoints/withdraw_status.sh:/entrypoints/withdraw_status.sh # Specific entrypoint script
          - ./local/app:/app # Local app directory
    
      worker_wallet_status:
        <<: *vars-workers
        entrypoint: /entrypoints/wallet_status.sh
        profiles: [ 'workers' ]
        volumes:
          - ./local/entrypoints/wallet_status.sh:/entrypoints/wallet_status.sh # Specific entrypoint script
          - ./local/app:/app # Local app directory
    
      worker_user_event:
        <<: *vars-workers
        entrypoint: /entrypoints/user_event.sh
        profiles: [ 'workers' ]
        volumes:
          - ./local/entrypoints/user_event.sh:/entrypoints/user_event.sh # Specific entrypoint script
          - ./local/app:/app # Local app directory
    
      worker_touchcoin:
        <<: *vars-workers
        entrypoint: /entrypoints/touchcoin.sh
        profiles: [ 'workers' ]
        volumes:
          - ./local/entrypoints/touchcoin.sh:/entrypoints/touchcoin.sh # Specific entrypoint script
          - ./local/app:/app # Local app directory
    
      worker_int_transfer:
        <<: *vars-workers
        entrypoint: /entrypoints/internal_transfer.sh
        profiles: [ 'workers' ]
        volumes:
          - ./local/entrypoints/internal_transfer.sh:/entrypoints/internal_transfer.sh # Specific entrypoint script
          - ./local/app:/app # Local app directory
    
      worker_cache_api:
        <<: *base-workers
        entrypoint: /entrypoints/cache_api.sh
        profiles: [ 'workers' ]
        volumes:
          - ./local/entrypoints/cache_api.sh:/entrypoints/cache_api.sh # Specific entrypoint script
          - ./local/app:/app # Local app directory
    
      worker_system_balance:
        <<: *vars-workers
        entrypoint: /entrypoints/system_balance.sh
        profiles: [ 'workers' ]
        volumes:
          - ./local/entrypoints/system_balance.sh:/entrypoints/system_balance.sh # Specific entrypoint script
          - ./local/app:/app # Local app directory
    
    # (Rest of your service configurations here...)
    

    Key Changes:

    • Added an additional volume mount - ./local/app:/app to each worker service so that the local app directory is accessible within the container.
    • Make sure you have a local/app directory with the necessary application code/configurations you want to share with your worker containers.

    This setup assumes a structure for local content, so adjust paths according to your actual directory layout. The containers will now have access to the contents of both the entrypoints and app directories.