HELK/docker/helk-kibana-notebook-analysis-basic.yml

version: '3.5'

services:
  helk-elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.5.1
    container_name: helk-elasticsearch
    logging:
      driver: "json-file"
      options:
        max-file: "9"
        max-size: "6m"
    secrets:
      - source: elasticsearch.yml
        target: /usr/share/elasticsearch/config/elasticsearch.yml
    volumes:
      - esdata:/usr/share/elasticsearch/data
      - ./helk-elasticsearch/scripts:/usr/share/elasticsearch/scripts
      - ./helk-elasticsearch/config/jvm.options:/usr/share/elasticsearch/config/jvm.options
    entrypoint: /usr/share/elasticsearch/scripts/elasticsearch-entrypoint.sh
    environment:
      - cluster.name=helk-cluster
      - node.name=helk-1
      - xpack.license.self_generated.type=basic
      - xpack.security.enabled=false
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nproc: 20480
      nofile:
        soft: 160000
        hard: 160000
    restart: always
    networks:
      helk:
  helk-logstash:
    image: docker.elastic.co/logstash/logstash:7.5.1
    container_name: helk-logstash
    logging:
      driver: "json-file"
      options:
        max-file: "9"
        max-size: "6m"
    secrets:
      - source: logstash.yml
        target: /usr/share/logstash/config/logstash.yml
    volumes:
      - ./helk-logstash/pipeline:/usr/share/logstash/pipeline
      - ./helk-logstash/output_templates:/usr/share/logstash/output_templates
      - ./helk-logstash/plugins:/usr/share/logstash/plugins
      - ./helk-logstash/enrichments/cti:/usr/share/logstash/cti
      - ./helk-logstash/scripts:/usr/share/logstash/scripts
    entrypoint: /usr/share/logstash/scripts/logstash-entrypoint.sh
    environment:
    - "HELK_LOGSTASH_JAVA_OPTS=-XX:-UseConcMarkSweepGC -XX:-UseCMSInitiatingOccupancyOnly -XX:+UseG1GC"
    ports:
      - "5044:5044"
      - "8531:8531"
      - "3515:3515"
    restart: always
    depends_on:
      - helk-kibana
    networks:
      helk:
  helk-kibana:
    image: docker.elastic.co/kibana/kibana:7.5.1
    container_name: helk-kibana
    logging:
      driver: "json-file"
      options:
        max-file: "9"
        max-size: "6m"
    secrets:
      - source: kibana.yml
        target: /usr/share/kibana/config/kibana.yml
    volumes:
      - ./helk-kibana/objects:/usr/share/kibana/objects
      - ./helk-kibana/scripts:/usr/share/kibana/scripts
      - ./helk-kibana/custom:/usr/share/kibana/custom
    entrypoint: /usr/share/kibana/scripts/kibana-entrypoint.sh
    restart: always
    depends_on:
      - helk-elasticsearch
    networks:
      helk:
  helk-nginx:
    image: cyb3rward0g/helk-nginx:0.0.7
    container_name: helk-nginx
    logging:
      driver: "json-file"
      options:
        max-file: "9"
        max-size: "6m"
    secrets:
      - source: htpasswd.users
        target: /etc/nginx/htpasswd.users
    volumes:
      - ./helk-nginx/config/basic-helk:/etc/nginx/sites-available/default
      - ./helk-nginx/scripts/:/opt/helk/scripts/
    entrypoint: /opt/helk/scripts/nginx-entrypoint.sh
    ports:
      - "80:80"
      - "443:443"
    restart: always
    depends_on:
      - helk-kibana
    networks:
      helk:
  helk-zookeeper:
    image: cyb3rward0g/helk-zookeeper:2.2.0
    container_name: helk-zookeeper
    logging:
      driver: "json-file"
      options:
        max-file: "5"
        max-size: "1m"
    restart: always
    depends_on:
      - helk-logstash
    networks:
      helk:
  helk-kafka-broker:
    image: cyb3rward0g/helk-kafka-broker:2.2.0
    container_name: helk-kafka-broker
    logging:
      driver: "json-file"
      options:
        max-file: "5"
        max-size: "1m"
    restart: always
    depends_on:
      - helk-zookeeper
    environment:
      KAFKA_BROKER_NAME: helk-kafka-broker
      KAFKA_BROKER_ID: 1
      KAFKA_BROKER_PORT: 9092
      REPLICATION_FACTOR: 1
      ADVERTISED_LISTENER: ${ADVERTISED_LISTENER}
      ZOOKEEPER_NAME: helk-zookeeper
      KAFKA_CREATE_TOPICS: winlogbeat, SYSMON_JOIN, filebeat, nxlog_mordor
      KAFKA_HEAP_OPTS: -Xmx1G -Xms1G
      LOG_RETENTION_HOURS: 4
    ports:
      - "9092:9092"
    networks:
      helk:
  helk-ksql-server:
    image: confluentinc/cp-ksql-server:5.1.3
    container_name: helk-ksql-server
    logging:
      driver: "json-file"
      options:
        max-file: "5"
        max-size: "1m"
    restart: always
    depends_on:
      - helk-kafka-broker
    environment:
      KSQL_BOOTSTRAP_SERVERS: helk-kafka-broker:9092
      KSQL_LISTENERS: http://0.0.0.0:8088
      KSQL_KSQL_SERVICE_ID: wardog
      KSQL_CUB_KAFKA_TIMEOUT: 300
      KSQL_KSQL_COMMIT_INTERVAL_MS: 2000
      KSQL_KSQL_CACHE_MAX_BYTES_BUFFERING: 10000000
      KSQL_KSQL_STREAMS_AUTO_OFFSET_RESET: earliest
      KSQL_HEAP_OPTS: -Xmx1g
      KSQL_OPTS: "-Dconfluent.support.metrics.enable=false"
    ports:
      - 8088:8088
    networks:
      helk:
  helk-ksql-cli:
    image: confluentinc/cp-ksql-cli:5.1.3
    container_name: helk-ksql-cli
    logging:
      driver: "json-file"
      options:
        max-file: "5"
        max-size: "1m"
    depends_on:
      - helk-ksql-server
    environment:
        KSQL_HEAP_OPTS: -Xmx1g
    entrypoint: /bin/sh
    tty: true
    networks:
      helk:
  helk-jupyter:
    build: helk-jupyter/
    container_name: helk-jupyter
    logging:
      driver: "json-file"
      options:
        max-file: "5"
        max-size: "1m"
    environment:
      JUPYTER_TYPE: lab
      JUPYTER_BASE_URL: /jupyter
    volumes:
      - notebooks:/opt/helk/jupyter/notebooks
    restart: always
    depends_on:
      - helk-logstash
    networks:
      helk:
  helk-spark-master:
    image: cyb3rward0g/helk-spark-master:2.4.3
    container_name: helk-spark-master
    logging:
      driver: "json-file"
      options:
        max-file: "5"
        max-size: "1m"
    environment:
      SPARK_MASTER_PORT: 7077
      SPARK_MASTER_WEBUI_PORT: 8080
    ports:
      - "8080:8080"
    restart: always
    depends_on:
      - helk-logstash
    networks:
      helk:
  helk-spark-worker:
    image: cyb3rward0g/helk-spark-worker:2.4.3
    container_name: helk-spark-worker
    logging:
      driver: "json-file"
      options:
        max-file: "5"
        max-size: "1m"
    environment:
      SPARK_MASTER: spark://helk-spark-master:7077
      SPARK_WORKER_MEMORY: 1g
      SPARK_WORKER_WEBUI_PORT: 8081
      SPARK_WORKER_PORT: 42950
    restart: always
    depends_on:
      - helk-spark-master
    networks:
      helk:

networks:
  helk:
    driver: bridge

volumes:
  esdata:
    driver: local
  notebooks:
    driver: local

secrets:
  elasticsearch.yml:
    file: ./helk-elasticsearch/config/elasticsearch.yml
  logstash.yml:
    file: ./helk-logstash/config/logstash.yml
  kibana.yml:
    file: ./helk-kibana/config/kibana.yml
  htpasswd.users:
    file: ./helk-nginx/htpasswd.users