Commit 713d9dfb by Amir Aharon

add fluentd pulsar files

parent 8b17adad
<source>
@type forward
port 24224
bind 0.0.0.0
</source>
<source>
@type exec
command sh /fluentd/plugins/plugin-run.sh
tag fluent-pulsar
format json
</source>
<match *.**>
@type elasticsearch
host elasticsearch
port 9200
index_name activityidx
type_name activity
include_timestamp true
</match>
\ No newline at end of file
# Fluentd related parameters
# Fluentd instance destinations. Default is only localhost:24224
fluentd.connect=fluentd-es:24224
# Fluentd client buffer configurations: https://github.com/komamitsu/fluency#buffer-configuration
#fluentd.client.buffer.chunk.initial.bytes = 1000000
#fluentd.client.buffer.chunk.retention.bytes = 2000000
#fluentd.client.buffer.max.bytes = 128000000
# Fixed event tag. Consumer sends events with fluentd.tag value to fluentd instance
#fluentd.tag=test
# Dynamic event tag with topic name. If you set "kafka.", generated event tag is "kafka.topic_name". Can't set both fluentd.tag and fluentd.tag.prefix at the same time
fluentd.tag.prefix=pulsar.
# Message format. json or regexp are supported. json is default. fluentd.record.pattern is used when use regexp format.
# Use joni, Oniguruma port for Java, for regular expression engine.
#fluentd.record.format=regexp
#fluentd.record.pattern=(?<text>.*)
# Consumed topics. Java's regex could be used like "test.*" for consuming test1, test2 ... testN topics
fluentd.consumer.topics=mcx/public/.*
# Supported patterns are "whitelist" or "blacklist". Default is "whitelist"
#fluentd.consumer.topics.pattern=blacklist
# Use parsed time instead of current time for fluentd event. fluentd.record.time.pattern is based on SimpleDateFormat: http://docs.oracle.com/javase/7/docs/api/java/text/SimpleDateFormat.html
#fluentd.record.time.field=published
#fluentd.record.time.pattern=dd/MMM/yyyy:HH:mm:ss Z
# The number of threads per consumer streams
fluentd.consumer.threads=1
# Batch size of Kafka offset commit. Default is 1000
#fluentd.consumer.batch.size=1000
# The path for backup un-flushed events during shutdown.
fluentd.consumer.backup.dir=/tmp/fluentd-consumer-backup/
# For debugging
fluentd.consumer.from.beginning=true
# Pulsar Consumer related parameters
pulsar.service.url=pulsar://pulsar:6650
#pulsar.consumer.subscription.name=
...@@ -2,7 +2,7 @@ version: '2' ...@@ -2,7 +2,7 @@ version: '2'
services: services:
##### Mongo ##### Mongo
mongodb: mongodb:
image: mongo image: mongo:3.4.4
ports: ports:
- "27027:27017" - "27027:27017"
volumes: volumes:
...@@ -13,28 +13,45 @@ services: ...@@ -13,28 +13,45 @@ services:
##### Redis ##### Redis
redis: redis:
image: redis image: redis
ports: # ports:
- "6389:6379" # - "6389:6379"
volumes: volumes:
- "/ext/redis/conf/:/usr/local/etc/redis/redis.conf" - "/ext/redis/conf/:/usr/local/etc/redis/redis.conf"
- "/ext/redis/:/data/" - "/ext/redis/:/data/"
networks: networks:
- backend - backend
### Orient ### Orient
# orientdb-old:
# image: aquabiota/orientdb-alpine-spatial
# command: /orientdb/bin/server.sh -Xms512m -Xmx512m
# restart: on-failure
# ports:
# - "2424:2424"
# - "2480:2480"
# environment:
# ORIENTDB_ROOT_PASSWORD: 'giptmgr'
# ORIENTDB_OPTS_MEMORY: "-Xms512m -Xmx2048m"
# volumes:
# - /ext/orientdb/config:/orientdb/config
# - /ext/orientdb/databases:/orientdb/databases
# - /ext/orientdb/backup:/orientdb/backup
# networks:
# - backend
orientdb: orientdb:
image: aquabiota/orientdb-alpine-spatial image: orientdb:3.0.22
# command: /orientdb/bin/server.sh -Xms512m -Xmx512m # command: /orientdb/bin/server.sh -Xms512m -Xmx512m
restart: on-failure restart: on-failure
ports: ports:
- "2424:2424" - "2424:2424"
- "2480:2480" - "2480:2480"
environment: environment:
ORIENTDB_ROOT_PASSWORD: 'giptmgr' ORIENTDB_ROOT_PASSWORD: 'giptmgrr'
ORIENTDB_OPTS_MEMORY: "-Xms512m -Xmx512m" ORIENTDB_OPTS_MEMORY: "-Xmx800m -Dstorage.diskCache.bufferSize=7200 -server -XX:+PerfDisableSharedMem -Dstorage.wal.syncOnPageFlush=false"
# ORIENTDB_OPTS_MEMORY: "-Xms512m -Xmx4096m"
volumes: volumes:
- /ext/orientdb/config:/orientdb/config # - /ext/orientdb-3/config:/orientdb/config
- /ext/orientdb/databases:/orientdb/databases - /ext/orientdb-3/databases:/orientdb/databases
- /ext/orientdb/backup:/orientdb/backup - /ext/orientdb-3/backup:/orientdb/backup
networks: networks:
- backend - backend
###Open Trip Planner ###Open Trip Planner
...@@ -61,27 +78,59 @@ services: ...@@ -61,27 +78,59 @@ services:
# JAVA_ARGS: "-Xms512m -Xmx512m" # JAVA_ARGS: "-Xms512m -Xmx512m"
networks: networks:
- backend - backend
elk: elasticsearch:
image: sebp/elk image: elasticsearch:5.5.1
ports: ports:
- "5601:5601" - "9234:9200"
- "9200:9200" - "9345:9300"
environment: environment:
ES_JAVA_OPTS: "-Xms4096m -Xmx4096m" ES_JAVA_OPTS: "-Xms1g -Xmx1g"
discovery.type: single-node
cluster.name: elasticsearch
# ES_HEAP_SIZE: "2048m"
ulimits:
memlock:
soft: -1
hard: -1
# mem_limit: 2048m
# mem_reservation: 1600m
volumes:
- "/ext/elasticsearch:/usr/share/elasticsearch/data"
- "/opt/mcx/config/elasticsearch/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml"
networks:
- backend
# user: elastic, pass:changeme
# before running ,run: sudo sysctl -w vm.max_map_count=262144
kibana:
image: kibana:5.5.1
# ports:
# - "5601:5601"
environment:
# ES_JAVA_OPTS: "-Xms4096m -Xmx4096m"
ES_HEAP_SIZE: "2048m" ES_HEAP_SIZE: "2048m"
volumes: volumes:
- "/ext/elasticsearch:/var/lib/elasticsearch" - "/ext/elasticsearch:/var/lib/elasticsearch"
networks: networks:
- backend - backend
fluentd-es: fluentd-es:
image: openfirmware/fluentd-elasticsearch image: municipalitybank.com:5050/ipgallery.esp/pulsar-fluentd-consumer/fluentd-es
ports: ports:
- "24224:24224" - "24224:24224"
- "8888:8888" - "24224:24224/udp"
# - "8888:8888"
volumes:
- /opt/mcx/config/fluentd/conf/fluent.conf:/fluentd/etc/fluent.conf
links: links:
- elk:elasticsearch - elasticsearch
networks:
- backend
pulsar-fluentd-consumer:
image: municipalitybank.com:5050/ipgallery.esp/pulsar-fluentd-consumer:2-6697132
volumes:
- /opt/mcx/config/fluentd/conf/fluentd-consumer.properties:/home/fluentd/fluentd-consumer.properties
networks: networks:
- backend - backend
# user: elastic, pass:changeme # user: elastic, pass:changeme
# before running ,run: sudo sysctl -w vm.max_map_count=262144 # before running ,run: sudo sysctl -w vm.max_map_count=262144
# add follwing line in /etc/sysctl.conf # add follwing line in /etc/sysctl.conf
...@@ -98,7 +147,7 @@ services: ...@@ -98,7 +147,7 @@ services:
INFLUXDB_USER_PASSWORD: "giptmgr" INFLUXDB_USER_PASSWORD: "giptmgr"
INFLUXDB_ADMIN_ENABLED: 'true' INFLUXDB_ADMIN_ENABLED: 'true'
INFLUXDB_ADMIN_BIND_ADDRESS: ":8083" INFLUXDB_ADMIN_BIND_ADDRESS: ":8083"
image: influxdb image: influxdb:1.4.2
ports: ports:
- "8084:8083" - "8084:8083"
- "8086:8086" - "8086:8086"
...@@ -106,14 +155,15 @@ services: ...@@ -106,14 +155,15 @@ services:
volumes: volumes:
- /ext/influxdb:/var/lib/influxdb - /ext/influxdb:/var/lib/influxdb
- /tmp/influxdb:/tmp - /tmp/influxdb:/tmp
- /opt/mcx/config/influxdb/influxdb.conf:/etc/influxdb/influxdb.conf
networks: networks:
- backend - backend
grafana: grafana:
environment: environment:
GF_INSTALL_PLUGINS: "grafana-piechart-panel,briangann-gauge-panel" GF_INSTALL_PLUGINS: "grafana-piechart-panel,briangann-gauge-panel"
GF_AUTH_ANONYMOUS_ENABLED: 'true' GF_AUTH_ANONYMOUS_ENABLED: 'true'
# GF_AUTH_ANONYMOUS_ORG_NAME: 'Main Org' GF_AUTH_ANONYMOUS_ORG_NAME: 'Main Org.'
GF_AUTH_ANONYMOUS_ORG_ROLE: 'Anonymous' GF_AUTH_ANONYMOUS_ORG_ROLE: 'Editor'
GF_SERVER_DOMAIN: 'ipgallery-mcz.com' GF_SERVER_DOMAIN: 'ipgallery-mcz.com'
GF_SERVER_ROOT_URL: '%(protocol)s://%(domain)s:/grafana' GF_SERVER_ROOT_URL: '%(protocol)s://%(domain)s:/grafana'
HTTP_USER: "root" HTTP_USER: "root"
...@@ -124,7 +174,7 @@ services: ...@@ -124,7 +174,7 @@ services:
INFLUXDB_NAME: "mcx_db" INFLUXDB_NAME: "mcx_db"
INFLUXDB_USER: "root" INFLUXDB_USER: "root"
INFLUXDB_PASS: "giptmgrr" INFLUXDB_PASS: "giptmgrr"
image: grafana/grafana image: grafana/grafana:4.6.0
ports: ports:
- "3000:3000" - "3000:3000"
- "8048:80" - "8048:80"
...@@ -132,6 +182,7 @@ services: ...@@ -132,6 +182,7 @@ services:
- influxdb - influxdb
volumes: volumes:
- /ext/grafana:/var/lib/grafana - /ext/grafana:/var/lib/grafana
- /ext/grafana/grafana.ini:/etc/grafana/grafana.ini
#user/pass = admin/admin #user/pass = admin/admin
networks: networks:
- backend - backend
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment