Commit 713d9dfb by Amir Aharon

add fluentd pulsar files

parent 8b17adad
<source>
@type forward
port 24224
bind 0.0.0.0
</source>
<source>
@type exec
command sh /fluentd/plugins/plugin-run.sh
tag fluent-pulsar
format json
</source>
<match *.**>
@type elasticsearch
host elasticsearch
port 9200
index_name activityidx
type_name activity
include_timestamp true
</match>
\ No newline at end of file
# Fluentd related parameters
# Fluentd instance destinations. Default is only localhost:24224
fluentd.connect=fluentd-es:24224
# Fluentd client buffer configurations: https://github.com/komamitsu/fluency#buffer-configuration
#fluentd.client.buffer.chunk.initial.bytes = 1000000
#fluentd.client.buffer.chunk.retention.bytes = 2000000
#fluentd.client.buffer.max.bytes = 128000000
# Fixed event tag. Consumer sends events with fluentd.tag value to fluentd instance
#fluentd.tag=test
# Dynamic event tag with topic name. If you set "kafka.", generated event tag is "kafka.topic_name". Can't set both fluentd.tag and fluentd.tag.prefix at the same time
fluentd.tag.prefix=pulsar.
# Message format. json or regexp are supported. json is default. fluentd.record.pattern is used when use regexp format.
# Use joni, Oniguruma port for Java, for regular expression engine.
#fluentd.record.format=regexp
#fluentd.record.pattern=(?<text>.*)
# Consumed topics. Java's regex could be used like "test.*" for consuming test1, test2 ... testN topics
fluentd.consumer.topics=mcx/public/.*
# Supported patterns are "whitelist" or "blacklist". Default is "whitelist"
#fluentd.consumer.topics.pattern=blacklist
# Use parsed time instead of current time for fluentd event. fluentd.record.time.pattern is based on SimpleDateFormat: http://docs.oracle.com/javase/7/docs/api/java/text/SimpleDateFormat.html
#fluentd.record.time.field=published
#fluentd.record.time.pattern=dd/MMM/yyyy:HH:mm:ss Z
# The number of threads per consumer streams
fluentd.consumer.threads=1
# Batch size of Kafka offset commit. Default is 1000
#fluentd.consumer.batch.size=1000
# The path for backup un-flushed events during shutdown.
fluentd.consumer.backup.dir=/tmp/fluentd-consumer-backup/
# For debugging
fluentd.consumer.from.beginning=true
# Pulsar Consumer related parameters
pulsar.service.url=pulsar://pulsar:6650
#pulsar.consumer.subscription.name=
......@@ -2,7 +2,7 @@ version: '2'
services:
##### Mongo
mongodb:
image: mongo
image: mongo:3.4.4
ports:
- "27027:27017"
volumes:
......@@ -13,28 +13,45 @@ services:
##### Redis
redis:
image: redis
ports:
- "6389:6379"
# ports:
# - "6389:6379"
volumes:
- "/ext/redis/conf/:/usr/local/etc/redis/redis.conf"
- "/ext/redis/:/data/"
networks:
- backend
### Orient
# orientdb-old:
# image: aquabiota/orientdb-alpine-spatial
# command: /orientdb/bin/server.sh -Xms512m -Xmx512m
# restart: on-failure
# ports:
# - "2424:2424"
# - "2480:2480"
# environment:
# ORIENTDB_ROOT_PASSWORD: 'giptmgr'
# ORIENTDB_OPTS_MEMORY: "-Xms512m -Xmx2048m"
# volumes:
# - /ext/orientdb/config:/orientdb/config
# - /ext/orientdb/databases:/orientdb/databases
# - /ext/orientdb/backup:/orientdb/backup
# networks:
# - backend
orientdb:
image: aquabiota/orientdb-alpine-spatial
image: orientdb:3.0.22
# command: /orientdb/bin/server.sh -Xms512m -Xmx512m
restart: on-failure
ports:
- "2424:2424"
- "2480:2480"
environment:
ORIENTDB_ROOT_PASSWORD: 'giptmgr'
ORIENTDB_OPTS_MEMORY: "-Xms512m -Xmx512m"
ORIENTDB_ROOT_PASSWORD: 'giptmgrr'
ORIENTDB_OPTS_MEMORY: "-Xmx800m -Dstorage.diskCache.bufferSize=7200 -server -XX:+PerfDisableSharedMem -Dstorage.wal.syncOnPageFlush=false"
# ORIENTDB_OPTS_MEMORY: "-Xms512m -Xmx4096m"
volumes:
- /ext/orientdb/config:/orientdb/config
- /ext/orientdb/databases:/orientdb/databases
- /ext/orientdb/backup:/orientdb/backup
# - /ext/orientdb-3/config:/orientdb/config
- /ext/orientdb-3/databases:/orientdb/databases
- /ext/orientdb-3/backup:/orientdb/backup
networks:
- backend
###Open Trip Planner
......@@ -61,27 +78,59 @@ services:
# JAVA_ARGS: "-Xms512m -Xmx512m"
networks:
- backend
elk:
image: sebp/elk
elasticsearch:
image: elasticsearch:5.5.1
ports:
- "5601:5601"
- "9200:9200"
- "9234:9200"
- "9345:9300"
environment:
ES_JAVA_OPTS: "-Xms4096m -Xmx4096m"
ES_JAVA_OPTS: "-Xms1g -Xmx1g"
discovery.type: single-node
cluster.name: elasticsearch
# ES_HEAP_SIZE: "2048m"
ulimits:
memlock:
soft: -1
hard: -1
# mem_limit: 2048m
# mem_reservation: 1600m
volumes:
- "/ext/elasticsearch:/usr/share/elasticsearch/data"
- "/opt/mcx/config/elasticsearch/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml"
networks:
- backend
# user: elastic, pass:changeme
# before running ,run: sudo sysctl -w vm.max_map_count=262144
kibana:
image: kibana:5.5.1
# ports:
# - "5601:5601"
environment:
# ES_JAVA_OPTS: "-Xms4096m -Xmx4096m"
ES_HEAP_SIZE: "2048m"
volumes:
- "/ext/elasticsearch:/var/lib/elasticsearch"
networks:
- backend
fluentd-es:
image: openfirmware/fluentd-elasticsearch
image: municipalitybank.com:5050/ipgallery.esp/pulsar-fluentd-consumer/fluentd-es
ports:
- "24224:24224"
- "8888:8888"
- "24224:24224/udp"
# - "8888:8888"
volumes:
- /opt/mcx/config/fluentd/conf/fluent.conf:/fluentd/etc/fluent.conf
links:
- elk:elasticsearch
- elasticsearch
networks:
- backend
pulsar-fluentd-consumer:
image: municipalitybank.com:5050/ipgallery.esp/pulsar-fluentd-consumer:2-6697132
volumes:
- /opt/mcx/config/fluentd/conf/fluentd-consumer.properties:/home/fluentd/fluentd-consumer.properties
networks:
- backend
# user: elastic, pass:changeme
# before running ,run: sudo sysctl -w vm.max_map_count=262144
# add follwing line in /etc/sysctl.conf
......@@ -98,7 +147,7 @@ services:
INFLUXDB_USER_PASSWORD: "giptmgr"
INFLUXDB_ADMIN_ENABLED: 'true'
INFLUXDB_ADMIN_BIND_ADDRESS: ":8083"
image: influxdb
image: influxdb:1.4.2
ports:
- "8084:8083"
- "8086:8086"
......@@ -106,14 +155,15 @@ services:
volumes:
- /ext/influxdb:/var/lib/influxdb
- /tmp/influxdb:/tmp
- /opt/mcx/config/influxdb/influxdb.conf:/etc/influxdb/influxdb.conf
networks:
- backend
grafana:
environment:
GF_INSTALL_PLUGINS: "grafana-piechart-panel,briangann-gauge-panel"
GF_AUTH_ANONYMOUS_ENABLED: 'true'
# GF_AUTH_ANONYMOUS_ORG_NAME: 'Main Org'
GF_AUTH_ANONYMOUS_ORG_ROLE: 'Anonymous'
GF_AUTH_ANONYMOUS_ORG_NAME: 'Main Org.'
GF_AUTH_ANONYMOUS_ORG_ROLE: 'Editor'
GF_SERVER_DOMAIN: 'ipgallery-mcz.com'
GF_SERVER_ROOT_URL: '%(protocol)s://%(domain)s:/grafana'
HTTP_USER: "root"
......@@ -124,7 +174,7 @@ services:
INFLUXDB_NAME: "mcx_db"
INFLUXDB_USER: "root"
INFLUXDB_PASS: "giptmgrr"
image: grafana/grafana
image: grafana/grafana:4.6.0
ports:
- "3000:3000"
- "8048:80"
......@@ -132,6 +182,7 @@ services:
- influxdb
volumes:
- /ext/grafana:/var/lib/grafana
- /ext/grafana/grafana.ini:/etc/grafana/grafana.ini
#user/pass = admin/admin
networks:
- backend
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment