Commit e9cb8f10 by Amir Aharon

add docs and new docker for flink with pulsar

parent 4246835c
......@@ -28,6 +28,9 @@ then su gallery
ssh -oKexAlgorithms=+diffie-hellman-group1-sha1 cassi@205.240.200.55
pass: g@ll3r&15
[elasticsearch]
- curator client for nanaging indices:
https://www.elastic.co/guide/en/elasticsearch/client/curator/4.2/pip.html
[Docker]
- docker viewer: docker run -d -p 9000:9000 --restart always -v /var/run/docker.sock:/var/run/docker.sock -v /opt/portainer:/data portainer/portainer
......
......@@ -46,15 +46,7 @@ curl -XPOST 'http://172.16.1.72:9200/activityidx/activity/_search' -d '{
}'
# delete by query
curl -XPOST 'http://172.16.1.72:9200/activityidx/activity/_delete_by_query?conflicts=proceed' -d '{
"query": {
"range": {
"published": {
"lte": "now-90d"
}
}
}
}'
curl -XPOST 'http://172.16.1.72:9200/activityidx/activity/_delete_by_query?conflicts=proceed' -d '{ "query": { "range": { "published": { "lte": "now-60d" } } }}'
curl -XGET "http://localhost:9200/activityidx/activity/_search" -d'
......@@ -189,3 +181,13 @@ es2csv -q 'type:"thing/traffic-lane" AND userId:"chicago/*"' -u 172.16.1.72:9200
es2csv -q 'type:"thing/traffic-lane" AND userId:"chicago/*" AND creationTime:>1519895494000' -u 172.16.1.72:9200 -o activities-traffic-chicago.csv -m 100000 -i activityidx -f creationTime id type metaData.loc metaData.measures.speed metaData.measures.vehicleCount
// CURATOR
command line app for managing elasticsearch indices.
Install:
our version of elasticsearch is 5.2 >> curator version 4.2
https://www.elastic.co/guide/en/elasticsearch/client/curator/4.2/index.html
- show indices:
curator_cli --host 172.16.1.72 --port 9200 show_indices
FROM flink:1.7
ADD bcprov-jdk16-1.45.jar /opt/flink/lib/
version: "2"
services:
jobmanager:
# image: flink:1.7
image: municipalitybank.com:5050/mcx/devops/flink-pulsar
expose:
- "6123"
ports:
- "8081:8081"
command: jobmanager
environment:
- JOB_MANAGER_RPC_ADDRESS=jobmanager
networks:
- backend
taskmanager:
# image: flink:1.7
image: municipalitybank.com:5050/mcx/devops/flink-pulsar
expose:
- "6121"
- "6122"
depends_on:
- jobmanager
command: taskmanager
links:
- "jobmanager:jobmanager"
environment:
- JOB_MANAGER_RPC_ADDRESS=jobmanager
networks:
- backend
pulsar:
image: apachepulsar/pulsar-standalone
ports:
- 8080:8080
- 8082:80
- 6650:6650
volumes:
- "/ext/pulsar/data:/pulsar/data"
networks:
- backend
networks:
backend:
driver: bridge
PULSAR-ADMIN:
Running standalone
docker run -it -p 80:80 -p 8080:8080 -p 6650:6650 -v $PWD/data:/pulsar/data apachepulsar/pulsar-standalone
First Time:
- create the tenant mcx
- create the namepsace for publishing activities
run "docker exec -it [pulsar-standalone-instance] bash"
and then execute admin commands:
- create tenant: bin/pulsar-admin tenants create mcx
- create namespace in tenant: bin/pulsar-admin namespaces create mcx/publish
- set replication clusters (needed to have policies, otherwise we can't publish):
bin/pulsar-admin namespaces set-clusters mcx/publish --clusters standalone
[curl]
- show tenant
curl -s http://localhost:8080/admin/v2/tenants/mcz
- show namespaces
curl -s http://localhost:8080/admin/v2/namespaces/mcx/publish
[pulsar-admin]
https://pulsar.apache.org/docs/latest/admin-api/overview/
[
{
"$schema": "http://json-schema.org/draft-04/schema#",
"title": "Event",
"description": "An Event",
"type": "object",
"properties": {
"id": {
"description": "The unique identifier for a event",
"type": "string"
},
"entityId": {
"description": "id of the entity related to this event",
"type": "string"
},
"tenant": {
"description": "tenant for this event",
"type": "string"
},
"type": {
"description": "type of the event",
"type": "string"
},
"creationTime": {
"description": "unix creation time milliseconds of this event",
"type": "number",
"format": "long"
},
"published": {
"description": "rfc 3339 iso time of this event",
"type": "string"
},
"loc": {
"description": "geojson location",
"type": "array",
"items": { "type":"number", "format":"double" }
},
"payload": {
"description": "payload data for this event type",
"type": "object"
}
}
},
{
"$schema": "http://json-schema.org/draft-04/schema#",
"title": "Control",
"description": "Control Message for creating simplealerts for events processing",
"type": "object",
"properties": {
"id": {
"description": "The unique identifier for an alert",
"type": "string"
},
"tenant": {
"description": "tenant for this control",
"type": "string"
},
"name": {
"description": "name for this alert",
"type": "string"
},
"description": {
"description": "control alert description",
"type": "string"
},
"jsonPath": {
"description": "the alert rule in json-path",
"type": "string"
}
}
}
]
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment