add docs and new docker for flink with pulsar

common/docs/UsefulCommands.txt

@@ -28,6 +28,9 @@ then su gallery
 ssh -oKexAlgorithms=+diffie-hellman-group1-sha1 cassi@205.240.200.55
 ssh -oKexAlgorithms=+diffie-hellman-group1-sha1 cassi@205.240.200.55
 pass: g@ll3r&15
 pass: g@ll3r&15
 
 
+[elasticsearch]
+- curator client for nanaging indices:
+  https://www.elastic.co/guide/en/elasticsearch/client/curator/4.2/pip.html
 
 
 [Docker]
 [Docker]
 - docker viewer: docker run -d -p 9000:9000 --restart always -v /var/run/docker.sock:/var/run/docker.sock -v /opt/portainer:/data portainer/portainer
 - docker viewer: docker run -d -p 9000:9000 --restart always -v /var/run/docker.sock:/var/run/docker.sock -v /opt/portainer:/data portainer/portainer

common/docs/elasticsearch-rest.txt

@@ -46,15 +46,7 @@ curl -XPOST 'http://172.16.1.72:9200/activityidx/activity/_search' -d '{
 }'
 }'
 
 
 # delete by query
 # delete by query
-curl -XPOST 'http://172.16.1.72:9200/activityidx/activity/_delete_by_query?conflicts=proceed' -d '{
- "query": {
-  "range": {
-   "published": {
-    "lte": "now-90d"
-   }
-  }
- }
-}'
+curl -XPOST 'http://172.16.1.72:9200/activityidx/activity/_delete_by_query?conflicts=proceed' -d '{ "query": {  "range": {   "published": {    "lte": "now-60d"   }  } }}'
 
 
 
 
 curl -XGET "http://localhost:9200/activityidx/activity/_search" -d'
 curl -XGET "http://localhost:9200/activityidx/activity/_search" -d'
@@ -189,3 +181,13 @@ es2csv -q 'type:"thing/traffic-lane" AND userId:"chicago/*"' -u 172.16.1.72:9200
 
 
 es2csv -q 'type:"thing/traffic-lane" AND userId:"chicago/*" AND creationTime:>1519895494000' -u 172.16.1.72:9200 -o activities-traffic-chicago.csv -m 100000 -i activityidx -f creationTime id type metaData.loc metaData.measures.speed metaData.measures.vehicleCount
 es2csv -q 'type:"thing/traffic-lane" AND userId:"chicago/*" AND creationTime:>1519895494000' -u 172.16.1.72:9200 -o activities-traffic-chicago.csv -m 100000 -i activityidx -f creationTime id type metaData.loc metaData.measures.speed metaData.measures.vehicleCount
 
 
+
+// CURATOR
+command line app for managing elasticsearch indices.
+Install:
+our version of elasticsearch is 5.2 >> curator version 4.2
+https://www.elastic.co/guide/en/elasticsearch/client/curator/4.2/index.html
+- show indices:
+  curator_cli --host 172.16.1.72 --port 9200 show_indices
+
+

docker/cep/flink-pulsar/Dockerfile

+FROM flink:1.7
+ADD bcprov-jdk16-1.45.jar /opt/flink/lib/
+
+

docker/composers/mcx-cep.yml

+version: "2"
+services:
+  jobmanager:
+    # image: flink:1.7
+    image: municipalitybank.com:5050/mcx/devops/flink-pulsar
+    expose:
+      - "6123"
+    ports:
+      - "8081:8081"
+    command: jobmanager
+    environment:
+      - JOB_MANAGER_RPC_ADDRESS=jobmanager
+    networks:
+      - backend
+
+  taskmanager:
+    # image: flink:1.7
+    image: municipalitybank.com:5050/mcx/devops/flink-pulsar
+    expose:
+      - "6121"
+      - "6122"
+    depends_on:
+      - jobmanager
+    command: taskmanager
+    links:
+      - "jobmanager:jobmanager"
+    environment:
+      - JOB_MANAGER_RPC_ADDRESS=jobmanager
+    networks:
+      - backend
+
+  pulsar:
+    image: apachepulsar/pulsar-standalone
+    ports:
+      - 8080:8080
+      - 8082:80
+      - 6650:6650
+    volumes:
+      - "/ext/pulsar/data:/pulsar/data"
+    networks:
+      - backend
+networks:
+  backend:
+    driver: bridge

products/MCX/External/Pulsar-Admin.txt

+PULSAR-ADMIN:
+
+Running standalone
+docker run -it -p 80:80 -p 8080:8080 -p 6650:6650 -v $PWD/data:/pulsar/data apachepulsar/pulsar-standalone
+
+First Time:
+- create the tenant mcx
+- create the namepsace for publishing activities
+run "docker exec -it [pulsar-standalone-instance] bash"
+and then execute admin commands:
+
+- create tenant: bin/pulsar-admin tenants create mcx
+- create namespace in tenant: bin/pulsar-admin namespaces create mcx/publish
+- set replication clusters (needed to have policies, otherwise we can't publish):
+    bin/pulsar-admin namespaces set-clusters mcx/publish --clusters standalone
+
+
+[curl]
+- show tenant
+curl -s http://localhost:8080/admin/v2/tenants/mcz
+- show namespaces
+curl -s http://localhost:8080/admin/v2/namespaces/mcx/publish
+
+[pulsar-admin]
+https://pulsar.apache.org/docs/latest/admin-api/overview/
+
+

products/MCX/SOE/models.json

+[
+    {
+        "$schema": "http://json-schema.org/draft-04/schema#",
+        "title": "Event",
+        "description": "An Event",
+        "type": "object",
+         
+        "properties": {
+         
+           "id": {
+              "description": "The unique identifier for a event",
+              "type": "string"
+           },
+             
+           "entityId": {
+              "description": "id of the entity related to this event",
+              "type": "string"
+           },
+           
+           "tenant": {
+            "description": "tenant for this event",
+            "type": "string"
+          },
+         
+          "type": {
+            "description": "type of the event",
+            "type": "string"
+           },
+         
+           "creationTime": {
+            "description": "unix creation time milliseconds of this event",
+            "type": "number",
+            "format": "long"
+           },
+           
+          "published": {
+            "description": "rfc 3339 iso time of this event",
+             "type": "string"
+          },
+
+          "loc": {
+            "description": "geojson location",
+            "type": "array",
+            "items": { "type":"number", "format":"double" }
+          },
+
+         "payload": {
+            "description": "payload data for this event type",
+            "type": "object"
+           }
+        }
+    },
+    
+    {
+        "$schema": "http://json-schema.org/draft-04/schema#",
+        "title": "Control",
+        "description": "Control Message for creating simplealerts for events processing",
+        "type": "object",
+         
+        "properties": {
+         
+           "id": {
+              "description": "The unique identifier for an alert",
+              "type": "string"
+           },
+             
+           "tenant": {
+              "description": "tenant for this control",
+              "type": "string"
+           },
+           
+           "name": {
+            "description": "name for this alert",
+            "type": "string"
+           },
+         
+           "description": {
+            "description": "control alert description",
+            "type": "string"
+           },
+
+           "jsonPath": {
+            "description": "the alert rule in json-path",
+            "type": "string"
+           }
+
+        }
+    }
+    
+    
+]
\ No newline at end of file