Skip to content

ipgallery / devops

Go to a project
Commit cac04193 by Adi Amir
Options

Merge branch 'master' of https://municipalitybank.com/mcx/devops

parents 24d2bdc7 21463da5
Inline Side-by-side
Showing with 1126 additions and 1 deletions
common/docs/Accounts.odt
No preview for this file type
common/docs/elasticsearch-rest.txt
...@@ -9,7 +9,7 @@ curl 172.16.1.72:9200/_cat/indices?v ...@@ -9,7 +9,7 @@ curl 172.16.1.72:9200/_cat/indices?v
curl -XPUT "http://localhost:9200/activityidx" -d'{ "mappings": { "activity": { "properties": { "metaData.loc": { "type": "geo_point" } } } } }' curl -XPUT "http://localhost:9200/activityidx" -d'{ "mappings": { "activity": { "properties": { "metaData.loc": { "type": "geo_point" } } } } }'
- https://www.elastic.co/blog/changing-mapping-with-zero-downtime - https://www.elastic.co/blog/changing-mapping-with-zero-downtime
Steps to change mapping type by reindexing and alias on a new index (using the kibana tools): Steps to change mapping type by reindexing and alias on a new index (using the kibana tools):
1 - get the mapping of the original index: GET activityidx/_mapping 1 - get the mapping of the original index: GET activityidx/_mapping (delete the starting field 'activityidx)
2 - create a json file with the changed mapping 2 - create a json file with the changed mapping
3 - curl -XPUT localhost:9200/activityidx_v1 -H 'Content-Type: application/json' -d @index-mapping (using curl) 3 - curl -XPUT localhost:9200/activityidx_v1 -H 'Content-Type: application/json' -d @index-mapping (using curl)
4 reindex to the new original to the new one: 4 reindex to the new original to the new one:
...@@ -203,6 +203,8 @@ POST /activityidx/activity/_search ...@@ -203,6 +203,8 @@ POST /activityidx/activity/_search
}\ }\
} }
{ "query": { "bool": { "must": [ { "query_string":{ "query":"tenant:fremont & type:\"report/traffic\" & metaData.event:TrafficJam" } } ], "filter" : { "geo_bounding_box" : { "metaData.loc" :{ "top_left" : { "lat" : 37.460391, "lon" : -122.167106 }, "bottom_right" : { "lat" : 37.152796, "lon" : -121.538847 } } }}}}}
///CSV ///CSV
pip install es2csv pip install es2csv
...@@ -214,6 +216,7 @@ es2csv -q 'type:"thing/traffic-lane" AND userId:"chicago/*" AND creationTime:>15 ...@@ -214,6 +216,7 @@ es2csv -q 'type:"thing/traffic-lane" AND userId:"chicago/*" AND creationTime:>15
es2csv -r -q '{ "query":{"bool":{"must":[{"query_string":{"query":"tenant:fremont & type:\"thing/traffic-lanes\"" }}],"filter":{"geo_bounding_box":{"metaData.loc":{"top_left":{"lat" : 37.629134,"lon" : -122.173405 }, "bottom_right" : { "lat" : 37.457418, "lon" : -121.835116 }}}}}}}' -u 172.16.1.72:9200 -o activities-traffic-fremont.csv -m 100000 -i activityidx -f creationTime id type metaData.loc metaData.toLocation metaData.measures.speed es2csv -r -q '{ "query":{"bool":{"must":[{"query_string":{"query":"tenant:fremont & type:\"thing/traffic-lanes\"" }}],"filter":{"geo_bounding_box":{"metaData.loc":{"top_left":{"lat" : 37.629134,"lon" : -122.173405 }, "bottom_right" : { "lat" : 37.457418, "lon" : -121.835116 }}}}}}}' -u 172.16.1.72:9200 -o activities-traffic-fremont.csv -m 100000 -i activityidx -f creationTime id type metaData.loc metaData.toLocation metaData.measures.speed
es2csv -r -q '{ "query": { "bool": { "must": [ { "query_string":{ "query":"metaData.event:TrafficJam" } } ], "filter" : { "geo_bounding_box" : { "metaData.loc" :{ "top_left" : { "lat" : 37.460391, "lon" : -122.167106 }, "bottom_right" : { "lat" : 37.152796, "lon" : -121.538847 } } }}}}}' -u 167.99.206.187:9200 -o activities-sanjose-traffic-reports.csv -m 5000000 -i activityidx -f creationTime id type metaData
// CURATOR // CURATOR
command line app for managing elasticsearch indices. command line app for managing elasticsearch indices.
......
common/scripts/install_protobuf3.sh 0 → 100644
#! /bin/bash
# Make sure you grab the latest version
curl -OL https://github.com/google/protobuf/releases/download/v3.6.1/protoc-3.6.1-linux-x86_64.zip
https://github.com/google/protobuf/releases/download/v3.6.1/protoc-3.6.1-linux-x86_64.zip
# Unzip
unzip protoc-3.6.1-linux-x86_64.zip -d protoc3
# Move protoc to /usr/local/bin/
sudo mv protoc3/bin/* /usr/local/bin/
# Move protoc3/include to /usr/local/include/
sudo mv protoc3/include/* /usr/local/include/
# Optional: change owner
sudo chown $USER /usr/local/bin/protoc
sudo chown -R $USER /usr/local/include/google
products/MCX/External/activity-esmapping.json 0 → 100644
{
"mappings": {
"activity": {
"properties": {
"action": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"creationTime": {
"type": "long"
},
"dest": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"metaData": {
"properties": {
"Loc": {
"type": "float"
},
"Problems": {
"properties": {
"Text": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"Topic": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"Type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"Profile": {
"properties": {
"age-range": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"ageRange": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"car-owner": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"carOwner": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"gender": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"street": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"ward": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"Sentiments": {
"properties": {
"Score": {
"type": "float"
},
"Text": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"Topic": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"Type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"Tags": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"__type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"address": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"agency_responsible": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"availability": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"city": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"congestion": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"created": {
"type": "long"
},
"crossWalk": {
"properties": {
"SFCNT": {
"type": "long"
},
"count": {
"type": "long"
}
}
},
"description": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"end": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"event": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"flows": {
"properties": {
"default": {
"type": "boolean"
},
"id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"source": {
"properties": {
"id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"workflowId": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"incidentId": {
"type": "long"
},
"label": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"lastModified": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"lat": {
"type": "float"
},
"loc": {
"type": "geo_point"
},
"lon": {
"type": "float"
},
"long": {
"type": "float"
},
"measures": {
"properties": {
"counter_direction": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"counter_direction_pedestrianCount": {
"type": "long"
},
"counter_direction_speed": {
"type": "long"
},
"counter_direction_vehicleCount": {
"type": "long"
},
"direction": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"max": {
"type": "float"
},
"mean": {
"type": "float"
},
"median": {
"type": "float"
},
"min": {
"type": "float"
},
"pedestrianCount": {
"type": "long"
},
"speed": {
"type": "float"
},
"tag": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"timestamp": {
"type": "long"
},
"unit": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"value": {
"type": "long"
},
"vehicleCount": {
"type": "long"
}
}
},
"media": {
"properties": {
"thumbnail": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"url": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"media_url": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"metaData": {
"properties": {
"activity": {
"properties": {
"action": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"creationTime": {
"type": "long"
},
"id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"metaData": {
"properties": {
"description": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"event": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"loc": {
"type": "float"
},
"measures": {
"properties": {
"speed": {
"type": "float"
}
}
},
"service_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"service_request_id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"traffic": {
"properties": {
"speed": {
"type": "float"
}
}
}
}
},
"published": {
"type": "date"
},
"tenant": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"userId": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"params": {
"properties": {
"_": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"box": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"dir": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"rt": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"stpid": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"vid": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"point": {
"properties": {
"coordinates": {
"type": "float"
},
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"problems": {
"properties": {
"Text": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"Topic": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"Type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"profile": {
"properties": {
"age-range": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"ageRange": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"car-owner": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"carOwner": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"gender": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"street": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"ward": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"requested_datetime": {
"type": "date"
},
"roadClosed": {
"type": "boolean"
},
"rules": {
"properties": {
"key": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"operator": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"value": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"sensorEvent": {
"properties": {
"max": {
"type": "float"
},
"mean": {
"type": "float"
},
"min": {
"type": "float"
}
}
},
"sentiments": {
"properties": {
"Score": {
"type": "float"
},
"Text": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"Topic": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"Type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"service_Name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"service_code": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"service_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"service_request_id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"severity": {
"type": "long"
},
"source": {
"type": "long"
},
"start": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"status": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"status_notes": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"tags": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"tenant": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"timestamp": {
"type": "long"
},
"toLocation": {
"type": "float"
},
"toPoint": {
"properties": {
"coordinates": {
"type": "float"
},
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"traffic": {
"properties": {
"direction": {
"type": "long"
},
"speed": {
"type": "long"
},
"vehicleCount": {
"type": "long"
}
}
},
"triggerId": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"type": {
"type": "long"
},
"updated_datetime": {
"type": "date"
},
"uri": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"userId": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"verified": {
"type": "boolean"
}
}
},
"published": {
"type": "date"
},
"query": {
"properties": {
"match_all": {
"type": "object"
},
"range": {
"properties": {
"published": {
"properties": {
"lte": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
}
}
},
"source": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"tenant": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"userId": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment