added system control scripts for mcx

docker/script/sys-check.sh

+#!/bin/bash
+
+check_dockers_up_status() {
+
+  SYS_STATUS="OK"
+  
+  CUR_TIME=`date`
+  echo -e "${CUR_TIME}   sys-check.sh started ..."
+
+  check_docker_up_status mcx-external.yml
+  check_docker_up_status mcx-backend.yml
+  check_docker_up_status mcx-frontend.yml
+
+  echo -e ""
+  if [ $SYS_STATUS == "OK" ];
+  then    
+    echo -e "\e[32m******************\e[0m"
+    echo -e "\e[32m* SYSTEM IS OK ! *\e[0m"
+    echo -e "\e[32m******************\e[0m"
+  else
+    echo -e "\e[31mSYSTEM HAS ERRORS\e[0m"      
+  fi
+
+}
+
+check_docker_up_status() {
+
+  DOCKER_FILE=$1
+
+  echo -e "\n\nchecking status of ${DOCKER_FILE} ..."
+  /usr/local/bin/docker-compose -f $DOCKER_FILE ps
+  
+  NUM_OF_COMPONENTS=`/usr/local/bin/docker-compose -f $DOCKER_FILE ps | grep 'docker_' | wc -l`  
+  NUM_OF_UP_COMPONENTS=`/usr/local/bin/docker-compose -f $DOCKER_FILE ps | grep 'docker_' | grep Up | wc -l`
+
+  if [ $NUM_OF_UP_COMPONENTS -eq $NUM_OF_COMPONENTS ];
+  then 
+    DOCKER_STATUS="OK"
+  else
+    DOCKER_STATUS="FAIL"
+    SYS_STATUS="FAIL"
+  fi;
+  
+  if [ "$DOCKER_STATUS" == "FAIL" ]
+  then
+    echo -e "\e[31mERROR: Found status errors in ${DOCKER_FILE}\e[0m"   
+        
+    send_docker_status_alert 
+  else        
+    echo -e "\e[32m${DOCKER_FILE} is up and running!\e[0m"    
+  fi
+}
+
+send_docker_status_alert() {
+
+  # title
+  HOST_NAME=`cat /proc/sys/kernel/hostname`
+  EMAIL_TITLE="MCX-ALERT(host:${HOST_NAME}) docker status error!"
+
+  # message
+  cat /dev/null > alert-msg.txt
+  echo -e "Found status errors in: ${DOCKER_FILE}" >> alert-msg.txt
+  /usr/local/bin/docker-compose -f $DOCKER_FILE ps | grep docker_ >> alert-msg.txt
+  echo " " >> alert-msg.txt
+
+  send_alert
+}
+
+send_alert() {
+ 
+  echo "To: ipgallery.alert@gmail.com" >> alert-mail.txt
+  echo "From: ipgallery.alert@gmail.com" >> alert-mail.txt
+
+  SUBJECT="Subject: ${EMAIL_TITLE}"
+  cat /dev/null > alert-mail.txt
+  echo $SUBJECT >> alert-mail.txt
+  echo "" >> alert-mail.txt
+  cat alert-msg.txt >> alert-mail.txt
+
+  # send alert once only
+  ALERT_ID=`md5sum alert-mail.txt | awk '{print $1}'`
+  ALERT_EXIST=`find alerts/ -name "*.txt" | grep $ALERT_ID`
+  if [ -z "$ALERT_EXIST" ]; then
+    # this alerts does not exists in alerts/ folder yet - send alert now !
+    /usr/sbin/ssmtp  ipgallery.alert@gmail.com < alert-mail.txt
+    if [ $? == 0 ]
+    then
+      echo -e "\e[31msent alert [${ALERT_ID}] to ipgallery.alert@gmail.com!\e[0m"        
+
+      # save alert to disk      
+      mv alert-mail.txt alerts/
+      mv alerts/alert-mail.txt alerts/$ALERT_ID.txt
+    else        
+      echo -e "\e[31mERROR: failed to send alert [${ALERT_ID}]\e[0m"     
+    fi
+  else    
+    echo -e "\e[31maleary sent mail for alert [${ALERT_ID}]. do nothing\e[0m"   
+  fi
+   
+}
+
+#####  main  #####
+
+
+cd /opt/mcx/config/docker/
+mkdir -p alerts
+
+check_dockers_up_status
+

docker/script/sys-down.sh

+#!/bin/bash
+
+
+stop_docker() {
+
+  DOCKER_FILE=$1
+
+  echo -e "stopping docker ${DOCKER_FILE} ..."
+  
+  docker-compose -f $DOCKER_FILE down  
+}
+
+#####  main  #####
+
+
+cd /opt/mcx/config/docker/
+
+stop_docker mcx-frontend.yml
+stop_docker mcx-backend.yml
+stop_docker mcx-external.yml
+

docker/script/sys-up.sh

+#!/bin/bash
+
+start_all() {
+
+  echo -e "starting sys-up ..."
+  
+  # clean system resources
+  echo -e "cleaning resources ..."
+  docker system prune -f
+  echo -e "cleaning alerts ..."
+  rm -f alerts/*.txt
+  echo -e ""
+
+  # running mcx-external.yml
+  start_docker mcx-external.yml  
+  wait_for_up_status mcx-external.yml
+  wait_for_tag_status docker_elk_1 "ui settings" "to green - Ready"
+  echo -e ""
+
+  # running mcx-backend.yml
+  start_docker mcx-backend.yml  
+  wait_for_up_status mcx-backend.yml
+  echo -e ""
+
+  # running mcx-frontend.yml
+  start_docker mcx-frontend.yml  
+  wait_for_up_status mcx-frontend.yml
+
+  echo -e ""
+  echo -e "\e[32m*********************************\e[0m"
+  echo -e "\e[32m* SYSTEM STARTED SUCCESSFULLY ! *\e[0m"
+  echo -e "\e[32m*********************************\e[0m"
+}
+
+start_docker() {
+
+  DOCKER_FILE=$1
+
+  echo -e "starting ${DOCKER_FILE} ..."
+  
+  docker-compose -f $DOCKER_FILE up -d > /dev/null 2>&1 
+  RET=`echo $?`
+
+  # check return value
+  if [ "$RET" != "0" ]
+  then
+    echo -e "\e[31mERROR: start docker ${DOCKER_FILE} failed. aborted!\e[0m"
+    exit -1
+  fi
+
+}
+
+wait_for_up_status() {
+
+  DOCKER_FILE=$1
+
+  i="0"
+  while [ $i -lt 10 ]
+    do
+      check_up_status $DOCKER_FILE
+      if [ "$UP_STATUS" != "OK" ]
+      then
+        echo -e "checking ${DOCKER_FILE}(${NUM_OF_COMPONENTS} components) status ..."
+        sleep 3               
+      else        
+        echo -e "\e[32m${DOCKER_FILE} is up and running!\e[0m"
+        return
+      fi
+
+      i=$[$i+1]
+    done
+
+  echo -e "\e[31mERROR: start docker ${DOCKER_FILE} failed. aborted!\e[0m" 
+  exit -1
+}
+
+check_up_status() {
+
+  DOCKER_FILE=$1
+
+  ###docker-compose -f $DOCKER_FILE ps
+  
+  NUM_OF_COMPONENTS=`docker-compose -f $DOCKER_FILE ps | grep 'docker_' | wc -l`  
+  NUM_OF_UP_COMPONENTS=`docker-compose -f $DOCKER_FILE ps | grep 'docker_' | grep Up | wc -l`
+
+  if [ $NUM_OF_UP_COMPONENTS -eq $NUM_OF_COMPONENTS ];
+  then 
+    UP_STATUS="OK"
+  else
+    UP_STATUS="FAIL"
+  fi;
+  
+}
+
+wait_for_tag_status() {
+
+  DOCKER_ID=$1
+  TAG_NAME=$2
+  STATUS_DESC=$3
+
+  echo -e "waiting for ready status in ${DOCKER_ID} ..."
+
+  i="0"
+  while [ $i -lt 10 ]
+    do
+      check_tag_status $DOCKER_ID $TAG_NAME $STATUS_DESC
+      if [ "$TAG_STATUS" != "OK" ]
+      then
+        echo -e "checking status in ${DOCKER_ID} ..."
+        sleep 3               
+      else        
+        echo -e "\e[32m${DOCKER_ID} status is ok!\e[0m"
+        return
+      fi
+
+      i=$[$i+1]
+    done
+
+  echo -e "\e[31mERROR: start docker ${DOCKER_ID} failed. aborted!\e[0m" 
+  exit -1
+}
+
+check_tag_status() {
+
+  DOCKER_ID=$1
+  TAG_NAME=$2
+  STATUS_DESC=$3
+
+  TAG_RESULT=`docker logs $DOCKER_ID | head -n 1000 | grep $TAG_NAME | grep $STATUS_DESC`
+  if [ -z "$TAG_RESULT" ]; then
+    TAG_STATUS="FAIL"
+  else
+    TAG_STATUS="OK"
+  fi
+  
+}
+
+#####  main  #####
+
+
+cd /opt/mcx/config/docker/
+
+start_all
+