infosecn1nja
/
HELK
mirror of https://github.com/infosecn1nja/HELK.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #152 from devdua/improved-git-handling-update-script 

Added git checks for stable update experience
keyword-vs-text-changes
Roberto Rodriguez 2019-01-24 11:54:47 -05:00 committed by GitHub
parent fa1c3aa5f3 eb0e6faba6
commit 9b7d224661
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 105 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

168
docker/helk_update.sh
View File

@ -8,8 +8,13 @@
# Script Author: Dev Dua (@devdua) # Script Author: Dev Dua (@devdua)
# License: GPL-3.0 # License: GPL-3.0
RED='\033[0;31m'
CYAN='\033[0;36m'
WAR='\033[1;33m'
STD='\033[0m'
if [[ $EUID -ne 0 ]]; then if [[ $EUID -ne 0 ]]; then
echo "[HELK-UPDATE-INFO] YOU MUST BE ROOT TO RUN THIS SCRIPT!!!" echo -e "${CYAN}[HELK-UPDATE-INFO]${STD} YOU MUST BE ROOT TO RUN THIS SCRIPT!!!"
exit 1 exit 1
fi fi
@ -26,8 +31,7 @@ set_helk_subscription(){
basic) break;; basic) break;;
trial) break;; trial) break;;
*) *)
echo -e "${RED}Error...${STD}" echo -e "${RED}[HELK-UPDATE-ERROR]${STD} Not a valid subscription. Valid Options: basic or trial"
echo "[HELK-UPDATE-ERROR] Not a valid subscription. Valid Options: basic or trial"
;; ;;
esac esac
done done
@ -53,8 +57,7 @@ set_helk_build(){
1) HELK_BUILD='helk-kibana-analysis';break ;; 1) HELK_BUILD='helk-kibana-analysis';break ;;
2) HELK_BUILD='helk-kibana-notebook-analysis-';break;; 2) HELK_BUILD='helk-kibana-notebook-analysis-';break;;
*) *)
echo -e "${RED}Error...${STD}" echo -e "${RED}[HELK-UPDATE-ERROR]${STD} Not a valid build"
echo "[HELK-UPDATE-ERROR] Not a valid build"
;; ;;
esac esac
done done
@ -63,37 +66,61 @@ set_helk_build(){
check_min_requirements(){ check_min_requirements(){
systemKernel="$(uname -s)" systemKernel="$(uname -s)"
echo "[HELK-UPDATE-INFO] HELK being hosted on a $systemKernel box" echo -e "${CYAN}[HELK-UPDATE-INFO]${STD} HELK being hosted on a $systemKernel box"
if [ "$systemKernel" == "Linux" ]; then if [ "$systemKernel" == "Linux" ]; then
AVAILABLE_MEMORY=$(awk '/MemAvailable/{printf "%.f", $2/1024/1024}' /proc/meminfo) AVAILABLE_MEMORY=$(awk '/MemAvailable/{printf "%.f", $2/1024/1024}' /proc/meminfo)
if [ "${AVAILABLE_MEMORY}" -ge "11" ] ; then if [ "${AVAILABLE_MEMORY}" -ge "11" ] ; then
echo "[HELK-UPDATE-INFO] Available Memory (GB): ${AVAILABLE_MEMORY}" echo -e "${CYAN}[HELK-UPDATE-INFO]${STD} Available Memory (GB): ${AVAILABLE_MEMORY}"
else else
echo "[HELK-UPDATE-ERROR] YOU DO NOT HAVE ENOUGH AVAILABLE MEMORY" echo -e "${RED}[HELK-UPDATE-ERROR]${STD} YOU DO NOT HAVE ENOUGH AVAILABLE MEMORY"
echo "[HELK-UPDATE-ERROR] Available Memory (GB): ${AVAILABLE_MEMORY}" echo -e "${RED}[HELK-UPDATE-ERROR]${STD} Available Memory (GB): ${AVAILABLE_MEMORY}"
echo "[HELK-UPDATE-ERROR] Check the requirements section in our UPDATE Wiki" echo -e "${RED}[HELK-UPDATE-ERROR]${STD} Check the requirements section in our UPDATE Wiki"
echo "[HELK-UPDATE-ERROR] UPDATE Wiki: https://github.com/Cyb3rWard0g/HELK/wiki/UPDATE" echo -e "${RED}[HELK-UPDATE-ERROR]${STD} UPDATE Wiki: https://github.com/Cyb3rWard0g/HELK/wiki/UPDATE"
exit 1 exit 1
fi fi
else else
echo "[HELK-UPDATE-INFO] Error retrieving memory info for $systemKernel. Make sure you have at least 11GB of available memory!" echo -e "${CYAN}[HELK-UPDATE-INFO]${STD} Error retrieving memory info for $systemKernel. Make sure you have at least 11GB of available memory!"
fi fi
# CHECK DOCKER DIRECTORY SPACE # CHECK DOCKER DIRECTORY SPACE
echo "[HELK-UPDATE-INFO] Making sure you assigned enough disk space to the current Docker base directory" echo -e "${CYAN}[HELK-UPDATE-INFO]${STD} Making sure you assigned enough disk space to the current Docker base directory"
AVAILABLE_DOCKER_DISK=$(df -m $(docker info --format '{{.DockerRootDir}}') | awk '$1 ~ /\//{printf "%.f\t\t", $4 / 1024}') AVAILABLE_DOCKER_DISK=$(df -m $(docker info --format '{{.DockerRootDir}}') | awk '$1 ~ /\//{printf "%.f\t\t", $4 / 1024}')
if [ "${AVAILABLE_DOCKER_DISK}" -ge "25" ]; then if [ "${AVAILABLE_DOCKER_DISK}" -ge "25" ]; then
echo "[HELK-UPDATE-INFO] Available Docker Disk: $AVAILABLE_DOCKER_DISK" echo -e "${CYAN}[HELK-UPDATE-INFO]${STD} Available Docker Disk: $AVAILABLE_DOCKER_DISK"
else else
echo "[HELK-UPDATE-ERROR] YOU DO NOT HAVE ENOUGH DOCKER DISK SPACE ASSIGNED" echo -e "${RED}[HELK-UPDATE-ERROR]${STD} YOU DO NOT HAVE ENOUGH DOCKER DISK SPACE ASSIGNED"
echo "[HELK-UPDATE-ERROR] Available Docker Disk: $AVAILABLE_DOCKER_DISK" echo -e "${RED}[HELK-UPDATE-ERROR]${STD} Available Docker Disk: $AVAILABLE_DOCKER_DISK"
echo "[HELK-UPDATE-ERROR] Check the requirements section in our UPDATE Wiki" echo -e "${RED}[HELK-UPDATE-ERROR]${STD} Check the requirements section in our UPDATE Wiki"
echo "[HELK-UPDATE-ERROR] UPDATE Wiki: https://github.com/Cyb3rWard0g/HELK/wiki/UPDATE" echo -e "${RED}[HELK-UPDATE-ERROR]${STD} UPDATE Wiki: https://github.com/Cyb3rWard0g/HELK/wiki/UPDATE"
exit 1 exit 1
fi fi
} }
check_git_status(){
GIT_STATUS=$(git status 2>&1)
RETURN_CODE=$?
echo -e "Git status: $GIT_STATUS_FATAL, RetVal : $RETURN_CODE" >> $LOGFILE
if [[ -z $GIT_STATUS_FATAL && $RETURN_CODE -gt 0 ]]; then
echo -e "${WAR}[HELK-UPDATE-WARNING]${STD} Git repository corrupted."
read -p "To fix this, all your local modifications to HELK will be overwritten. Do you wish to continue? (y/n) " -n 1 -r
echo
if [[ $REPLY =~ ^[Yy]$ ]]; then
GIT_REPO_CLEAN=0
echo -e "${CYAN}[HELK-UPDATE-INFO]${STD} HELK will now be refreshed. Re-initializing .git..."
cd ..
git init >> $LOGFILE
cd docker
echo -e "${CYAN}[HELK-UPDATE-INFO]${STD} Git repository fixed. Fetching latest version of HELK..."
else
exit
fi
else
echo -e "${CYAN}[HELK-UPDATE-INFO]${STD} Sanity check passed."
fi
}
check_github(){ check_github(){
if [ -x "$(command -v git)" ]; then if [ -x "$(command -v git)" ]; then
echo -e "Git is available" >> $LOGFILE echo -e "Git is available" >> $LOGFILE
else else
@ -101,50 +128,61 @@ check_github(){
apt-get -qq update >> $LOGFILE 2>&1 && apt-get -qqy install git-core >> $LOGFILE 2>&1 apt-get -qq update >> $LOGFILE 2>&1 && apt-get -qqy install git-core >> $LOGFILE 2>&1
ERROR=$? ERROR=$?
if [ $ERROR -ne 0 ]; then if [ $ERROR -ne 0 ]; then
"[!] Could not install Git (Error Code: $ERROR). Check $LOGFILE for details." echo -e "${RED}[!]${STD} Could not install Git (Error Code: $ERROR). Check $LOGFILE for details."
exit 1 exit 1
fi fi
echo "Git successfully installed." >> $LOGFILE echo "Git successfully installed." >> $LOGFILE
fi fi
if [[ -z "$(git remote | grep helk-repo)" ]]; then echo -e "${CYAN}[HELK-UPDATE-INFO]${STD} Sanity check..."
git remote add helk-repo https://github.com/Cyb3rWard0g/HELK.git >> $LOGFILE 2>&1 check_git_status
else
echo "HELK repo exists" >> $LOGFILE 2>&1
fi
git remote update >> $LOGFILE 2>&1
COMMIT_DIFF=$(git rev-list --count master...helk-repo/master)
CURRENT_COMMIT=$(git rev-parse HEAD)
REMOTE_LATEST_COMMIT=$(git rev-parse helk-repo/master)
echo "HEAD commits --> Current: $CURRENT_COMMIT | Remote: $REMOTE_LATEST_COMMIT" >> $LOGFILE 2>&1
if [ ! "$COMMIT_DIFF" == "0" ]; then
echo "Possibly new release available. Commit diff --> $COMMIT_DIFF" >> $LOGFILE 2>&1
IS_MASTER_BEHIND=$(git branch -v | grep master | grep behind)
# IF HELK HAS BEEN CLONED FROM OFFICIAL REPO
if [ ! "$CURRENT_COMMIT" == "$REMOTE_LATEST_COMMIT" ]; then
echo "Difference in HEAD commits --> Current: $CURRENT_COMMIT | Remote: $REMOTE_LATEST_COMMIT" >> $LOGFILE 2>&1
echo "[HELK-UPDATE-INFO] New release available. Pulling new code."
git checkout master >> $LOGFILE 2>&1
git pull helk-repo master >> $LOGFILE 2>&1
REBUILD_NEEDED=1
touch /tmp/helk-update
echo $REBUILD_NEEDED > /tmp/helk-update
# IF HELK HAS BEEN CLONED FROM THE OFFICIAL REPO & MODIFIED
elif [[ -z $IS_MASTER_BEHIND ]]; then
echo "Current master branch ahead of remote branch, possibly modified. Exiting..." >> $LOGFILE 2>&1
echo "[HELK-UPDATE-INFO] No updates available."
if [ $GIT_REPO_CLEAN == 1 ]; then
if [[ -z "$(git remote | grep helk-repo)" ]]; then
git remote add helk-repo https://github.com/Cyb3rWard0g/HELK.git >> $LOGFILE 2>&1
else else
echo "Repository misconfigured. Exiting..." >> $LOGFILE 2>&1 echo "HELK repo exists" >> $LOGFILE 2>&1
echo "[HELK-UPDATE-INFO] No updates available." fi
git remote update >> $LOGFILE 2>&1
COMMIT_DIFF=$(git rev-list --count master...helk-repo/master 2>&1)
CURRENT_COMMIT=$(git rev-parse HEAD 2>&1)
REMOTE_LATEST_COMMIT=$(git rev-parse helk-repo/master 2>&1)
echo "HEAD commits --> Current: $CURRENT_COMMIT | Remote: $REMOTE_LATEST_COMMIT" >> $LOGFILE 2>&1
if [ ! "$COMMIT_DIFF" == "0" ]; then
echo "Possibly new release available. Commit diff --> $COMMIT_DIFF" >> $LOGFILE 2>&1
IS_MASTER_BEHIND=$(git branch -v | grep master | grep behind)
# IF HELK HAS BEEN CLONED FROM OFFICIAL REPO
if [ ! "$CURRENT_COMMIT" == "$REMOTE_LATEST_COMMIT" ]; then
echo "Difference in HEAD commits --> Current: $CURRENT_COMMIT | Remote: $REMOTE_LATEST_COMMIT" >> $LOGFILE 2>&1
echo -e "${CYAN}[HELK-UPDATE-INFO]${STD} New release available. Pulling new code."
git checkout master >> $LOGFILE 2>&1
git clean -d -fx . >> $LOGFILE 2>&1
git pull helk-repo master >> $LOGFILE 2>&1
REBUILD_NEEDED=1
touch /tmp/helk-update
echo $REBUILD_NEEDED > /tmp/helk-update
# IF HELK HAS BEEN CLONED FROM THE OFFICIAL REPO & MODIFIED
elif [[ -z $IS_MASTER_BEHIND ]]; then
echo "Current master branch ahead of remote branch, possibly modified. Exiting..." >> $LOGFILE 2>&1
echo -e "${CYAN}[HELK-UPDATE-INFO]${STD} No updates available."
else
echo "Repository misconfigured. Exiting..." >> $LOGFILE 2>&1
echo -e "${CYAN}[HELK-UPDATE-INFO]${STD} No updates available."
fi
else
echo -e "${CYAN}[HELK-UPDATE-INFO]${STD} No updates available."
fi fi
else else
echo "[HELK-UPDATE-INFO] No updates available." cd ..
git clean -d -fx . >> $LOGFILE 2>&1
git remote add helk-repo https://github.com/Cyb3rWard0g/HELK.git >> $LOGFILE 2>&1
git pull helk-repo master >> $LOGFILE 2>&1
fi fi
} }
@ -153,42 +191,43 @@ update_helk() {
set_helk_build set_helk_build
set_helk_subscription set_helk_subscription
echo -e "[HELK-UPDATE-INFO] Stopping HELK and starting update" echo -e "${CYAN}[HELK-UPDATE-INFO]${STD} Stopping HELK and starting update"
COMPOSE_CONFIG="${HELK_BUILD}-${SUBSCRIPTION_CHOICE}.yml" COMPOSE_CONFIG="${HELK_BUILD}-${SUBSCRIPTION_CHOICE}.yml"
## ****** Setting KAFKA ADVERTISED_LISTENER environment variable *********** ## ****** Setting KAFKA ADVERTISED_LISTENER environment variable ***********
export ADVERTISED_LISTENER=$HOST_IP export ADVERTISED_LISTENER=$HOST_IP
echo "[HELK-UPDATE-INFO] Building & running HELK from $COMPOSE_CONFIG file.." echo -e "${CYAN}[HELK-UPDATE-INFO]${STD} Building & running HELK from $COMPOSE_CONFIG file.."
docker-compose -f $COMPOSE_CONFIG down >> $LOGFILE 2>&1 docker-compose -f $COMPOSE_CONFIG down >> $LOGFILE 2>&1
ERROR=$? ERROR=$?
if [ $ERROR -ne 0 ]; then if [ $ERROR -ne 0 ]; then
echo -e "[!] Could not stop HELK via docker-compose (Error Code: $ERROR). You're possibly running a different HELK license than chosen - $license_choice" echo -e "${RED}[!]${STD} Could not stop HELK via docker-compose (Error Code: $ERROR). You're possibly running a different HELK license than chosen - $license_choice"
exit 1 exit 1
fi fi
check_min_requirements check_min_requirements
echo "[HELK-UPDATE-INFO] Rebuilding HELK via docker-compose" echo -e "${CYAN}[HELK-UPDATE-INFO]${STD} Rebuilding HELK via docker-compose"
docker-compose -f $COMPOSE_CONFIG up --build -d -V --force-recreate --always-recreate-deps >> $LOGFILE 2>&1 docker-compose -f $COMPOSE_CONFIG up --build -d -V --force-recreate --always-recreate-deps >> $LOGFILE 2>&1
ERROR=$? ERROR=$?
if [ $ERROR -ne 0 ]; then if [ $ERROR -ne 0 ]; then
echo -e "[!] Could not run HELK via docker-compose (Error Code: $ERROR). Check $LOGFILE for details." echo -e "${RED}[!]${STD} Could not run HELK via docker-compose (Error Code: $ERROR). Check $LOGFILE for details."
exit 1 exit 1
fi fi
secs=$((3 * 60)) secs=$((3 * 60))
while [ $secs -gt 0 ]; do while [ $secs -gt 0 ]; do
echo -ne "\033[0K\r[HELK-UPDATE-INFO] Rebuild succeeded, waiting $secs seconds for services to start..." echo -ne "\033[0K\r${CYAN}[HELK-UPDATE-INFO]${STD} Rebuild succeeded, waiting $secs seconds for services to start..."
sleep 1 sleep 1
: $((secs--)) : $((secs--))
done done
echo -e "\n[HELK-UPDATE-INFO] YOUR HELK HAS BEEN UPDATED!" echo -e "\n${CYAN}[HELK-UPDATE-INFO]${STD} YOUR HELK HAS BEEN UPDATED!"
echo 0 > /tmp/helk-update echo 0 > /tmp/helk-update
exit 1 exit 1
} }
LOGFILE="/var/log/helk-update.log" LOGFILE="/var/log/helk-update.log"
REBUILD_NEEDED=0 REBUILD_NEEDED=0
GIT_REPO_CLEAN=1
if [[ -e /tmp/helk-update ]]; then if [[ -e /tmp/helk-update ]]; then
UPDATES_FETCHED=`cat /tmp/helk-update` UPDATES_FETCHED=`cat /tmp/helk-update`
@ -199,12 +238,15 @@ if [[ -e /tmp/helk-update ]]; then
fi fi
fi fi
echo "[HELK-UPDATE-INFO] Checking GitHub for updates..." echo -e "${CYAN}[HELK-UPDATE-INFO]${STD} Checking GitHub for updates..."
check_github check_github
if [ $REBUILD_NEEDED == 1 ]; then if [ $REBUILD_NEEDED == 1 ]; then
update_helk update_helk
else elif [ $GIT_REPO_CLEAN == 0 ]; then
echo -e "[HELK-UPDATE-INFO] YOUR HELK IS ALREADY UP-TO-DATE." echo -e "${CYAN}[HELK-UPDATE-INFO]${STD} HELK repository refreshed, please terminate this shell & run the update script again."
exit 1 exit 1
fi else
echo -e "${CYAN}[HELK-UPDATE-INFO]${STD} YOUR HELK IS ALREADY UP-TO-DATE."
exit 1
fi