From f8dfb708a8e1b6eeecdc1e50dab82918792b2b95 Mon Sep 17 00:00:00 2001 From: Pete Nelson Date: Thu, 8 Nov 2018 19:36:55 +0000 Subject: [PATCH 01/28] add a script for purging deployed packages --- purge_capes.sh | 75 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 purge_capes.sh diff --git a/purge_capes.sh b/purge_capes.sh new file mode 100644 index 0000000..6a577d9 --- /dev/null +++ b/purge_capes.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +# warn user +read -p " +WARNING: pressing enter now will attempt to remove all customization + performed by the deploy script. Press Ctrl-C now to abort." VAR + +# stop all the services +sudo systemctl stop mattermost.service +sudo systemctl stop filebeat.service +sudo systemctl stop metricbeat.service +sudo systemctl stop heartbeat.service +sudo systemctl stop nginx.service +sudo systemctl stop murmur.service +sudo systemctl stop thehive.service +sudo systemctl stop hackmd.service +sudo systemctl stop gitea.service +sudo systemctl stop cortex.service +sudo systemctl stop kibana.service +sudo systemctl stop elasticsearch.service + +# disable all the services +sudo systemctl disable mattermost.service +sudo systemctl disable filebeat.service +sudo systemctl disable metricbeat.service +sudo systemctl disable heartbeat.service +sudo systemctl disable nginx.service +sudo systemctl disable murmur.service +sudo systemctl disable thehive.service +sudo systemctl disable hackmd.service +sudo systemctl disable gitea.service +sudo systemctl disable cortex.service +sudo systemctl disable kibana.service +sudo systemctl disable elasticsearch.service + +# TODO: need to rewrite conf file with a D or R? +#sudo systemd-tmpfiles --create /etc/tmpfiles.d/murmur.conf + +#sudo firewall-cmd --remove-port=80/tcp --remove-port=3000/tcp --remove-port=4000/tcp --remove-port=5000/tcp --remove-port=5601/tcp --remove-port=9000/tcp --remove-port=9001/tcp --remove-port=7000/tcp --remove-port=7000/udp --permanent +#sudo firewall-cmd --reload + +sudo yum erase -y kibana-5.6.5 metricbeat-5.6.5 filebeat-5.6.5 heartbeat-5.6.5 +sudo rm -f -r /etc/filebeat/ /etc/heartbeat/ /etc/metricbeat/ /etc/kibana/ + +sudo rm -f /usr/share/nginx/html/cyberchef.htm + +sudo yum erase -y nginx httpd-tools +sudo rm -f -r /usr/share/nginx/html/* +sudo rm -f -r /etc/nginx/ + +sudo yum erase -y thehive cortex +sudo rm -f /etc/thehive/application.conf* /etc/cortex/application.conf* +sudo rmdir /etc/cortex/ /etc/thehive/ /opt/cortex/* /opt/cortex/ /opt/thehive/* /opt/thehive/ + +sudo yum erase -y elasticsearch-5.6.0 +sudo rm -f /etc/elasticsearch/elasticsearch.yml* + +sudo rm -f /etc/systemd/system/gitea.service* +sudo rm -f -r /opt/gitea +mysql -uroot -e "DROP DATABASE gitea;" +sudo yum erase -y wandisco-git-release-7-2 + +mysql -uroot -e "DROP DATABASE hackmd;" +sudo rm -f /etc/systemd/system/hackmd.service* +sudo rm -f -r /opt/hackmd + +mysql -uroot -e "DROP DATABASE mattermost;" +sudo rm -f /etc/systemd/system/mattermost.service* +sudo rmdir /opt/mattermost/data /opt/mattermost/ + +sudo rm -f /etc/tmpfiles.d/murmur.conf* /etc/systemd/system/murmur.service* /etc/logrotate.d/murmur* /etc/murmur.ini* +sudo rm -f -r /opt/murmur /var/log/murmur + +echo MariaDB has not been uninstalled. +echo dependencies of the main packages have not been uninstalled. From cdc0204ffe972dad2ff0c6e23501e3129611fce8 Mon Sep 17 00:00:00 2001 From: Pete Nelson Date: Thu, 8 Nov 2018 20:11:44 +0000 Subject: [PATCH 02/28] simplify yum.conf gpgcheck update --- deploy_capes.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/deploy_capes.sh b/deploy_capes.sh index dccad88..ac03d06 100644 --- a/deploy_capes.sh +++ b/deploy_capes.sh @@ -8,8 +8,7 @@ # To fix this, we are going to disable the GPG signature and local RPM GPG signature checking. # I'm open to other options here. # RHEL's official statement on this: https://access.redhat.com/solutions/2850911 -sudo sed -i 's/repo_gpgcheck=1/repo_gpgcheck=0/' /etc/yum.conf -sudo sed -i 's/localpkg_gpgcheck=1/localpkg_gpgcheck=0/' /etc/yum.conf +sudo sed -i 's/gpgcheck=1/gpgcheck=0/' /etc/yum.conf ################################ ##### Collect Credentials ###### From 3fb0537c67bf3cd5d10785dd00e1d88f37e5774a Mon Sep 17 00:00:00 2001 From: Pete Nelson Date: Thu, 8 Nov 2018 20:13:15 +0000 Subject: [PATCH 03/28] remove silence on password reads since we will print out later anyway --- deploy_capes.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/deploy_capes.sh b/deploy_capes.sh index ac03d06..2647f02 100644 --- a/deploy_capes.sh +++ b/deploy_capes.sh @@ -17,23 +17,23 @@ sudo sed -i 's/gpgcheck=1/gpgcheck=0/' /etc/yum.conf clear # Create your Gitea passphrase echo "Create your Gitea passphrase for the MySQL database and press [Enter]. You will create your Gitea administration credentials after the installation." -read -s giteapassphrase +read giteapassphrase # Create your HackMD passphrase echo "Create your HackMD passphrase for the MySQL database and press [Enter]. You will create your specific HackMD credentials after the installation." -read -s hackmdpassphrase +read hackmdpassphrase # Create your Mattermost passphrase echo "Create your Mattermost passphrase for the MySQL database and press [Enter]. You will create your Mattermost administration credentials after the installation." -read -s mattermostpassphrase +read mattermostpassphrase # Create your Mumble passphrase echo "Create your Mumble SuperUser passphrase and press [Enter]." -read -s mumblepassphrase +read mumblepassphrase # Create your CAPES Landing Page passphrase echo "Create your CAPES Landing Page passphrase for the account \"operator\" and press [Enter]." -read -s capespassphrase +read capespassphrase # Set your IP address as a variable. This is for instructions below. IP="$(hostname -I | sed -e 's/[[:space:]]*$//')" From 2c2a90650703b81988998c62b11a3a968164b28e Mon Sep 17 00:00:00 2001 From: Pete Nelson Date: Thu, 8 Nov 2018 20:15:36 +0000 Subject: [PATCH 04/28] check before adding hosts entry (idempotent) --- deploy_capes.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy_capes.sh b/deploy_capes.sh index 2647f02..95d31f1 100644 --- a/deploy_capes.sh +++ b/deploy_capes.sh @@ -36,10 +36,10 @@ echo "Create your CAPES Landing Page passphrase for the account \"operator\" and read capespassphrase # Set your IP address as a variable. This is for instructions below. -IP="$(hostname -I | sed -e 's/[[:space:]]*$//')" +IP="$(hostname -I | sed -e 's/ .*$//')" # Update your Host file -echo "$IP $HOSTNAME" | sudo tee -a /etc/hosts +grep "$IP" /etc/hosts || echo "$IP $HOSTNAME" | sudo tee -a /etc/hosts ################################ ######## Configure NTP ######### From 13956df99b5346e93ddc9f7b63309214261ef1cb Mon Sep 17 00:00:00 2001 From: Pete Nelson Date: Thu, 8 Nov 2018 20:56:07 +0000 Subject: [PATCH 05/28] check for ntpd or chrony config before installing (idempotent) --- deploy_capes.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/deploy_capes.sh b/deploy_capes.sh index 95d31f1..a199730 100644 --- a/deploy_capes.sh +++ b/deploy_capes.sh @@ -50,6 +50,8 @@ grep "$IP" /etc/hosts || echo "$IP $HOSTNAME" | sudo tee -a /etc/hosts sudo timedatectl set-timezone UTC # Set NTP. If you have already set your NTP in accordance with your local standards, you may comment this out. +if [ ! -e /etc/chrony.conf -a ! -e /etc/ntp.conf ]; then + sudo bash -c 'cat > /etc/chrony.conf < Date: Thu, 8 Nov 2018 20:57:58 +0000 Subject: [PATCH 06/28] global yum update is a maint task, not install --- deploy_capes.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy_capes.sh b/deploy_capes.sh index a199730..f1d84cc 100644 --- a/deploy_capes.sh +++ b/deploy_capes.sh @@ -110,7 +110,7 @@ fi ################################ # Prepare the environment -sudo yum -y install bzip2 && sudo yum -y update +sudo yum -y install bzip2 sudo groupadd -r murmur sudo useradd -r -g murmur -m -d /var/lib/murmur -s /sbin/nologin murmur sudo mkdir -p /var/log/murmur From db6a579162fe66bdb9dffae17641b4d7376bd31e Mon Sep 17 00:00:00 2001 From: Pete Nelson Date: Thu, 8 Nov 2018 20:59:38 +0000 Subject: [PATCH 07/28] use tee for all root file writes rather than subshell --- deploy_capes.sh | 42 ++++++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/deploy_capes.sh b/deploy_capes.sh index f1d84cc..736e616 100644 --- a/deploy_capes.sh +++ b/deploy_capes.sh @@ -52,7 +52,7 @@ sudo timedatectl set-timezone UTC # Set NTP. If you have already set your NTP in accordance with your local standards, you may comment this out. if [ ! -e /etc/chrony.conf -a ! -e /etc/ntp.conf ]; then -sudo bash -c 'cat > /etc/chrony.conf < /dev/null # Use public servers from the pool.ntp.org project. # Please consider joining the pool (http://www.pool.ntp.org/join.html). server 0.centos.pool.ntp.org iburst @@ -99,7 +99,7 @@ logchange 0.5 logdir /var/log/chrony #log measurements statistics tracking -EOF' +EOF sudo systemctl enable chronyd.service sudo systemctl start chronyd.service @@ -133,7 +133,7 @@ sudo sed -i 's/\#registerName=Mumble\ Server/registerName=CAPES\ -\ Mumble\ Serv sudo sed -i 's/port=64738/port=7000/' /etc/murmur.ini # Rotate logs -sudo bash -c 'cat > /etc/logrotate.d/murmur < /dev/null /var/log/murmur/*log { su murmur murmur dateext @@ -146,10 +146,10 @@ sudo bash -c 'cat > /etc/logrotate.d/murmur < /dev/null 2>/dev/null || true endscript } -EOF' +EOF # Creating the systemd service -sudo bash -c 'cat > /etc/systemd/system/murmur.service < /dev/null [Unit] Description=Mumble Server (Murmur) Requires=network-online.target @@ -164,12 +164,12 @@ ExecReload=/bin/kill -s HUP $MAINPID [Install] WantedBy=multi-user.target -EOF' +EOF # Generate the pid directory for Murmur: -sudo bash -c 'cat > /etc/tmpfiles.d/murmur.conf < /dev/null d /var/run/murmur 775 murmur murmur -EOF' +EOF ################################ ########## Mattermost ########## @@ -211,7 +211,7 @@ cd - mysql -u root -e "ALTER TABLE mattermost.Audits ENGINE = MyISAM;ALTER TABLE mattermost.ChannelMembers ENGINE = MyISAM;ALTER TABLE mattermost.Channels ENGINE = MyISAM;ALTER TABLE mattermost.ClusterDiscovery ENGINE = MyISAM;ALTER TABLE mattermost.Commands ENGINE = MyISAM;ALTER TABLE mattermost.CommandWebhooks ENGINE = MyISAM;ALTER TABLE mattermost.Compliances ENGINE = MyISAM;ALTER TABLE mattermost.Emoji ENGINE = MyISAM;ALTER TABLE mattermost.FileInfo ENGINE = MyISAM;ALTER TABLE mattermost.IncomingWebhooks ENGINE = MyISAM;ALTER TABLE mattermost.Jobs ENGINE = MyISAM;ALTER TABLE mattermost.Licenses ENGINE = MyISAM;ALTER TABLE mattermost.OAuthAccessData ENGINE = MyISAM;ALTER TABLE mattermost.OAuthApps ENGINE = MyISAM;ALTER TABLE mattermost.OAuthAuthData ENGINE = MyISAM;ALTER TABLE mattermost.OutgoingWebhooks ENGINE = MyISAM;ALTER TABLE mattermost.Posts ENGINE = MyISAM;ALTER TABLE mattermost.Preferences ENGINE = MyISAM;ALTER TABLE mattermost.Reactions ENGINE = MyISAM;ALTER TABLE mattermost.Sessions ENGINE = MyISAM;ALTER TABLE mattermost.Status ENGINE = MyISAM;ALTER TABLE mattermost.Systems ENGINE = MyISAM;ALTER TABLE mattermost.TeamMembers ENGINE = MyISAM;ALTER TABLE mattermost.Teams ENGINE = MyISAM;ALTER TABLE mattermost.Tokens ENGINE = MyISAM;ALTER TABLE mattermost.UserAccessTokens ENGINE = MyISAM;ALTER TABLE mattermost.Users ENGINE = MyISAM;" # Create the Mattermost service -sudo bash -c 'cat > /etc/systemd/system/mattermost.service < /dev/null [Unit] Description=Mattermost After=syslog.target network.target mariadb.service @@ -227,7 +227,7 @@ LimitNOFILE=49152 [Install] WantedBy=multi-user.target -EOF' +EOF sudo chmod 664 /etc/systemd/system/mattermost.service ################################ @@ -266,7 +266,7 @@ sudo useradd -s /usr/sbin/nologin hackmd sudo chown -R hackmd:hackmd /opt/hackmd # Creating the HackMD service -sudo bash -c 'cat > /etc/systemd/system/hackmd.service < /dev/null [Unit] Description=HackMD Service Requires=network-online.target @@ -281,7 +281,7 @@ ExecStart=/bin/npm start production --prefix /opt/hackmd/ [Install] WantedBy=multi-user.target -EOF' +EOF ################################ ########## Gitea ############### @@ -322,7 +322,7 @@ sudo chown -R gitea:gitea /opt/gitea sudo chmod 744 /opt/gitea/gitea # Create the Gitea service -sudo bash -c 'cat > /etc/systemd/system/gitea.service < /dev/null [Unit] Description=Gitea (Git with a cup of tea) After=syslog.target @@ -347,7 +347,7 @@ Environment=USER=gitea HOME=/home/gitea [Install] WantedBy=multi-user.target -EOF' +EOF ################################ ########### TheHive ############ @@ -361,14 +361,14 @@ sudo yum install https://artifacts.elastic.co/downloads/elasticsearch/elasticsea sudo yum install python36u python36u-pip python36u-devel -y # Configure Elasticsearch -sudo bash -c 'cat > /etc/elasticsearch/elasticsearch.yml < /dev/null network.host: 127.0.0.1 cluster.name: hive script.inline: true thread_pool.index.queue_size: 100000 thread_pool.search.queue_size: 100000 thread_pool.bulk.queue_size: 1000 -EOF' +EOF # Collect the Cortex analyzers sudo git clone https://github.com/TheHive-Project/Cortex-Analyzers.git /opt/cortex/ @@ -382,24 +382,22 @@ sudo yum install https://dl.bintray.com/thehive-project/rpm-stable/thehive-proje sudo yum install thehive cortex -y # Configure TheHive Project secret key -(cat << _EOF_ +sudo tee -a /etc/thehive/application.conf << _EOF_ > /dev/null # Secret key # ~~~~~ # The secret key is used to secure cryptographics functions. # If you deploy your application to several instances be sure to use the same key! play.crypto.secret="$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 64 | head -n 1)" _EOF_ -) | sudo tee -a /etc/thehive/application.conf # Configure Cortex secret key -(cat << _EOF_ +sudo tee -a /etc/cortex/application.conf << _EOF_ > /dev/null # Secret key # ~~~~~ # The secret key is used to secure cryptographics functions. # If you deploy your application to several instances be sure to use the same key! play.crypto.secret="$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 64 | head -n 1)" _EOF_ -) | sudo tee -a /etc/cortex/application.conf # Add the future Python package, install the Cortex Analyzers, and adjust the Python 3 path to 3.6 sudo pip install future @@ -442,7 +440,7 @@ sudo chmod 640 /etc/cortex/application.conf sudo sed -i '16i\\t-Dhttp.port=9001 \\' /etc/systemd/system/cortex.service # Connect TheHive to Cortex -sudo bash -c 'cat >> /etc/thehive/application.conf < /dev/null # Cortex play.modules.enabled += connectors.cortex.CortexConnector cortex { @@ -451,7 +449,7 @@ cortex { key = "Cortex-API-key-see-post-installation-instructions" } } -EOF' +EOF ################################ ############ Nginx ############# From 8fb6c573084249d556445e3b5896fe615053e1b3 Mon Sep 17 00:00:00 2001 From: Pete Nelson Date: Thu, 8 Nov 2018 21:00:24 +0000 Subject: [PATCH 08/28] capture proxy from yum (prepwork) --- deploy_capes.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/deploy_capes.sh b/deploy_capes.sh index 736e616..358bc1f 100644 --- a/deploy_capes.sh +++ b/deploy_capes.sh @@ -41,6 +41,17 @@ IP="$(hostname -I | sed -e 's/ .*$//')" # Update your Host file grep "$IP" /etc/hosts || echo "$IP $HOSTNAME" | sudo tee -a /etc/hosts +################################ +######### Proxy detect ######### +################################ + +PROXY=$(sed -n '/^proxy=/s/.*=//p' < /etc/yum.conf) +echo PROXY=$PROXY +if [ -n "$PROXY" ]; then + PIP_PROXY="--proxy $(echo $PROXY | sed 's/.*\/\///')" + CURL_PROXY="--proxy $PROXY" +fi + ################################ ######## Configure NTP ######### ################################ From ba28995f41da633cb3578d824028e7f1a243ec71 Mon Sep 17 00:00:00 2001 From: Pete Nelson Date: Thu, 8 Nov 2018 21:06:11 +0000 Subject: [PATCH 09/28] move firewalld install down by its config (prepwork) --- deploy_capes.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deploy_capes.sh b/deploy_capes.sh index 358bc1f..74d755e 100644 --- a/deploy_capes.sh +++ b/deploy_capes.sh @@ -187,7 +187,7 @@ EOF ################################ # Install dependencies -sudo yum install epel-release mariadb-server firewalld -y +sudo yum install epel-release mariadb-server -y # Configure MariaDB sudo systemctl start mariadb.service @@ -525,6 +525,7 @@ sudo sed -i "s/#server\.host: \"localhost\"/server\.host: \"0\.0\.0\.0\"/" /etc/ ########## Firewall ############ ################################ +sudo yum install firewalld -y # Port 80 - Nginx # Port 3000 - HackMD # Port 4000 - Gitea From 77da1fd340a3c43854b842452d2f81e3786df958 Mon Sep 17 00:00:00 2001 From: Pete Nelson Date: Thu, 8 Nov 2018 21:40:58 +0000 Subject: [PATCH 10/28] reload daemons after installing chrony --- deploy_capes.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/deploy_capes.sh b/deploy_capes.sh index 74d755e..ac71e0e 100644 --- a/deploy_capes.sh +++ b/deploy_capes.sh @@ -111,6 +111,7 @@ logchange 0.5 logdir /var/log/chrony #log measurements statistics tracking EOF +sudo systemctl daemon-reload sudo systemctl enable chronyd.service sudo systemctl start chronyd.service From 64593b364824b8e12550965e05429d2511df2274 Mon Sep 17 00:00:00 2001 From: Pete Nelson Date: Thu, 8 Nov 2018 21:41:40 +0000 Subject: [PATCH 11/28] make murmur install idempotent --- deploy_capes.sh | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/deploy_capes.sh b/deploy_capes.sh index ac71e0e..e5e4cbf 100644 --- a/deploy_capes.sh +++ b/deploy_capes.sh @@ -121,6 +121,10 @@ fi ########### Mumble ############# ################################ +if systemctl is-active murmur.service; then + echo "Mumble is active" +else + # Prepare the environment sudo yum -y install bzip2 sudo groupadd -r murmur @@ -130,8 +134,8 @@ sudo chown murmur:murmur /var/log/murmur sudo chmod 0770 /var/log/murmur # Download binaries -curl -OL https://github.com/mumble-voip/mumble/releases/download/1.2.19/murmur-static_x86-1.2.19.tar.bz2 -tar xjf murmur-static_x86-1.2.19.tar.bz2 +curl -OL4 $CURL_PROXY https://github.com/mumble-voip/mumble/releases/download/1.2.19/murmur-static_x86-1.2.19.tar.bz2 +tar xjf murmur-static_x86-1.2.19.tar.bz2 || exit sudo mkdir -p /opt/murmur sudo cp -r murmur-static_x86-1.2.19/* /opt/murmur sudo cp murmur-static_x86-1.2.19/murmur.ini /etc/murmur.ini @@ -183,6 +187,14 @@ sudo tee /etc/tmpfiles.d/murmur.conf << EOF > /dev/null d /var/run/murmur 775 murmur murmur EOF +sudo systemctl daemon-reload +sudo systemctl start murmur.service && sudo systemctl enable murmur.service + +# Configure the Murmur SuperUser account +sudo /opt/murmur/murmur.x86 -ini /etc/murmur.ini -supw $mumblepassphrase + +fi + ################################ ########## Mattermost ########## ################################ @@ -559,7 +571,6 @@ sudo systemctl enable mattermost.service sudo systemctl enable elasticsearch.service sudo systemctl enable thehive.service sudo systemctl enable cortex.service -sudo systemctl enable murmur.service # Start all the services sudo systemctl start elasticsearch.service @@ -568,16 +579,12 @@ sudo systemctl start cortex.service sudo systemctl start gitea.service sudo systemctl start hackmd.service sudo systemctl start thehive.service -sudo systemctl start murmur.service sudo systemctl start nginx.service sudo systemctl start heartbeat.service sudo systemctl start metricbeat.service sudo systemctl start filebeat.service sudo systemctl start mattermost.service -# Configure the Murmur SuperUser account -sudo /opt/murmur/murmur.x86 -ini /etc/murmur.ini -supw $mumblepassphrase - ################################ ### Secure MySQL installtion ### ################################ From 61c1611082b38f1a5b6cd96b2f1c18f70378a572 Mon Sep 17 00:00:00 2001 From: Pete Nelson Date: Thu, 8 Nov 2018 21:48:13 +0000 Subject: [PATCH 12/28] make mattermost install idempotent --- deploy_capes.sh | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/deploy_capes.sh b/deploy_capes.sh index e5e4cbf..0281156 100644 --- a/deploy_capes.sh +++ b/deploy_capes.sh @@ -201,16 +201,21 @@ fi # Install dependencies sudo yum install epel-release mariadb-server -y +sudo systemctl start mariadb.service +sudo systemctl enable mariadb.service + +if systemctl is-active mattermost.service; then + echo "MatterMost is active" +else # Configure MariaDB -sudo systemctl start mariadb.service mysql -u root -e "CREATE DATABASE mattermost;" mysql -u root -e "GRANT ALL PRIVILEGES ON mattermost.* TO 'mattermost'@'localhost' IDENTIFIED BY '$mattermostpassphrase';" # Build Mattermost sudo mkdir -p /opt/mattermost/data -sudo curl -L https://releases.mattermost.com/4.9.2/mattermost-4.9.2-linux-amd64.tar.gz -o /opt/mattermost/mattermost.tar.gz -sudo tar -xzf /opt/mattermost/mattermost.tar.gz -C /opt/ +sudo curl -L $CURL_PROXY https://releases.mattermost.com/4.9.2/mattermost-4.9.2-linux-amd64.tar.gz -o /opt/mattermost/mattermost.tar.gz +sudo tar -xzf /opt/mattermost/mattermost.tar.gz -C /opt/ || exit # Add the Mattermost user with no login sudo useradd -s /usr/sbin/nologin mattermost @@ -227,12 +232,12 @@ sudo sed -i "s/mattermost_test/mattermost/" /opt/mattermost/config/config.json sudo sed -i "s/8065/5000/" /opt/mattermost/config/config.json # Create the Mattermost tables -cd /opt/mattermost/bin/ +pushd /opt/mattermost/bin/ sudo -u mattermost /opt/mattermost/bin/./platform -cd - +popd # Correct the MariaDB formatting -mysql -u root -e "ALTER TABLE mattermost.Audits ENGINE = MyISAM;ALTER TABLE mattermost.ChannelMembers ENGINE = MyISAM;ALTER TABLE mattermost.Channels ENGINE = MyISAM;ALTER TABLE mattermost.ClusterDiscovery ENGINE = MyISAM;ALTER TABLE mattermost.Commands ENGINE = MyISAM;ALTER TABLE mattermost.CommandWebhooks ENGINE = MyISAM;ALTER TABLE mattermost.Compliances ENGINE = MyISAM;ALTER TABLE mattermost.Emoji ENGINE = MyISAM;ALTER TABLE mattermost.FileInfo ENGINE = MyISAM;ALTER TABLE mattermost.IncomingWebhooks ENGINE = MyISAM;ALTER TABLE mattermost.Jobs ENGINE = MyISAM;ALTER TABLE mattermost.Licenses ENGINE = MyISAM;ALTER TABLE mattermost.OAuthAccessData ENGINE = MyISAM;ALTER TABLE mattermost.OAuthApps ENGINE = MyISAM;ALTER TABLE mattermost.OAuthAuthData ENGINE = MyISAM;ALTER TABLE mattermost.OutgoingWebhooks ENGINE = MyISAM;ALTER TABLE mattermost.Posts ENGINE = MyISAM;ALTER TABLE mattermost.Preferences ENGINE = MyISAM;ALTER TABLE mattermost.Reactions ENGINE = MyISAM;ALTER TABLE mattermost.Sessions ENGINE = MyISAM;ALTER TABLE mattermost.Status ENGINE = MyISAM;ALTER TABLE mattermost.Systems ENGINE = MyISAM;ALTER TABLE mattermost.TeamMembers ENGINE = MyISAM;ALTER TABLE mattermost.Teams ENGINE = MyISAM;ALTER TABLE mattermost.Tokens ENGINE = MyISAM;ALTER TABLE mattermost.UserAccessTokens ENGINE = MyISAM;ALTER TABLE mattermost.Users ENGINE = MyISAM;" +mysql -u root -e "ALTER TABLE mattermost.Audits ENGINE = MyISAM;ALTER TABLE mattermost.ChannelMembers ENGINE = MyISAM;ALTER TABLE mattermost.Channels ENGINE = MyISAM;ALTER TABLE mattermost.ClusterDiscovery ENGINE = MyISAM;ALTER TABLE mattermost.Commands ENGINE = MyISAM;ALTER TABLE mattermost.CommandWebhooks ENGINE = MyISAM;ALTER TABLE mattermost.Compliances ENGINE = MyISAM;ALTER TABLE mattermost.Emoji ENGINE = MyISAM;ALTER TABLE mattermost.FileInfo ENGINE = MyISAM;ALTER TABLE mattermost.IncomingWebhooks ENGINE = MyISAM;ALTER TABLE mattermost.Jobs ENGINE = MyISAM;ALTER TABLE mattermost.Licenses ENGINE = MyISAM;ALTER TABLE mattermost.OAuthAccessData ENGINE = MyISAM;ALTER TABLE mattermost.OAuthApps ENGINE = MyISAM;ALTER TABLE mattermost.OAuthAuthData ENGINE = MyISAM;ALTER TABLE mattermost.OutgoingWebhooks ENGINE = MyISAM;ALTER TABLE mattermost.Posts ENGINE = MyISAM;ALTER TABLE mattermost.Preferences ENGINE = MyISAM;ALTER TABLE mattermost.Reactions ENGINE = MyISAM;ALTER TABLE mattermost.Sessions ENGINE = MyISAM;ALTER TABLE mattermost.Status ENGINE = MyISAM;ALTER TABLE mattermost.Systems ENGINE = MyISAM;ALTER TABLE mattermost.TeamMembers ENGINE = MyISAM;ALTER TABLE mattermost.Teams ENGINE = MyISAM;ALTER TABLE mattermost.Tokens ENGINE = MyISAM;ALTER TABLE mattermost.UserAccessTokens ENGINE = MyISAM;ALTER TABLE mattermost.Users ENGINE = MyISAM;" || exit # Create the Mattermost service sudo tee /etc/systemd/system/mattermost.service << EOF > /dev/null @@ -254,6 +259,11 @@ WantedBy=multi-user.target EOF sudo chmod 664 /etc/systemd/system/mattermost.service +sudo systemctl daemon-reload +sudo systemctl start mattermost.service && sudo systemctl enable mattermost.service + +fi + ################################ ############ HackMD ############ ################################ @@ -567,7 +577,6 @@ sudo systemctl enable metricbeat.service sudo systemctl enable mariadb.service sudo systemctl enable hackmd.service sudo systemctl enable gitea.service -sudo systemctl enable mattermost.service sudo systemctl enable elasticsearch.service sudo systemctl enable thehive.service sudo systemctl enable cortex.service @@ -583,7 +592,6 @@ sudo systemctl start nginx.service sudo systemctl start heartbeat.service sudo systemctl start metricbeat.service sudo systemctl start filebeat.service -sudo systemctl start mattermost.service ################################ ### Secure MySQL installtion ### From 04ceb498d2ccb9ebb77cb6037f85eaf955e7095c Mon Sep 17 00:00:00 2001 From: Pete Nelson Date: Fri, 9 Nov 2018 02:11:15 +0000 Subject: [PATCH 13/28] make hackmd install idempotent --- deploy_capes.sh | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/deploy_capes.sh b/deploy_capes.sh index 0281156..18141d6 100644 --- a/deploy_capes.sh +++ b/deploy_capes.sh @@ -50,6 +50,7 @@ echo PROXY=$PROXY if [ -n "$PROXY" ]; then PIP_PROXY="--proxy $(echo $PROXY | sed 's/.*\/\///')" CURL_PROXY="--proxy $PROXY" + GIT_PROXY="-c http.proxy=$PROXY" fi ################################ @@ -271,13 +272,27 @@ fi # Install dependencies sudo yum install npm gcc-c++ git -y +if systemctl is-active hackmd.service; then + echo "HackMD is active" +else + +# set npm's proxy config if needed +if [ "$PROXY" ]; then + sudo npm config set proxy $PROXY + sudo npm config set https-proxy $PROXY + sudo git config --global https.proxy $PROXY + sudo git config --global http.proxy $PROXY + sudo git config --global url."https://".insteadOf git:// +fi + # Stage HackMD for building -sudo npm install -g uws node-gyp tap webpack grunt yarn -sudo yarn add -D webpack-cli -sudo git clone https://github.com/hackmdio/hackmd.git /opt/hackmd/ -cd /opt/hackmd +sudo rm -rf /opt/hackmd +sudo git clone $GIT_PROXY https://github.com/hackmdio/hackmd.git /opt/hackmd || exit +sudo npm install -g uws node-gyp tap webpack grunt yarn || exit +sudo yarn add -D webpack-cli || exit +pushd /opt/hackmd sudo bin/setup -cd - +popd # Set up the HackMD database mysql -u root -e "CREATE DATABASE hackmd;" @@ -317,6 +332,11 @@ ExecStart=/bin/npm start production --prefix /opt/hackmd/ WantedBy=multi-user.target EOF +sudo systemctl daemon-reload +sudo systemctl start hackmd.service && sudo systemctl enable hackmd.service + +fi + ################################ ########## Gitea ############### ################################ @@ -575,7 +595,6 @@ sudo systemctl enable heartbeat.service sudo systemctl enable filebeat.service sudo systemctl enable metricbeat.service sudo systemctl enable mariadb.service -sudo systemctl enable hackmd.service sudo systemctl enable gitea.service sudo systemctl enable elasticsearch.service sudo systemctl enable thehive.service @@ -586,7 +605,6 @@ sudo systemctl start elasticsearch.service sudo systemctl start kibana.service sudo systemctl start cortex.service sudo systemctl start gitea.service -sudo systemctl start hackmd.service sudo systemctl start thehive.service sudo systemctl start nginx.service sudo systemctl start heartbeat.service From 3f70a6003e11eb65d8b2696c76b268cc48180a38 Mon Sep 17 00:00:00 2001 From: Pete Nelson Date: Fri, 9 Nov 2018 02:17:20 +0000 Subject: [PATCH 14/28] make gitea install idempotent --- deploy_capes.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/deploy_capes.sh b/deploy_capes.sh index 18141d6..5d312ef 100644 --- a/deploy_capes.sh +++ b/deploy_capes.sh @@ -347,6 +347,10 @@ fi sudo yum install http://opensource.wandisco.com/centos/7/git/x86_64/wandisco-git-release-7-2.noarch.rpm -y sudo yum update git -y +if systemctl is-active gitea.service; then + echo "Gitea is active" +else + # Configure MariaDB mysql -u root -e "CREATE DATABASE gitea;" mysql -u root -e "GRANT ALL PRIVILEGES ON gitea.* TO 'gitea'@'localhost' IDENTIFIED BY '$giteapassphrase';" @@ -371,7 +375,7 @@ sudo useradd -s /usr/sbin/nologin gitea # Grab Gitea and make it a home sudo mkdir -p /opt/gitea -sudo curl -L https://dl.gitea.io/gitea/master/gitea-master-linux-amd64 -o /opt/gitea/gitea +sudo curl -L $CURL_PROXY https://dl.gitea.io/gitea/master/gitea-master-linux-amd64 -o /opt/gitea/gitea sudo chown -R gitea:gitea /opt/gitea sudo chmod 744 /opt/gitea/gitea @@ -403,6 +407,11 @@ Environment=USER=gitea HOME=/home/gitea WantedBy=multi-user.target EOF +sudo systemctl daemon-reload +sudo systemctl start gitea.service && sudo systemctl enable gitea.service + +fi + ################################ ########### TheHive ############ ################################ @@ -595,7 +604,6 @@ sudo systemctl enable heartbeat.service sudo systemctl enable filebeat.service sudo systemctl enable metricbeat.service sudo systemctl enable mariadb.service -sudo systemctl enable gitea.service sudo systemctl enable elasticsearch.service sudo systemctl enable thehive.service sudo systemctl enable cortex.service @@ -604,7 +612,6 @@ sudo systemctl enable cortex.service sudo systemctl start elasticsearch.service sudo systemctl start kibana.service sudo systemctl start cortex.service -sudo systemctl start gitea.service sudo systemctl start thehive.service sudo systemctl start nginx.service sudo systemctl start heartbeat.service From aa6e4709b8fd692921c5e20c2c4d9b1ac02c739b Mon Sep 17 00:00:00 2001 From: Pete Nelson Date: Fri, 9 Nov 2018 03:34:50 +0000 Subject: [PATCH 15/28] make elasticsearch+thehive+cortex install idempotent (WIP) --- deploy_capes.sh | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/deploy_capes.sh b/deploy_capes.sh index 5d312ef..5d03feb 100644 --- a/deploy_capes.sh +++ b/deploy_capes.sh @@ -423,6 +423,10 @@ sudo rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch sudo yum install https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.6.0.rpm https://centos7.iuscommunity.org/ius-release.rpm libffi-devel python-devel python-pip ssdeep-devel ssdeep-libs perl-Image-ExifTool file-devel -y sudo yum install python36u python36u-pip python36u-devel -y +if systemctl is-active elasticsearch.service; then + echo "ElasticSearch is active" +else + # Configure Elasticsearch sudo tee /etc/elasticsearch/elasticsearch.yml << EOF > /dev/null network.host: 127.0.0.1 @@ -434,7 +438,8 @@ thread_pool.bulk.queue_size: 1000 EOF # Collect the Cortex analyzers -sudo git clone https://github.com/TheHive-Project/Cortex-Analyzers.git /opt/cortex/ +rm -rf /opt/cortex +sudo git clone $GIT_PROXY https://github.com/TheHive-Project/Cortex-Analyzers.git /opt/cortex/ # Install TheHive Project and Cortex # TheHive Project is the incident tracker, Cortex is your analysis engine. @@ -463,7 +468,7 @@ play.crypto.secret="$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 64 | head _EOF_ # Add the future Python package, install the Cortex Analyzers, and adjust the Python 3 path to 3.6 -sudo pip install future +sudo pip install $PIP_PROXY future for d in /opt/cortex/analyzers/*/ ; do (cat $d/requirements.txt >> requirements.staged); done sort requirements.staged | uniq > requirements.txt rm requirements.staged @@ -474,8 +479,8 @@ sed -i "s/urllib2/urllib2\;python_version<='2.7'/" requirements.txt sed -i "s/ssdeep/ssdeep\;python_version>='3.5'/" requirements.txt echo "urllib3;python_version>='3.5'" >> requirements.txt sed -i '/requestscortexutils/d' requirements.txt -sudo /usr/bin/pip2.7 install -r requirements.txt -sudo /usr/bin/pip3.6 install -r requirements.txt +sudo /usr/bin/pip2.7 install $PIP_PROXY -r requirements.txt +sudo /usr/bin/pip3.6 install $PIP_PROXY -r requirements.txt rm requirements.txt for d in /opt/cortex/analyzers/* ; do (sudo /usr/bin/sed -i 's/python3/python3.6/' $d/*.py); done @@ -514,6 +519,12 @@ cortex { } EOF +sudo systemctl start elasticsearch.service && sudo systemctl enable elasticsearch.service +sudo systemctl start cortex.service && sudo systemctl enable cortex.service +sudo systemctl start thehive.service && sudo systemctl enable thehive.service + +fi + ################################ ############ Nginx ############# ################################ @@ -604,15 +615,9 @@ sudo systemctl enable heartbeat.service sudo systemctl enable filebeat.service sudo systemctl enable metricbeat.service sudo systemctl enable mariadb.service -sudo systemctl enable elasticsearch.service -sudo systemctl enable thehive.service -sudo systemctl enable cortex.service # Start all the services -sudo systemctl start elasticsearch.service sudo systemctl start kibana.service -sudo systemctl start cortex.service -sudo systemctl start thehive.service sudo systemctl start nginx.service sudo systemctl start heartbeat.service sudo systemctl start metricbeat.service From 6801c26facac4f501df003a10411a2051423d9fe Mon Sep 17 00:00:00 2001 From: Pete Nelson Date: Fri, 9 Nov 2018 06:41:53 +0000 Subject: [PATCH 16/28] add more proxy variants --- deploy_capes.sh | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/deploy_capes.sh b/deploy_capes.sh index 5d03feb..0e2d4e9 100644 --- a/deploy_capes.sh +++ b/deploy_capes.sh @@ -45,12 +45,22 @@ grep "$IP" /etc/hosts || echo "$IP $HOSTNAME" | sudo tee -a /etc/hosts ######### Proxy detect ######### ################################ -PROXY=$(sed -n '/^proxy=/s/.*=//p' < /etc/yum.conf) -echo PROXY=$PROXY -if [ -n "$PROXY" ]; then - PIP_PROXY="--proxy $(echo $PROXY | sed 's/.*\/\///')" - CURL_PROXY="--proxy $PROXY" - GIT_PROXY="-c http.proxy=$PROXY" +FULL_PRXY=$(sed -n '/^proxy=/s/.*=//p' < /etc/yum.conf) +echo FULL_PRXY=$FULL_PRXY +if [ -n "$FULL_PRXY" ]; then + PRXY_TUPL="$(echo $FULL_PRXY | sed 's/.*\/\///')" + PRXY_HOST="$(echo $FULL_PRXY | sed 's/.*\/\/\(.*\):.*/\1/')" + PRXY_PORT="$(echo $FULL_PRXY | sed 's/.*\/\/.*://')" + echo PRXY_TUPL=$PRXY_TUPL + echo PRXY_HOST=$PRXY_HOST + echo PRXY_PORT=$PRXY_PORT + + PIP_PROXY="--proxy $PRXY_TUPL" + CURL_PROXY="--proxy $FULL_PRXY" + GIT_PROXY="-c http.proxy=$FULL_PRXY" + RPM_PROXY="--httpproxy $PRXY_TUPL" + ES_PROXY="ES_JAVA_OPTS=-Dhttps.proxyHost=$PRXY_HOST\ -Dhttps.proxyPort=$PRXY_PORT" + echo $ES_PROXY fi ################################ From def49dd7cadeb84c91b262e2ca442e64f7570a1e Mon Sep 17 00:00:00 2001 From: Pete Nelson Date: Fri, 9 Nov 2018 06:44:23 +0000 Subject: [PATCH 17/28] continue with elasticsearch install updates --- deploy_capes.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/deploy_capes.sh b/deploy_capes.sh index 0e2d4e9..f987e2b 100644 --- a/deploy_capes.sh +++ b/deploy_capes.sh @@ -429,7 +429,7 @@ fi # Install Dependencies sudo yum install java-1.8.0-openjdk.x86_64 gcc-c++ -y sudo yum groupinstall "Development Tools" -y -sudo rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch +sudo rpm --import $RPM_PROXY https://artifacts.elastic.co/GPG-KEY-elasticsearch sudo yum install https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.6.0.rpm https://centos7.iuscommunity.org/ius-release.rpm libffi-devel python-devel python-pip ssdeep-devel ssdeep-libs perl-Image-ExifTool file-devel -y sudo yum install python36u python36u-pip python36u-devel -y @@ -448,15 +448,14 @@ thread_pool.bulk.queue_size: 1000 EOF # Collect the Cortex analyzers -rm -rf /opt/cortex +sudo rm -rf /opt/cortex sudo git clone $GIT_PROXY https://github.com/TheHive-Project/Cortex-Analyzers.git /opt/cortex/ # Install TheHive Project and Cortex # TheHive Project is the incident tracker, Cortex is your analysis engine. # If you're going to be using this offline, you can remove the Cortex install (sudo yum install thehive -y). -sudo rpm --import https://dl.bintray.com/cert-bdf/rpm/repodata/repomd.xml.key +sudo rpm --import $RPM_PROXY https://dl.bintray.com/thehive-project/rpm-stable/repodata/repomd.xml.key sudo yum install https://dl.bintray.com/thehive-project/rpm-stable/thehive-project-release-1.1.0-1.noarch.rpm -y -#sudo yum install https://dl.bintray.com/cert-bdf/rpm/thehive-project-release-1.0.0-3.noarch.rpm -y sudo yum install thehive cortex -y # Configure TheHive Project secret key @@ -490,6 +489,12 @@ sed -i "s/ssdeep/ssdeep\;python_version>='3.5'/" requirements.txt echo "urllib3;python_version>='3.5'" >> requirements.txt sed -i '/requestscortexutils/d' requirements.txt sudo /usr/bin/pip2.7 install $PIP_PROXY -r requirements.txt + +# HACK: why must I manually install these? +sudo /usr/bin/pip3.6 install $PIP_PROXY six +sudo /usr/bin/pip3.6 install $PIP_PROXY pytest-runner +sudo /usr/bin/pip3.6 install $PIP_PROXY cffi + sudo /usr/bin/pip3.6 install $PIP_PROXY -r requirements.txt rm requirements.txt for d in /opt/cortex/analyzers/* ; do (sudo /usr/bin/sed -i 's/python3/python3.6/' $d/*.py); done @@ -529,6 +534,7 @@ cortex { } EOF +sudo systemctl daemon-reload sudo systemctl start elasticsearch.service && sudo systemctl enable elasticsearch.service sudo systemctl start cortex.service && sudo systemctl enable cortex.service sudo systemctl start thehive.service && sudo systemctl enable thehive.service From 52d6a6db769d78fc69298ca30bf6febb37993543 Mon Sep 17 00:00:00 2001 From: Pete Nelson Date: Fri, 9 Nov 2018 06:46:05 +0000 Subject: [PATCH 18/28] add mkdir guard before writing files with tee --- deploy_capes.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/deploy_capes.sh b/deploy_capes.sh index f987e2b..98382fb 100644 --- a/deploy_capes.sh +++ b/deploy_capes.sh @@ -160,6 +160,7 @@ sudo sed -i 's/\#registerName=Mumble\ Server/registerName=CAPES\ -\ Mumble\ Serv sudo sed -i 's/port=64738/port=7000/' /etc/murmur.ini # Rotate logs +sudo mkdir -p /etc/logrotate.d sudo tee /etc/logrotate.d/murmur << EOF > /dev/null /var/log/murmur/*log { su murmur murmur @@ -176,6 +177,7 @@ sudo tee /etc/logrotate.d/murmur << EOF > /dev/null EOF # Creating the systemd service +sudo mkdir -p /etc/systemd/system sudo tee /etc/systemd/system/murmur.service << EOF > /dev/null [Unit] Description=Mumble Server (Murmur) @@ -194,6 +196,7 @@ WantedBy=multi-user.target EOF # Generate the pid directory for Murmur: +sudo mkdir -p /etc/tmpfiles.d sudo tee /etc/tmpfiles.d/murmur.conf << EOF > /dev/null d /var/run/murmur 775 murmur murmur EOF @@ -438,6 +441,7 @@ if systemctl is-active elasticsearch.service; then else # Configure Elasticsearch +sudo mkdir -p /etc/elasticsearch sudo tee /etc/elasticsearch/elasticsearch.yml << EOF > /dev/null network.host: 127.0.0.1 cluster.name: hive @@ -459,6 +463,7 @@ sudo yum install https://dl.bintray.com/thehive-project/rpm-stable/thehive-proje sudo yum install thehive cortex -y # Configure TheHive Project secret key +sudo mkdir -p /etc/thehive sudo tee -a /etc/thehive/application.conf << _EOF_ > /dev/null # Secret key # ~~~~~ @@ -468,6 +473,7 @@ play.crypto.secret="$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 64 | head _EOF_ # Configure Cortex secret key +sudo mkdir -p /etc/cortex sudo tee -a /etc/cortex/application.conf << _EOF_ > /dev/null # Secret key # ~~~~~ @@ -574,6 +580,7 @@ sudo curl https://gchq.github.io/CyberChef/cyberchef.htm -o /usr/share/nginx/htm ################################ sudo yum install -y https://artifacts.elastic.co/downloads/beats/heartbeat/heartbeat-5.6.5-x86_64.rpm +sudo mkdir -p /etc/heartbeat sudo cp beats/heartbeat.yml /etc/heartbeat/heartbeat.yml sudo sed -i "s/passphrase/$capespassphrase/" /etc/heartbeat/heartbeat.yml @@ -581,6 +588,7 @@ sudo sed -i "s/passphrase/$capespassphrase/" /etc/heartbeat/heartbeat.yml ######### Filebeat ############# ################################ sudo yum install -y https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-5.6.5-x86_64.rpm +sudo mkdir -p /etc/filebeat sudo cp beats/filebeat.yml /etc/filebeat/filebeat.yml sudo /usr/share/elasticsearch/bin/elasticsearch-plugin install ingest-user-agent sudo /usr/share/elasticsearch/bin/elasticsearch-plugin install ingest-geoip @@ -590,6 +598,7 @@ sudo /usr/share/elasticsearch/bin/elasticsearch-plugin install ingest-geoip ################################ sudo yum install -y https://artifacts.elastic.co/downloads/beats/metricbeat/metricbeat-5.6.5-x86_64.rpm +sudo mkdir -p /etc/metricbeat sudo cp beats/metricbeat.yml /etc/metricbeat/metricbeat.yml sudo sed -i "s/hostname/$HOSTNAME/" /etc/metricbeat/metricbeat.yml From 912ffcb59641bb612707a9d09ffcdcea41981f30 Mon Sep 17 00:00:00 2001 From: Pete Nelson Date: Fri, 9 Nov 2018 06:48:45 +0000 Subject: [PATCH 19/28] move service starts to just after each install --- deploy_capes.sh | 43 +++++++++++++++++-------------------------- 1 file changed, 17 insertions(+), 26 deletions(-) diff --git a/deploy_capes.sh b/deploy_capes.sh index 98382fb..d99a84c 100644 --- a/deploy_capes.sh +++ b/deploy_capes.sh @@ -201,6 +201,7 @@ sudo tee /etc/tmpfiles.d/murmur.conf << EOF > /dev/null d /var/run/murmur 775 murmur murmur EOF +sudo systemd-tmpfiles --create /etc/tmpfiles.d/murmur.conf sudo systemctl daemon-reload sudo systemctl start murmur.service && sudo systemctl enable murmur.service @@ -215,8 +216,8 @@ fi # Install dependencies sudo yum install epel-release mariadb-server -y -sudo systemctl start mariadb.service -sudo systemctl enable mariadb.service +sudo systemctl daemon-reload +sudo systemctl start mariadb.service && sudo systemctl enable mariadb.service if systemctl is-active mattermost.service; then echo "MatterMost is active" @@ -573,7 +574,10 @@ sudo rm /usr/share/nginx/html/build_operate_maintain.md /usr/share/nginx/html/de ################################ # Collect CyberChef -sudo curl https://gchq.github.io/CyberChef/cyberchef.htm -o /usr/share/nginx/html/cyberchef.htm +sudo curl $CURL_PROXY https://gchq.github.io/CyberChef/cyberchef.htm -o /usr/share/nginx/html/cyberchef.htm + +sudo systemctl daemon-reload +sudo systemctl start nginx.service && sudo systemctl enable nginx.service ################################ ######## Heartbeat ############# @@ -583,16 +587,22 @@ sudo yum install -y https://artifacts.elastic.co/downloads/beats/heartbeat/heart sudo mkdir -p /etc/heartbeat sudo cp beats/heartbeat.yml /etc/heartbeat/heartbeat.yml sudo sed -i "s/passphrase/$capespassphrase/" /etc/heartbeat/heartbeat.yml +sudo systemctl daemon-reload +sudo systemctl start heartbeat.service && sudo systemctl enable heartbeat.service ################################ ######### Filebeat ############# ################################ + sudo yum install -y https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-5.6.5-x86_64.rpm sudo mkdir -p /etc/filebeat sudo cp beats/filebeat.yml /etc/filebeat/filebeat.yml sudo /usr/share/elasticsearch/bin/elasticsearch-plugin install ingest-user-agent sudo /usr/share/elasticsearch/bin/elasticsearch-plugin install ingest-geoip +sudo systemctl daemon-reload +sudo systemctl start filebeat.service && sudo systemctl enable filebeat.service + ################################ ######## Metricbeat ############ ################################ @@ -601,6 +611,8 @@ sudo yum install -y https://artifacts.elastic.co/downloads/beats/metricbeat/metr sudo mkdir -p /etc/metricbeat sudo cp beats/metricbeat.yml /etc/metricbeat/metricbeat.yml sudo sed -i "s/hostname/$HOSTNAME/" /etc/metricbeat/metricbeat.yml +sudo systemctl daemon-reload +sudo systemctl start metricbeat.service && sudo systemctl enable metricbeat.service ################################ ########### Kibana ############# @@ -608,6 +620,8 @@ sudo sed -i "s/hostname/$HOSTNAME/" /etc/metricbeat/metricbeat.yml sudo yum install -y https://artifacts.elastic.co/downloads/kibana/kibana-5.6.5-x86_64.rpm sudo sed -i "s/#server\.host: \"localhost\"/server\.host: \"0\.0\.0\.0\"/" /etc/kibana/kibana.yml +sudo systemctl daemon-reload +sudo systemctl start kibana.service && sudo systemctl enable kibana.service ################################ ########## Firewall ############ @@ -625,29 +639,6 @@ sudo yum install firewalld -y sudo firewall-cmd --add-port=80/tcp --add-port=3000/tcp --add-port=4000/tcp --add-port=5000/tcp --add-port=5601/tcp --add-port=9000/tcp --add-port=9001/tcp --add-port=7000/tcp --add-port=7000/udp --permanent sudo firewall-cmd --reload -################################ -########## Services ############ -################################ - -# Prepare the service environment -sudo systemd-tmpfiles --create /etc/tmpfiles.d/murmur.conf -sudo systemctl daemon-reload - -# Configure services for autostart -sudo systemctl enable nginx.service -sudo systemctl enable kibana.service -sudo systemctl enable heartbeat.service -sudo systemctl enable filebeat.service -sudo systemctl enable metricbeat.service -sudo systemctl enable mariadb.service - -# Start all the services -sudo systemctl start kibana.service -sudo systemctl start nginx.service -sudo systemctl start heartbeat.service -sudo systemctl start metricbeat.service -sudo systemctl start filebeat.service - ################################ ### Secure MySQL installtion ### ################################ From f50849454fbf860cfa0d8c3d5767f39ead47f2a1 Mon Sep 17 00:00:00 2001 From: Pete Nelson Date: Fri, 9 Nov 2018 06:49:52 +0000 Subject: [PATCH 20/28] edit index.html after copy instead of in install dir --- deploy_capes.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy_capes.sh b/deploy_capes.sh index d99a84c..8989d18 100644 --- a/deploy_capes.sh +++ b/deploy_capes.sh @@ -560,12 +560,12 @@ sudo htpasswd -bc /etc/nginx/.htpasswd operator $capespassphrase sudo sed -i '43 a \\tauth_basic "CAPES Login";' /etc/nginx/nginx.conf sudo sed -i '44 a \\tauth_basic_user_file /etc/nginx/.htpasswd;' /etc/nginx/nginx.conf -# Update the landing page index file -sed -i "s/your-ip/$IP/" landing_page/index.html - # Move landing page framework into Nginx's working directory sudo cp -r landing_page/* /usr/share/nginx/html/ +# Update the landing page index file +sudo sed -i "s/your-ip/$IP/" /usr/share/nginx/html/index.html + # Perform a little housekeeping sudo rm /usr/share/nginx/html/build_operate_maintain.md /usr/share/nginx/html/deploy_landing_page.sh /usr/share/nginx/html/README.md From d417ad6460de52737e3bb51b85a70bc59e0a8165 Mon Sep 17 00:00:00 2001 From: Pete Nelson Date: Fri, 9 Nov 2018 07:10:41 +0000 Subject: [PATCH 21/28] add hackmd icon and section on landing page --- landing_page/images/hackmd.png | Bin 0 -> 1491 bytes landing_page/index.html | 10 +++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 landing_page/images/hackmd.png diff --git a/landing_page/images/hackmd.png b/landing_page/images/hackmd.png new file mode 100644 index 0000000000000000000000000000000000000000..6160865e10da7b2dfaa17cbb16b51c53405d8569 GIT binary patch literal 1491 zcmeAS@N?(olHy`uVBq!ia0vp^6(G#P1|%(0%q{^b$r9IylHmNblJdl&R0hYC{G?O` z&)mfH)S%SFl*+=BsWuD@%u1Od5hW46K32*3xq68pHF_1f1wh>l3^w)^1&PVosU-?Y zsp*+{wo31J?^jaDOtDo8H}y5}EpSfF$n>ZxN)4{^3rViZPPR-@vbR&PsjvbXkegbP zs8ErclUHn2VXFi-*9yo63F|80+w{G(j&jGsVi;+11F*+}O#`#n`~m(AC7!$jQjk($LY-$kf!) z&D9mA*Cju>G&eP`1g19yq1PFwUQklVEdbi=l3J8mmYU*Ll%J~r_OewbZns$CG!Lpb z1-Dxqaq86vIz}H9u}BdO69T3l5EGtkfgE_kPt60S_99@iE?c<4mVtrE-P6S}q~g|_ zyLWvbJBTy{vR`zF;!s_}s?nIL&7tbR8Q`>{VZKnA_l+$MoI6+^3%Z?R_Sk(YqWbx> znoAxZgnVl{<{pvTEYY?w_?}n;BliXd2x1mXU<`A5Fh4wezJ1=Zjd_K;U1l74eEIUh z{z9wEceV-VP0RaMS0{NqTU&TObH!wy@1HH#e%nyr^SGQ_*nVbaa!Q_MU;JYW`>+S4 zTA?* zy)ugc1%g>41>cr1J-tW~fWQW4OLrN4~Rc)51R+bKfp{y@lIi`G^0tM-6Yx zUSnW6dx}hju)xn_v)5EjJ$mt+nk5^z0oV~@39_f3KbW*syR%U@)^lI?wh~p#4U>6K zf_Qa2+e(feS7BrWTLiL>C!&EBij0}Qn`KWe3v7(P5pwYDVy64mqVji2pTw5z-1Bu& z_5(Im#@EYp!{4?U7sr-_t5sa9jVayn=$n7_{#Ckdi5nt?*(G2xjBv@7-3-hOl3csi UeoDCL2`XkiUHx3vIVCg!0IT&K5&!@I literal 0 HcmV?d00001 diff --git a/landing_page/index.html b/landing_page/index.html index bd3d7cd..e50943e 100755 --- a/landing_page/index.html +++ b/landing_page/index.html @@ -112,7 +112,15 @@

-
Rocket Chat
+
Rocket Chat
+

HackMD

+

+
+ +
+ +
+
Rocket Chat

Mattermost

From d58381996f4f0a113d7e17eabd9f3c2b6a139803 Mon Sep 17 00:00:00 2001 From: Pete Nelson Date: Fri, 9 Nov 2018 07:11:12 +0000 Subject: [PATCH 22/28] update removals in purge script --- purge_capes.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/purge_capes.sh b/purge_capes.sh index 6a577d9..bd6ff7c 100644 --- a/purge_capes.sh +++ b/purge_capes.sh @@ -48,12 +48,11 @@ sudo yum erase -y nginx httpd-tools sudo rm -f -r /usr/share/nginx/html/* sudo rm -f -r /etc/nginx/ -sudo yum erase -y thehive cortex -sudo rm -f /etc/thehive/application.conf* /etc/cortex/application.conf* -sudo rmdir /etc/cortex/ /etc/thehive/ /opt/cortex/* /opt/cortex/ /opt/thehive/* /opt/thehive/ +sudo yum erase -y cortex thehive +sudo rm -rf /etc/cortex/ /etc/thehive/ /opt/cortex/ /opt/thehive/ sudo yum erase -y elasticsearch-5.6.0 -sudo rm -f /etc/elasticsearch/elasticsearch.yml* +sudo rm -rf /etc/elasticsearch/ sudo rm -f /etc/systemd/system/gitea.service* sudo rm -f -r /opt/gitea @@ -66,7 +65,7 @@ sudo rm -f -r /opt/hackmd mysql -uroot -e "DROP DATABASE mattermost;" sudo rm -f /etc/systemd/system/mattermost.service* -sudo rmdir /opt/mattermost/data /opt/mattermost/ +sudo rm -f -r /opt/mattermost/ sudo rm -f /etc/tmpfiles.d/murmur.conf* /etc/systemd/system/murmur.service* /etc/logrotate.d/murmur* /etc/murmur.ini* sudo rm -f -r /opt/murmur /var/log/murmur From b8807d1df3f8b6e5fa3e58d8400e21bb910483dd Mon Sep 17 00:00:00 2001 From: Pete Nelson Date: Fri, 9 Nov 2018 08:07:03 +0000 Subject: [PATCH 23/28] silence the is-active service checks --- deploy_capes.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/deploy_capes.sh b/deploy_capes.sh index 8989d18..a9bbddb 100644 --- a/deploy_capes.sh +++ b/deploy_capes.sh @@ -132,7 +132,7 @@ fi ########### Mumble ############# ################################ -if systemctl is-active murmur.service; then +if systemctl -q is-active murmur.service; then echo "Mumble is active" else @@ -219,7 +219,7 @@ sudo yum install epel-release mariadb-server -y sudo systemctl daemon-reload sudo systemctl start mariadb.service && sudo systemctl enable mariadb.service -if systemctl is-active mattermost.service; then +if systemctl -q is-active mattermost.service; then echo "MatterMost is active" else @@ -286,7 +286,7 @@ fi # Install dependencies sudo yum install npm gcc-c++ git -y -if systemctl is-active hackmd.service; then +if systemctl -q is-active hackmd.service; then echo "HackMD is active" else @@ -361,7 +361,7 @@ fi sudo yum install http://opensource.wandisco.com/centos/7/git/x86_64/wandisco-git-release-7-2.noarch.rpm -y sudo yum update git -y -if systemctl is-active gitea.service; then +if systemctl -q is-active gitea.service; then echo "Gitea is active" else @@ -437,7 +437,7 @@ sudo rpm --import $RPM_PROXY https://artifacts.elastic.co/GPG-KEY-elasticsearch sudo yum install https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.6.0.rpm https://centos7.iuscommunity.org/ius-release.rpm libffi-devel python-devel python-pip ssdeep-devel ssdeep-libs perl-Image-ExifTool file-devel -y sudo yum install python36u python36u-pip python36u-devel -y -if systemctl is-active elasticsearch.service; then +if systemctl -q is-active elasticsearch.service; then echo "ElasticSearch is active" else From f6732ea50d31cbca20055df98412a6d848cc691f Mon Sep 17 00:00:00 2001 From: Pete Nelson Date: Fri, 9 Nov 2018 15:33:02 +0000 Subject: [PATCH 24/28] get es plugins install working with a proxy --- deploy_capes.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/deploy_capes.sh b/deploy_capes.sh index a9bbddb..04b885f 100644 --- a/deploy_capes.sh +++ b/deploy_capes.sh @@ -59,7 +59,8 @@ if [ -n "$FULL_PRXY" ]; then CURL_PROXY="--proxy $FULL_PRXY" GIT_PROXY="-c http.proxy=$FULL_PRXY" RPM_PROXY="--httpproxy $PRXY_TUPL" - ES_PROXY="ES_JAVA_OPTS=-Dhttps.proxyHost=$PRXY_HOST\ -Dhttps.proxyPort=$PRXY_PORT" + ES_PROXY="export ES_JAVA_OPTS=-Dhttps.proxyHost=$PRXY_HOST\ -Dhttps.proxyPort=$PRXY_PORT; echo \$ES_JAVA_OPTS; " + echo $ES_PROXY fi @@ -597,8 +598,8 @@ sudo systemctl start heartbeat.service && sudo systemctl enable heartbeat.servic sudo yum install -y https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-5.6.5-x86_64.rpm sudo mkdir -p /etc/filebeat sudo cp beats/filebeat.yml /etc/filebeat/filebeat.yml -sudo /usr/share/elasticsearch/bin/elasticsearch-plugin install ingest-user-agent -sudo /usr/share/elasticsearch/bin/elasticsearch-plugin install ingest-geoip +sudo bash -c "$ES_PROXY /usr/share/elasticsearch/bin/elasticsearch-plugin install ingest-user-agent" +sudo bash -c "$ES_PROXY /usr/share/elasticsearch/bin/elasticsearch-plugin install ingest-geoip" sudo systemctl daemon-reload sudo systemctl start filebeat.service && sudo systemctl enable filebeat.service From 1a52097411d6afd32e70341a764dd1603b1b0b9c Mon Sep 17 00:00:00 2001 From: Pete Nelson Date: Fri, 9 Nov 2018 15:37:02 +0000 Subject: [PATCH 25/28] correct missing tee append that was breaking thehive install --- deploy_capes.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy_capes.sh b/deploy_capes.sh index 04b885f..5545fc2 100644 --- a/deploy_capes.sh +++ b/deploy_capes.sh @@ -531,7 +531,7 @@ sudo chmod 640 /etc/cortex/application.conf sudo sed -i '16i\\t-Dhttp.port=9001 \\' /etc/systemd/system/cortex.service # Connect TheHive to Cortex -sudo tee /etc/thehive/application.conf << EOF > /dev/null +sudo tee -a /etc/thehive/application.conf << EOF > /dev/null # Cortex play.modules.enabled += connectors.cortex.CortexConnector cortex { From f396b29becd5cd1aaf9fe87b8bf4df15bed2645b Mon Sep 17 00:00:00 2001 From: Pete Nelson Date: Fri, 9 Nov 2018 15:43:08 +0000 Subject: [PATCH 26/28] change confi path for thehive key so it starts --- deploy_capes.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy_capes.sh b/deploy_capes.sh index 5545fc2..24dbcae 100644 --- a/deploy_capes.sh +++ b/deploy_capes.sh @@ -471,7 +471,7 @@ sudo tee -a /etc/thehive/application.conf << _EOF_ > /dev/null # ~~~~~ # The secret key is used to secure cryptographics functions. # If you deploy your application to several instances be sure to use the same key! -play.crypto.secret="$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 64 | head -n 1)" +play.http.secret.key="$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 64 | head -n 1)" _EOF_ # Configure Cortex secret key From 505e7b6f4eef929271429158097c3e1b963a08c7 Mon Sep 17 00:00:00 2001 From: Pete Nelson Date: Mon, 12 Nov 2018 13:49:07 +0000 Subject: [PATCH 27/28] use remove instead of erase when purging --- purge_capes.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/purge_capes.sh b/purge_capes.sh index bd6ff7c..d58250a 100644 --- a/purge_capes.sh +++ b/purge_capes.sh @@ -39,25 +39,25 @@ sudo systemctl disable elasticsearch.service #sudo firewall-cmd --remove-port=80/tcp --remove-port=3000/tcp --remove-port=4000/tcp --remove-port=5000/tcp --remove-port=5601/tcp --remove-port=9000/tcp --remove-port=9001/tcp --remove-port=7000/tcp --remove-port=7000/udp --permanent #sudo firewall-cmd --reload -sudo yum erase -y kibana-5.6.5 metricbeat-5.6.5 filebeat-5.6.5 heartbeat-5.6.5 +sudo yum remove -y kibana-5.6.5 metricbeat-5.6.5 filebeat-5.6.5 heartbeat-5.6.5 sudo rm -f -r /etc/filebeat/ /etc/heartbeat/ /etc/metricbeat/ /etc/kibana/ sudo rm -f /usr/share/nginx/html/cyberchef.htm -sudo yum erase -y nginx httpd-tools +sudo yum remove -y nginx httpd-tools sudo rm -f -r /usr/share/nginx/html/* sudo rm -f -r /etc/nginx/ -sudo yum erase -y cortex thehive +sudo yum remove -y cortex thehive sudo rm -rf /etc/cortex/ /etc/thehive/ /opt/cortex/ /opt/thehive/ -sudo yum erase -y elasticsearch-5.6.0 +sudo yum remove -y elasticsearch-5.6.0 sudo rm -rf /etc/elasticsearch/ sudo rm -f /etc/systemd/system/gitea.service* sudo rm -f -r /opt/gitea mysql -uroot -e "DROP DATABASE gitea;" -sudo yum erase -y wandisco-git-release-7-2 +sudo yum remove -y wandisco-git-release-7-2 mysql -uroot -e "DROP DATABASE hackmd;" sudo rm -f /etc/systemd/system/hackmd.service* From 7f0e2a6eefbe043c38e6f5d3a3c9d1fb32b54603 Mon Sep 17 00:00:00 2001 From: Pete Nelson Date: Mon, 12 Nov 2018 13:57:27 +0000 Subject: [PATCH 28/28] remove yarn related temp files after use --- deploy_capes.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/deploy_capes.sh b/deploy_capes.sh index 24dbcae..e736ca7 100644 --- a/deploy_capes.sh +++ b/deploy_capes.sh @@ -308,6 +308,7 @@ sudo yarn add -D webpack-cli || exit pushd /opt/hackmd sudo bin/setup popd +sudo rm -f yarn.lock package.json # Set up the HackMD database mysql -u root -e "CREATE DATABASE hackmd;"