Commit abf07d54 authored by Carlos Torres's avatar Carlos Torres
Browse files

Improving roles, doing variable checks.

parent 990d92d8
......@@ -5,6 +5,32 @@ set -e
HOSTIP=$1
DOMAINS=$2
api_access_token=$3
http_port_list=$4
https_port_list=$5
humanity_validator_port_list=$6
if [ "$(echo -ne ${http_port_list} | wc -m)" -eq 0 ]; then
echo "FAILED: Checking variables. You must provide valid http ports."
exit 0
fi
if [ "$(echo -ne ${https_port_list} | wc -m)" -eq 0 ]; then
echo "FAILED: Checking variables. You must provide valid https ports."
exit 0
fi
if [ "$(echo -ne ${humanity_validator_port_list} | wc -m)" -eq 0 ]; then
echo "FAILED: Checking variables. You must provide valid humanity validator ports."
exit 0
fi
all_ports=($(echo ${http_port_list[*]}) $(echo ${https_port_list[*]}) $(echo ${humanity_validator_port_list[*]}))
if [[ $( awk -v RS=" " ' a[$0]++ ' <<< "${all_ports[@]} " ) ]]; then
duplicates=($(printf '%s\n' "${all_ports[@]}"|awk '!($0 in seen){seen[$0];next} 1'))
echo "FAILED: Checking variables. Found duplicates ports: ${duplicates[*]}."
exit 0
fi
if ! [[ $HOSTIP =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
echo "FAILED: Checking variables. ansible_host (ip): $HOSTIP is incorrectly."
......
......@@ -26,7 +26,7 @@
- "{{ install_dir_list }}"
- name: Check variables
script: "{{ role_path }}/files/checking.sh '{{ ansible_host }}' '{{ domains|join(' ') }}' '{{ api_access_token }}'"
script: "{{ role_path }}/files/checking.sh '{{ ansible_host }}' '{{ domains|join(' ') }}' '{{ api_access_token }}' '{{ http_port_list|join(' ') }}' '{{ https_port_list|join(' ') }}' '{{ humanity_validator_port_list|join(' ') }}'"
register: check_variables
- debug: var=check_variables.stdout
......
......@@ -5,6 +5,32 @@ set -e
HOSTIP=$1
DOMAINS=$2
api_access_token=$3
http_port_list=$4
https_port_list=$5
humanity_validator_port_list=$6
if [ "$(echo -ne ${http_port_list} | wc -m)" -eq 0 ]; then
echo "FAILED: Checking variables. You must provide valid http ports."
exit 0
fi
if [ "$(echo -ne ${https_port_list} | wc -m)" -eq 0 ]; then
echo "FAILED: Checking variables. You must provide valid https ports."
exit 0
fi
if [ "$(echo -ne ${humanity_validator_port_list} | wc -m)" -eq 0 ]; then
echo "FAILED: Checking variables. You must provide valid humanity validator ports."
exit 0
fi
all_ports=($(echo ${http_port_list[*]}) $(echo ${https_port_list[*]}) $(echo ${humanity_validator_port_list[*]}))
if [[ $( awk -v RS=" " ' a[$0]++ ' <<< "${all_ports[@]} " ) ]]; then
duplicates=($(printf '%s\n' "${all_ports[@]}"|awk '!($0 in seen){seen[$0];next} 1'))
echo "FAILED: Checking variables. Found duplicates ports: ${duplicates[*]}."
exit 0
fi
if ! [[ $HOSTIP =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
echo "FAILED: Checking variables. ansible_host (ip): $HOSTIP is incorrectly."
......
......@@ -26,7 +26,7 @@
- "{{ install_dir_list }}"
- name: Check variables
script: "{{ role_path }}/files/checking.sh '{{ ansible_host }}' '{{ domains|join(' ') }}' '{{ api_access_token }}'"
script: "{{ role_path }}/files/checking.sh '{{ ansible_host }}' '{{ domains|join(' ') }}' '{{ api_access_token }}' '{{ http_port_list|join(' ') }}' '{{ https_port_list|join(' ') }}' '{{ humanity_validator_port_list|join(' ') }}'"
register: check_variables
- debug: var=check_variables.stdout
......
#!/bin/bash
#echo "Updating grok_exporter_conf.yml"
set -e
INSTALLERS_DIR=$1
INSTALLDIRS=$2
FILE='/srv/grok_exporter/grok_exporter_conf.yml'
FILESUPERVISORDALL="/supervisor/*.log"
FILESUPERVISORD="/supervisor/supervisord.log"
for INSTALLDIR in $INSTALLDIRS
do
if ! grep -Fq "$INSTALLDIR$FILESUPERVISORDALL" "$FILE"; then
if [ -f "$INSTALLDIR$FILESUPERVISORD" ]; then
sed -i -e '/#to-add-entries/r '$INSTALLERS_DIR'/data/grok_exporter_single_conf.txt' $FILE
sed -i '/#to-add-paths/ a \ \-\ \'$INSTALLDIR$FILESUPERVISORDALL $FILE
else
echo "FAILED: $INSTALLDIR$FILESUPERVISORD does not exist, make sure it is in place."
exit 0
fi
fi
done
echo "SUCCESS: Updating grok_exporter_conf.yml"
exit 0
......@@ -67,7 +67,6 @@
group: "{{ grok_exporter_system_group }}"
mode: '0775'
- name: Cleaning grok_exporter temporary folder
file:
state: absent
......@@ -75,7 +74,6 @@
remote_src: yes
- name: Copy custom grok_exporter/patterns to the remote server
# You can use synchronize instead of copy, in case the user 'ansible_user' be root.
copy:
src: "{{ role_path }}/files/grok_patterns/"
dest: "/srv/grok_exporter/patterns"
......@@ -83,19 +81,14 @@
- name: Include task configure_dmesg.yml
include_tasks: "{{ role_path }}/tasks/configure_dmesg.yml"
- name: Upload grok_exporter config to /srv/grok_exporter/grok_exporter_conf.yml
- name: Upload grok_exporter config skeleton to /srv/grok_exporter/grok_exporter_conf.yml
template:
src: "{{ role_path }}/templates/grok_exporter_conf.jinja2"
src: "{{ role_path }}/templates/grok_exporter_conf_skeleton.jinja2"
dest: /srv/grok_exporter/grok_exporter_conf.yml
owner: "{{ grok_exporter_system_user }}"
group: "{{ grok_exporter_system_group }}"
force: no
- name: Include task updating_grok_exporter_config.yml
include_tasks: "{{ role_path }}/tasks/updating_grok_exporter_config.yml"
with_list: "{{deployments_dirs}}"
- name: Upload grok_exporter.service
template:
src: "{{ role_path }}/templates/grok_exporter.service.jinja2"
......
---
# Updating grok_exporter_conf.yml
- name: Uploading grok_exporter_single_conf.txt
template:
src: "{{ role_path }}/templates/grok_exporter_single_conf.jinja2"
dest: "{{installers_dir}}/data/grok_exporter_single_conf.txt"
- name: Updating grok_exporter config
script: "{{ role_path }}/files/updating_grok_exporter_config.sh '{{ installers_dir }}' '{{ item }}'"
register: check_result_up_sup
- debug: var=check_result_up_sup.stdout
- fail:
msg: "{{check_result_up_sup.stdout}}"
when: "'FAILED' in check_result_up_sup.stdout"
---
global:
config_version: 2
input:
type: file
paths:
- /var/log/dmesg/*.log
#to-add-paths
readall: false
fail_on_missing_logfile: false
grok:
patterns_dir: /srv/grok_exporter/patterns
additional_patterns:
- FPMDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{ISO8601_TIMEZONE}
metrics:
- type: counter
name: no_of_segfaults
help: Example counter metric with labels.
match: '%{SYSLOGPROG}[:]%{SPACE}%{SEGFAULT:segfault}%{SPACE}at%{SPACE}%{BASE16NUM}%{SPACE}ip%{SPACE}%{BASE16NUM:ip}%{SPACE}sp%{SPACE}%{BASE16NUM:sp}%{SPACE}%{ERROR_SE:status}%{SPACE}%{NUMBER:error_number}%{SPACE}in%{SPACE}%{GREEDYDATA:memory_area}'
path: /var/log/dmesg/*.log
#to-add-entries
server:
protocol: http
host: {{ item.server.host | default(ansible_default_ipv4.address) }}
port: {{ grok_exporter_port }}
########## {{ item | regex_replace('^\\/|\\/$', '') | regex_replace('\\/', '_') | regex_replace('[^\w-]', '_') }} ##########
- type: counter
name: usher_exit_status_{{ item | regex_replace('^\\/|\\/$', '') | regex_replace('\\/', '_') | regex_replace('[^\w-]', '_') | regex_replace('[^\w-]', '_') }}
help: Example counter metric with labels.
match: '%{TIMESTAMP_ISO8601:timestamp},(?<pid>.{3})%{SPACE}%{LOGLEVEL:info}%{SPACE}%{EXITED:status}:%{SPACE}%{USHER:service}%{SPACE}[(]%{EXIT}%{SPACE}%{STATUS}%{SPACE}%{EXIT_STATUS:exit_status}[;]%{SPACE}%{NOT}%{SPACE}%{EXPECTED}[)]'
path: {{ item }}/supervisor/*.log
- type: counter
name: periodic_tasks_exit_status_{{ item | regex_replace('^\\/|\\/$', '') | regex_replace('\\/', '_') | regex_replace('[^\w-]', '_') }}
help: Example counter metric with labels.
match: '%{TIMESTAMP_ISO8601:timestamp},(?<pid>.{3})%{SPACE}%{LOGLEVEL:info}%{SPACE}%{EXITED:status}:%{SPACE}%{PERIODIC_TASKS:service}%{SPACE}[(]%{EXIT}%{SPACE}%{STATUS}%{SPACE}%{EXIT_STATUS:exit_status}[;]%{SPACE}%{NOT}%{SPACE}%{EXPECTED}[)]'
path: {{ item }}/supervisor/*.log
- type: counter
name: humanity_validator_exit_status_{{ item | regex_replace('^\\/|\\/$', '') | regex_replace('\\/', '_') | regex_replace('[^\w-]', '_') }}
help: Example counter metric with labels.
match: '%{TIMESTAMP_ISO8601:timestamp},(?<pid>.{3})%{SPACE}%{LOGLEVEL:info}%{SPACE}%{EXITED:status}:%{SPACE}%{HUMANITY_VALIDATOR:service}%{SPACE}[(]%{EXIT}%{SPACE}%{STATUS}%{SPACE}%{EXIT_STATUS:exit_status}[;]%{SPACE}%{NOT}%{SPACE}%{EXPECTED}[)]'
path: {{ item }}/supervisor/*.log
- type: counter
name: bots_exit_status_{{ item | regex_replace('^\\/|\\/$', '') | regex_replace('\\/', '_') | regex_replace('[^\w-]', '_') }}
help: Example counter metric with labels.
match: '%{TIMESTAMP_ISO8601:timestamp},(?<pid>.{3})%{SPACE}%{LOGLEVEL:info}%{SPACE}%{EXITED:status}:%{SPACE}%{BOTS:service}%{SPACE}[(]%{EXIT}%{SPACE}%{STATUS}%{SPACE}%{EXIT_STATUS:exit_status}[;]%{SPACE}%{NOT}%{SPACE}%{EXPECTED}[)]'
path: {{ item }}/supervisor/*.log
- type: counter
name: shimmercat_exit_status_{{ item | regex_replace('^\\/|\\/$', '') | regex_replace('\\/', '_') | regex_replace('[^\w-]', '_') }}
help: Example counter metric with labels.
match: '%{TIMESTAMP_ISO8601:timestamp},(?<pid>.{3})%{SPACE}%{LOGLEVEL:info}%{SPACE}%{EXITED:status}:%{SPACE}%{SHIMMERCAT:service}%{SPACE}[(]%{EXIT}%{SPACE}%{STATUS}%{SPACE}%{EXIT_STATUS:exit_status}[;]%{SPACE}%{NOT}%{SPACE}%{EXPECTED}[)]'
path: {{ item }}/supervisor/*.log
- type: counter
name: redis_exit_status_{{ item | regex_replace('^\\/|\\/$', '') | regex_replace('\\/', '_') | regex_replace('[^\w-]', '_') }}
help: Example counter metric with labels.
match: '%{TIMESTAMP_ISO8601:timestamp},(?<pid>.{3})%{SPACE}%{LOGLEVEL:info}%{SPACE}%{EXITED:status}:%{SPACE}%{REDIS:service}%{SPACE}[(]%{EXIT}%{SPACE}%{STATUS}%{SPACE}%{EXIT_STATUS:exit_status}[;]%{SPACE}%{NOT}%{SPACE}%{EXPECTED}[)]'
path: {{ item }}/supervisor/*.log
- type: counter
name: sc_logs_agent_exit_status_{{ item | regex_replace('^\\/|\\/$', '') | regex_replace('\\/', '_') | regex_replace('[^\w-]', '_') }}
help: Example counter metric with labels.
match: '%{TIMESTAMP_ISO8601:timestamp},(?<pid>.{3})%{SPACE}%{LOGLEVEL:info}%{SPACE}%{EXITED:status}:%{SPACE}%{SC_LOGS_AGENT:service}%{SPACE}[(]%{EXIT}%{SPACE}%{STATUS}%{SPACE}%{EXIT_STATUS:exit_status}[;]%{SPACE}%{NOT}%{SPACE}%{EXPECTED}[)]'
path: {{ item }}/supervisor/*.log
- type: counter
name: celery_exit_status_{{ item | regex_replace('^\\/|\\/$', '') | regex_replace('\\/', '_') | regex_replace('[^\w-]', '_') }}
help: Example counter metric with labels.
match: '%{TIMESTAMP_ISO8601:timestamp},(?<pid>.{3})%{SPACE}%{LOGLEVEL:info}%{SPACE}%{EXITED:status}:%{SPACE}%{CELERY:service}%{SPACE}[(]%{EXIT}%{SPACE}%{STATUS}%{SPACE}%{EXIT_STATUS:exit_status}[;]%{SPACE}%{NOT}%{SPACE}%{EXPECTED}[)]'
path: {{ item }}/supervisor/*.log
## Defaults for install_haproxy role
haproxyconfig_option: "option-2" # option-1 or option-2, or option-3, or option-4
haproxy_auth_pass: "GkM6G)jN3(jkH"
global
log /dev/log local0
log /dev/log local1 notice
# chroot /srv/haproxy
user shimmercat
group shimmercat
listen stats
bind :9000
mode http
stats enable
stats realm Haproxy\ Statistics
stats uri /haproxy_stats
stats auth shimmercat:4.15.0-32-generic
frontend http_to_https_redirects
bind *:80
timeout client 20s
mode http
# ShimmerCat can do HTTP to HTTPS redirects, and so can Haproxy
redirect scheme https code 301 if !{ ssl_fc }
frontend https
bind *:443
timeout client 3m
option tcplog
log /dev/log local0
mode tcp
tcp-request inspect-delay 5s
tcp-request content accept if { req_ssl_hello_type 1 }
#https-entry
#https-def-back
#https-backend-bare-entry
frontend to_application
bind *:4430
timeout client 3m
timeout server 3m
timeout connect 5s
mode http
#https-redirect-entry
#https-redirect-def-back
#https-redirect-backend-bare-entry
......@@ -11,7 +11,7 @@ listen stats
stats enable
stats realm Haproxy\ Statistics
stats uri /haproxy_stats
stats auth shimmercat:{{ haproxy_auth_pass }}
stats auth shimmercat:jNmh5Fg(H4Dfc
frontend http_to_https_redirects
bind *:80
......
......@@ -13,15 +13,15 @@
- name: Ensure files are 0644
command: find /srv/haproxy/program -type f -exec chmod 0755 {} \;
- name: Copy haproxy_skeleton config to the remote server
- name: Copy haproxy config to the remote server
template:
src: "{{ role_path }}/templates/config/{{ haproxyconfig_option }}/haproxy_skeleton/haproxy.cfg.jinja2"
src: "{{ role_path }}/templates/haproxy.cfg.jinja2"
dest: /srv/haproxy/data/haproxy.cfg
force: no
- name: Copy haproxy whitelist to the remote server
template:
src: "{{ role_path }}/templates/config/{{ haproxyconfig_option }}/haproxy_skeleton/whitelist.lst.jinja2"
src: "{{ role_path }}/templates/whitelist.lst.jinja2"
dest: /srv/haproxy/data/whitelist.lst
force: no
......
global
log /dev/log local0
log /dev/log local1 notice
# chroot /srv/haproxy
user shimmercat
group shimmercat
listen stats
bind :9000
mode http
stats enable
stats realm Haproxy\ Statistics
stats uri /haproxy_stats
stats auth shimmercat:{{ haproxy_auth_pass }}
frontend http_to_https_redirects
bind *:80
timeout client 20s
mode http
# ShimmerCat can do HTTP to HTTPS redirects, and so can Haproxy
redirect scheme https code 301 if !{ ssl_fc }
frontend https
bind *:443
timeout client 30m
mode tcp
## Sticky tables to deter script kiddies, comment for load-testing
# stick-table type ip size 100k expire 30s store conn_cur
# Where to have a whitelist, if you need one
# tcp-request connection accept if { src -f /srv/haproxy/data/whitelist.lst }
# tcp-request connection reject if { src_conn_cur ge 20000 }
# tcp-request connection track-sc1 src
## end sticky tables section
## Reject some connection
# acl is-to-reject src 111.11.11.116 55.55.0.0/16
# tcp-request connection reject if is-to-reject
tcp-request inspect-delay 5s
tcp-request content accept if { req_ssl_hello_type 1 }
#https-entry
#https-def-back
#https-backend-bare-entry
frontend to_application
bind *:4430
timeout client 30m
timeout server 5m
timeout connect 5m
mode http
#https-redirect-entry
#https-redirect-def-back
#https-redirect-backend-bare-entry
global
log /dev/log local0
log /dev/log local1 notice
# chroot /srv/haproxy
user shimmercat
group shimmercat
listen stats
bind :9000
mode http
stats enable
stats realm Haproxy\ Statistics
stats uri /haproxy_stats
stats auth shimmercat:{{ haproxy_auth_pass }}
frontend http_to_https_redirects
bind *:80
timeout client 20s
mode http
# ShimmerCat can do HTTP to HTTPS redirects, and so can Haproxy
redirect scheme https code 301 if !{ ssl_fc }
frontend https
bind *:443
timeout client 30m
mode tcp
## Sticky tables to deter script kiddies, comment for load-testing
# stick-table type ip size 100k expire 30s store conn_cur
# Where to have a whitelist, if you need one
# tcp-request connection accept if { src -f /srv/haproxy/data/whitelist.lst }
# tcp-request connection reject if { src_conn_cur ge 20000 }
# tcp-request connection track-sc1 src
## end sticky tables section
## Reject some connection
# acl is-to-reject src 111.11.11.116 55.55.0.0/16
# tcp-request connection reject if is-to-reject
tcp-request inspect-delay 5s
tcp-request content accept if { req_ssl_hello_type 1 }
#https-entry
#https-def-back
#https-backend-bare-entry
frontend to_application
bind *:4430
timeout client 30m
timeout server 5m
timeout connect 5m
mode http
#https-redirect-entry
#https-redirect-def-back
#https-redirect-backend-bare-entry
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment