commit a17b5e9262259775f31fae4103a26304441fc928 Author: Sven Holz Date: Mon Jun 13 23:35:46 2022 +0200 inital upload diff --git a/Docker_Build/Dockerfile b/Docker_Build/Dockerfile new file mode 100644 index 0000000..5a2b0db --- /dev/null +++ b/Docker_Build/Dockerfile @@ -0,0 +1,12 @@ +# syntax=docker/dockerfile:1 + +FROM python:3.9.13-slim-bullseye + +WORKDIR /app + +COPY requirements.txt requirements.txt +RUN pip3 install -r requirements.txt + +COPY . . + +CMD [ "python3", "latency_monitor.py"] \ No newline at end of file diff --git a/Docker_Build/config.ini b/Docker_Build/config.ini new file mode 100644 index 0000000..65d3a47 --- /dev/null +++ b/Docker_Build/config.ini @@ -0,0 +1,35 @@ +# Config File Usage +# +# all MUST options are marked with #!!! and have to be configured +# +# Config file ist IGNORED if equivalent environment variables are passed +# +# + + +[influx] # SOME are MUST; influxdb2 connection relevant options +#!!! infx_url = http://YOUR_INFLUX_DB_V2:8086 +#!!! infx_token = YOUR_INFLUX_DB_V2_API_TOKEN +#!!! infx_bucket = YOUR_INFLUX_DB_V2_BUCKET +#!!! infx_org = YOUR_INFLUX_DB_V2_ORGANIZATION +# infx_batch_size = 60 # OPTIONAL; default is '60' +# infx_flush_interval = 30_000 # OPTIONAL; default is '30_000' +# infx_jitter_interval = 5_000 # OPTIONAL; default is '5_000' +# infx_retry_interval = 5_000 # OPTIONAL; default is '5_000' + + +[hosts] # ONE HOST is MUST, rest ist optional; IP or FQDN +#!!! host1 = 8.8.8.8 +# host2 = 8.8.4.4 +# host3 = 1.1.1.1 + +[hosts_timer] # OPTIONAL; ping interval in seconds; default is '5' +# host1 = 1 +# host2 = 3 +# host3 = 5 + +[hosts_location] # OPTIONAL; location of host; default is 'unknown' +# host1 = Google-DNS1 +# host2 = Google-DNS2 +# host3 = Cloudflare-DNS + diff --git a/Docker_Build/latency_monitor.py b/Docker_Build/latency_monitor.py new file mode 100644 index 0000000..a0be2f0 --- /dev/null +++ b/Docker_Build/latency_monitor.py @@ -0,0 +1,171 @@ +import sys +import os +import datetime +import configparser +from threading import * +from time import * +from pythonping import ping +# pip3 install influxdb-client +from influxdb_client import InfluxDBClient, Point, WriteOptions, WritePrecision + +class MyInfluxDB(): + """ + Class for InfluxDB connections. + + Reads config from *config.ini*: + [influx] + infx_url = Influx2 URL like: *http://10.0.0.1:8086* (must) + infx_token = Influx2 Token for corresponding bucket (must) + infx_bucket = Influx2 Bucket (must) + infx_org = Influx2 Organization (must) + + Attributes + ---------- + n.a. + + Methods + ------- + write(host(string), host_location(string), ping_response(string)) + writes received data to Influxdb: + host: string which includes IP or FQDN + host_location: string which includes location of the host + ping_response: string which includes the ping reply in ms + """ + + def __init__(self): + + ## IF ENVIRONMENT VARIABLES ARE PASSED IGNORE CONFIG FILE + if 'INFLUX_URL' in os.environ: + + INFX_URL = os.environ['INFLUX_URL'] + INFX_TOKEN = os.environ['INFLUX_TOKEN'] + INFX_BUCKET = os.environ['INFLUX_BUCKET'] + INFX_ORG = os.environ['INFLUX_ORG'] + INFX_BATCH = os.getenv('INFLUX_BATCH', 60) # if not set use default + INFX_FINT = os.getenv('INFLUX_FINT', 30_000) # if not set use default + INFX_JINT = os.getenv('INFLUX_JINT', 5_000) # if not set use default + INFX_RINT = os.getenv('INFLUX_RINT', 5_000) # if not set use default + + else: + + config_file = os.path.join(os.path.dirname(__file__), 'config.ini') + config = configparser.ConfigParser() + config.read(config_file) + + INFX_URL = config['influx']['infx_url'] + INFX_TOKEN = config['influx']['infx_token'] + INFX_BUCKET = config['influx']['infx_bucket'] + INFX_ORG = config['influx']['infx_org'] + INFX_BATCH = config.get('influx', 'infx_batch_size', fallback=60) # if not set use default + INFX_FINT = config.get('influx', 'infx_flush_interval', fallback=30_000) # if not set use default + INFX_JINT = config.get('influx', 'infx_jitter_interval', fallback=5_000) # if not set use default + INFX_RINT = config.get('influx', 'infx_retry_interval', fallback=5_000) # if not set use default + + # This one is needed in our methods + self.INFX_BUCKET = INFX_BUCKET + + # create influxdb client + self.client = InfluxDBClient(url=INFX_URL, + token=INFX_TOKEN, + org=INFX_ORG) + + # create influxdb write api + self.write_api = self.client.write_api(write_options=WriteOptions(batch_size=INFX_BATCH, + flush_interval=INFX_FINT, + jitter_interval=INFX_JINT, + retry_interval=INFX_RINT)) + + def __del__(self): + self.client.close() + + def write(self, host, host_location, ping_response): + self.host = host + self.host_location = host_location + self.ping_response = ping_response + self.influx_timestamp = int(time_ns()) + self.data_point = Point("latency_monitor").tag("location", self.host_location).tag("host", self.host).field("latency", self.ping_response).time(self.influx_timestamp) + self.write_api.write(bucket=self.INFX_BUCKET, + record=self.data_point, + write_precision='s') + + +class ThreadPing(Thread): + """ + Class of type thread which *pings* given hosts and passes data to InfluxDB2. + - one thread for each ping + - passes ping results to given InfluxDB2 + + Arguments + ---------- + db: InfluxDB Object + host: string which includes IP or FQDN + host_timer: integer which defines how often pings are send in seconds (min. 1) + host_location: string which includes location of the host + """ + def __init__(self, db, host, host_timer, host_location): + Thread.__init__(self) + self.MyDB = db + self.host = host + self.host_timer = host_timer + self.host_location = host_location + + def run(self): + self.starttime = time() + while True: + self.ping_response_list = ping(self.host, count=1) + self.ping_response = self.ping_response_list.rtt_avg_ms + self.MyDB.write(self.host, self.host_location, self.ping_response) + sleep(self.host_timer - ((time() - self.starttime) % 1)) + + +def main(): + + MyDB = MyInfluxDB() + + # Place to store running threads... + my_threads = [] + + ## IF ENVIRONMENT VARIABLES ARE PASSED IGNORE CONFIG FILE + if 'TARGET_HOST' in os.environ: + host = os.environ['TARGET_HOST'] + host_timer = int(os.getenv('TARGET_TIMER', 5)) + host_location = os.getenv('TARGET_LOCATION', 'unknown') + + # Create Thread + print("Creating thread for: %s, with interval: %s and location: %s" %(host, host_timer, host_location)) + thread = ThreadPing(MyDB, host, host_timer, host_location) + my_threads.append(thread) + thread.start() + + else: + + ## Read Config file + config_file = os.path.join(os.path.dirname(__file__), 'config.ini') + config = configparser.ConfigParser() + config.read(config_file) + host_items = config.items("hosts") + + # Create thread for each configured host + for key, host in host_items: + + # Check if hosts timer is set otherwise use "5" (means 5 seconds) + host_timer = int(config.get('hosts_timer', key, fallback=5)) + + # Check if hosts location is set otherwise use "unknown" + host_location = config.get('hosts_location', key, fallback="unknown") + + # Create Thread + print("Creating thread for: %s, with interval: %s and location: %s" %(host, host_timer, host_location)) + thread = ThreadPing(MyDB, host, host_timer, host_location) + my_threads.append(thread) + thread.start() + + # Join one child thread otherwise main thread will stop (endless loop is also an option) + for thread in my_threads: + thread.join() + + +if __name__ == '__main__': + main() + + diff --git a/Docker_Build/requirements.txt b/Docker_Build/requirements.txt new file mode 100644 index 0000000..0b5798d --- /dev/null +++ b/Docker_Build/requirements.txt @@ -0,0 +1,2 @@ +influxdb-client==1.27.0 +pythonping==1.1.1 diff --git a/Docker_Build/template_config.ini b/Docker_Build/template_config.ini new file mode 100644 index 0000000..65d3a47 --- /dev/null +++ b/Docker_Build/template_config.ini @@ -0,0 +1,35 @@ +# Config File Usage +# +# all MUST options are marked with #!!! and have to be configured +# +# Config file ist IGNORED if equivalent environment variables are passed +# +# + + +[influx] # SOME are MUST; influxdb2 connection relevant options +#!!! infx_url = http://YOUR_INFLUX_DB_V2:8086 +#!!! infx_token = YOUR_INFLUX_DB_V2_API_TOKEN +#!!! infx_bucket = YOUR_INFLUX_DB_V2_BUCKET +#!!! infx_org = YOUR_INFLUX_DB_V2_ORGANIZATION +# infx_batch_size = 60 # OPTIONAL; default is '60' +# infx_flush_interval = 30_000 # OPTIONAL; default is '30_000' +# infx_jitter_interval = 5_000 # OPTIONAL; default is '5_000' +# infx_retry_interval = 5_000 # OPTIONAL; default is '5_000' + + +[hosts] # ONE HOST is MUST, rest ist optional; IP or FQDN +#!!! host1 = 8.8.8.8 +# host2 = 8.8.4.4 +# host3 = 1.1.1.1 + +[hosts_timer] # OPTIONAL; ping interval in seconds; default is '5' +# host1 = 1 +# host2 = 3 +# host3 = 5 + +[hosts_location] # OPTIONAL; location of host; default is 'unknown' +# host1 = Google-DNS1 +# host2 = Google-DNS2 +# host3 = Cloudflare-DNS + diff --git a/README.md b/README.md new file mode 100644 index 0000000..e718434 --- /dev/null +++ b/README.md @@ -0,0 +1,164 @@ +# Docker Based Latency Monitor + +Docker container which tracks latency of one or many hosts and reports to InfluxDBv2. + +## Description + +This Docker Container is able to track the latency of one or many targets and reports all data to a given InfluxDBv2. + +It´s based on python3 an makes usage of following python libraries: + +- pythonping +- influxdb_client +- threading +- sys +- os +- datetime +- configparser +- time + +Configuration can be passed via ENV **OR** configuration file. + +In case of using the ENV option you are just able to monitor **ONE** target for more targets please use the configuration file. + +Also some influx connection options are just configurable via config file but normally they are not needed. + +## Getting Started + +### Requirements + +- Docker +- Docker-Compose +- InfluxDB Version >= 2 +- pythonping needs root privileges so same for the container + +### ENV Variables + +Name | Example | Usage | Option/Must +:------: | :-----: | :-----: | :-----: +INFLUX_URL | http://10.0.0.1:8086 | InfluxDB Host | must +INFLUX_TOKEN | eWOcp-MCv2Y3IJPlER7wcCiuYkV8HdC8d7...ICKirhw0lwEczRNnrIoTqZAg== | InfluxDB API Token | must +INFLUX_BUCKET | latency | InfluxDB Bucket | must +INFLUX_ORG | MyOrg | InfluxDB Organization | must +TARGET_HOST | 8.8.8.8 | Monitored Host (IP/FQDN) | must + +### Config File + +**Instead** of using the ENV variables you can use a config file. + +**Keep in mind it´s a OR decision not a AND** + +See [template_config.ini](./Docker_Build/template_config.ini) + +Rename the file to *config.ini* make your changes and add it as a volume mount to the container: + +#### Docker-Compose Style + +``` + volumes: + - /YOUR_PATH/config.ini:/app/config.ini +``` + +#### Docker-CLI Style + +``` + docker latency-monitor -v /YOUR_PATH/config.ini:/app/config.ini +``` + +### Compose Files + +#### FULL-STACK + +1st thing to do is creating the *docker-compose.yml: + +``` +cp docker-compose-full_stack.yml docker-compose.yml +``` + +##### Certificate + +*Traefik* will act as a proxy and ensures the usage of TLS so it needs your certificate and key file. + +within the *docker-compose.yml* you will find: + +``` + - ./traefik/mycert.crt:/certs/cert.crt:ro + - ./traefik/mycert.key:/certs/privkey.key:ro +``` + +so please place your certificate file as *./traefik/mycert.crt* and the key file as *./traefik/mycert.key*. + +Thats it + +##### Variables +You need to configure Variables in following files to make the compose work: + +- **file** + - VARIABLE1 + - VARIABLE2 + - VARIABLE3 + +----- + +- **docker-compose.yml** *(was docker-compose-full_stack.yml before* + - PLACE_YOUR_FQDN_HERE (3 times) + +----- + +- **.env** *(env needs to be renamed to .env)* + - YOUR_PATH_TO_CONTAINER_STATIC_DATA + - YOUR_ADMIN_USER + - YOUR_ADMIN_PASSWORD + - YOUR_ORGANIZATION + - YOUR_BUCKET_NAME + - YOUR_ADMIN_TOKEN + +----- + +- **grafana/provisioning/datasources/grafana-datasource.yml** + - YOUR_ADMIN_TOKEN + - YOUR_ORGANIZATION + - YOUR_BUCKET_NAME + + +##### File Permissions +Because we are configuring *grafana* for permanent data storing and *grafana* actually runs with *UID* + *GID*: *472:472* it´s necessary to change permisson of die permanent storage directory we have configured. + +The directory build from the following config part of grafana within the docker-compose.yml: + +``` +${MyPath}/grafana/var_lib +``` + +*MyPath* was configured earlier in the *.env* file. + +so let´s assume the following: + +MyPath = /opt/docker/containers/ + +then you have to do the following + +``` +chown -R 472:472 /opt/docker/containers/grafana/var_lib +``` + + + + + + +## Authors + +Contributors names and contact info + +* [Sven Holz](mailto:code+latency-monitor@planet-espresso.com) + +## Version History + +* 0.1 + * Initial Release + +## License + +free to use + diff --git a/docker-compose-full_stack.yml b/docker-compose-full_stack.yml new file mode 100644 index 0000000..30e4f49 --- /dev/null +++ b/docker-compose-full_stack.yml @@ -0,0 +1,143 @@ +version: '2.4' + +services: + + # DOCKERPROXY + dockerproxy: + container_name: dockerproxy + hostname: dockerproxy + restart: always + environment: + CONTAINERS: 1 + image: tecnativa/docker-socket-proxy + labels: + - "com.centurylinklabs.watchtower.enable=true" + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - /etc/timezone:/etc/timezone:ro + - /etc/localtime:/etc/localtime:ro + networks: + - net-dockerproxy + + # TRAEFIK + traefik: + container_name: traefik + hostname: traefik + depends_on: + - dockerproxy + environment: + - TZ=Europe/Berlin + image: traefik:latest + labels: + - "traefik.docker.network=net-webproxy" + - "providers.file.filename=/dynamic_conf.yml" + - "traefik.enable=true" + ports: + - 80:80 + - 443:443 + - 8086:8086 + restart: always + volumes: + - ./traefik/dynamic_conf.yml:/dynamic_conf.yml:ro + - ./traefik/traefik.yml:/traefik.yml:ro + - ./traefik/mycert.crt:/certs/cert.crt:ro + - ./traefik/mycert.key:/certs/privkey.key:ro + - /etc/timezone:/etc/timezone:ro + - /etc/localtime:/etc/localtime:ro + networks: + - net-dockerproxy + - net-webproxy + - net-webproxy-no-inet + + + # INFLUXDB + influxdb: + container_name: influxdb + hostname: influxdb + image: influxdb + depends_on: + - traefik + restart: always + labels: + - "traefik.enable=true" + - "traefik.http.routers.influxdb-secure.entrypoints=influxdb" + - "traefik.http.routers.influxdb-secure.rule=Host(`PLACE_YOUR_FQDN_HERE`)" + - "traefik.http.routers.influxdb-secure.tls=true" + - "traefik.http.routers.influxdb-secure.service=influxdb" + - "traefik.http.routers.influxdb-secure.middlewares=secHeaders@file" + - "traefik.http.services.influxdb.loadbalancer.server.port=8086" + - "traefik.docker.network=net-webproxy-no-inet" + volumes: + - ${MyPath}/influxdb/data:/var/lib/influxdb2 + - ${MyPath}/influxdb/config:/etc/influxdb2 + - /etc/timezone:/etc/timezone:ro + - /etc/localtime:/etc/localtime:ro + environment: + - DOCKER_INFLUXDB_INIT_MODE + - DOCKER_INFLUXDB_INIT_USERNAME + - DOCKER_INFLUXDB_INIT_PASSWORD + - DOCKER_INFLUXDB_INIT_ORG + - DOCKER_INFLUXDB_INIT_BUCKET + - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN + - DOCKER_INFLUXDB_INIT_RETENTION + networks: + - net-webproxy-no-inet + + # GRAFANA + grafana: + container_name: grafana + hostname: grafana + image: grafana/grafana + depends_on: + - influxdb + restart: always + labels: + - "traefik.enable=true" + - "traefik.http.routers.grafana.entrypoints=http" + - "traefik.http.routers.grafana.rule=Host(`PLACE_YOUR_FQDN_HERE`)" + - "traefik.http.middlewares.grafana-https-redirect.redirectscheme.scheme=https" + - "traefik.http.routers.grafana.middlewares=grafana-https-redirect" + - "traefik.http.routers.grafana-secure.entrypoints=https" + - "traefik.http.routers.grafana-secure.rule=Host(`PLACE_YOUR_FQDN_HERE`)" + - "traefik.http.routers.grafana-secure.tls=true" + - "traefik.http.routers.grafana-secure.service=grafana" + - "traefik.http.routers.grafana-secure.middlewares=secHeaders@file" + - "traefik.http.services.grafana.loadbalancer.server.port=3000" + - "traefik.docker.network=net-webproxy-no-inet" + volumes: + - ./grafana/provisioning:/etc/grafana/provisioning:ro + - ${MyPath}/grafana/var_lib:/var/lib/grafana + - /etc/timezone:/etc/timezone:ro + - /etc/localtime:/etc/localtime:ro + environment: + - GF_AUTH_ANONYMOUS_ENABLED + - GF_SECURITY_ALLOW_EMBEDDING + - GF_INSTALL_PLUGINS + networks: + - net-webproxy-no-inet + +# latency-monitor + latency-monitor: + container_name: latency-monitor + hostname: latency-monitor + build: ./Docker_Build + depends_on: + - influxdb + restart: always + environment: + - INFLUX_URL + - INFLUX_TOKEN + - INFLUX_BUCKET + - INFLUX_ORG + - TARGET_HOST + networks: + - net-webproxy + - net-webproxy-no-inet + +networks: + net-dockerproxy: + external: false + net-webproxy: + external: false + net-webproxy-no-inet: + external: false \ No newline at end of file diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml new file mode 100644 index 0000000..e69de29 diff --git a/env b/env new file mode 100644 index 0000000..e6624d7 --- /dev/null +++ b/env @@ -0,0 +1,37 @@ +#################################################### +# Basic configuration options +#################################################### + +#################################################### +# PATH + CO +#################################################### + +MyPath=/YOUR_PATH_TO_CONTAINER_STATIC_DATA/ + + +#################################################### +# INFLUXDB +#################################################### +DOCKER_INFLUXDB_INIT_MODE=setup +DOCKER_INFLUXDB_INIT_USERNAME=YOUR_ADMIN_USER +DOCKER_INFLUXDB_INIT_PASSWORD=YOUR_ADMIN_PASSWORD +DOCKER_INFLUXDB_INIT_ORG=YOUR_ORGANIZATION +DOCKER_INFLUXDB_INIT_BUCKET=YOUR_BUCKET_NAME +DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=YOUR_ADMIN_TOKEN +DOCKER_INFLUXDB_INIT_RETENTION=52w + +#################################################### +# GRAFANA +#################################################### +GF_AUTH_ANONYMOUS_ENABLED=true +GF_SECURITY_ALLOW_EMBEDDING=true +GF_INSTALL_PLUGINS= + +#################################################### +# LATENCY-MONITOR +#################################################### +INFLUX_URL=YOUR_INFLUXDB_URL +INFLUX_TOKEN=YOUR_ADMIN_TOKEN +INFLUX_BUCKET=YOUR_BUCKET_NAME +INFLUX_ORG=YOUR_ORGANIZATION +TARGET_HOST=YOUR_MONITORED_TARGET \ No newline at end of file diff --git a/grafana/provisioning/datasources/grafana-datasource.yml b/grafana/provisioning/datasources/grafana-datasource.yml new file mode 100644 index 0000000..97f452c --- /dev/null +++ b/grafana/provisioning/datasources/grafana-datasource.yml @@ -0,0 +1,15 @@ +apiVersion: 1 + +datasources: + - name: influxdbv2 + type: influxdb + access: proxy + url: http://influxdb:8086 + isDefault: true + secureJsonData: + token: YOUR_ADMIN_TOKEN + jsonData: + version: Flux + organization: YOUR_ORGANIZATION + defaultBucket: YOUR_BUCKET_NAME + tlsSkipVerify: true \ No newline at end of file diff --git a/traefik/dynamic_conf.yml b/traefik/dynamic_conf.yml new file mode 100644 index 0000000..e86f676 --- /dev/null +++ b/traefik/dynamic_conf.yml @@ -0,0 +1,33 @@ +tls: + certificates: + - certFile: /certs/cert.crt + keyFile: /certs/privkey.key + stores: + - default + options: + default: + minVersion: VersionTLS12 + cipherSuites: + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + curvePreferences: + - CurveP521 + - CurveP384 + sniStrict: false +http: + middlewares: + secHeaders: + headers: + browserXssFilter: true + contentTypeNosniff: true + frameDeny: true + sslRedirect: true + #HSTS Configuration + stsIncludeSubdomains: true + stsPreload: true + stsSeconds: 31536000 + customFrameOptionsValue: "SAMEORIGIN" \ No newline at end of file diff --git a/traefik/traefik.yml b/traefik/traefik.yml new file mode 100644 index 0000000..357b743 --- /dev/null +++ b/traefik/traefik.yml @@ -0,0 +1,30 @@ +api: + dashboard: true +log: + level: ERROR +accessLog: {} +entryPoints: + http: + address: ":80" + https: + address: ":443" + influxdb: + address: ":8086" +providers: + docker: + endpoint: "tcp://dockerproxy:2375" + exposedByDefault: false + file: + filename: "/dynamic_conf.yml" +metrics: + prometheus: + entryPoint: metrics + addServicesLabels: true + addEntryPointsLabels: true + buckets: + - 0.1 + - 0.3 + - 1.2 + - 5.0 + +