diff options
| -rw-r--r-- | .gitignore | 2 | ||||
| -rw-r--r-- | README.md | 18 | ||||
| -rw-r--r-- | docker-compose.yml | 21 | ||||
| -rw-r--r-- | mqtt-client/Dockerfile | 2 | ||||
| -rw-r--r-- | mqtt-client/mqtt-client.py | 60 | ||||
| -rw-r--r-- | mqtt-client/requirements.txt | 1 | ||||
| -rw-r--r-- | prometheus/prometheus.yml | 16 | ||||
| -rw-r--r-- | switch-snmp/Dockerfile | 2 | ||||
| -rw-r--r-- | switch-snmp/mikrotik.py | 28 | ||||
| -rw-r--r-- | switch-snmp/requirements.txt | 3 | ||||
| -rw-r--r-- | switch-snmp/switches.py | 57 |
11 files changed, 171 insertions, 39 deletions
@@ -7,6 +7,8 @@ config.env influxdb-config/ influxdb-data/ node-red-data/ +prometheus/web.yml +prometheus/web.yml # Byte-compiled / optimized / DLL files __pycache__/ @@ -1,13 +1,11 @@ -# power.eda.gay +# SNMP and MQTT Power Logger & Visualizer -Logs Tasmota-flashed power usage monitors, and TP-Link Omada POE switches, to InfluxDB and Grafana using MQTT and SNMP. +Logs Tasmota-flashed power usage monitors, and TP-Link Omada/Mikrotik POE switches, to InfluxDB and Grafana using MQTT, SNMP and prometheus. Also logs Zigbee informtion with a Tasmota-flashed Zigbee bridge. -Looking for the Mikrotik POE usage monitor/exporter? That's been moved to [MikrotikPOEPowerExporter](https://github.com/jwansek/MikrotikPOEPowerExporter) +Looking for the Mikrotik POE usage monitor/exporter? That's in [`mikrotik.py`](/switch-snmp/mikrotik.py) - - - + ## Setup @@ -29,3 +27,11 @@ Looking for the Mikrotik POE usage monitor/exporter? That's been moved to [Mikro You must enable SNMP in the Omada controller with the community string `tplink`:  + +Moreover mikrotik switches must be set up with an appropriate SSH key pair so they can be polled through SSH + +## MQTT setup + +We are using a [Tasmota-flashed zigbee coordinator](https://www.aliexpress.com/item/1005005254486268.html) to transmit zigbee messages to our MQTT castor, and [Tasmota-flashed plugs](https://www.aliexpress.com/item/1005008427641332.htm) for logging power from the wall over MQTT. Both must be configured with an appropriate friendlyname and told access the MQTT castor. + +  diff --git a/docker-compose.yml b/docker-compose.yml index 1775b7e..fe8fd2e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,6 +35,7 @@ services: - ./config.env depends_on: - influxdb + - pushgateway restart: unless-stopped snmp_client: @@ -46,6 +47,7 @@ services: - ./config.env depends_on: - influxdb + - pushgateway restart: unless-stopped volumes: - ./switch-snmp/omada-switches.conf:/app/omada-switches.conf @@ -56,12 +58,31 @@ services: image: grafana/grafana:8.5.27 ports: - 3000:3000 + environment: + - GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=grafana-sankey-panel depends_on: - influxdb + - prometheus restart: unless-stopped volumes: - grafana:/var/lib/grafana + prometheus: + image: prom/prometheus + volumes: + - ./prometheus/:/etc/prometheus/ + command: + - --config.file=/etc/prometheus/prometheus.yml + - --web.config.file=/etc/prometheus/web.yml + ports: + - 9090:9090 + restart: unless-stopped + + pushgateway: + image: prom/pushgateway + restart: unless-stopped + + volumes: mosquitto-data: mosquitto-logs: diff --git a/mqtt-client/Dockerfile b/mqtt-client/Dockerfile index ad59f9b..12c8a53 100644 --- a/mqtt-client/Dockerfile +++ b/mqtt-client/Dockerfile @@ -6,5 +6,7 @@ RUN apt-get install -y python3-pip iputils-ping COPY . /app WORKDIR /app RUN pip3 install -r requirements.txt +RUN pip3 install docker +RUN pip3 install -r TasmotaCLI/requirements.txt ENTRYPOINT ["python3"] CMD ["mqtt-client.py"] diff --git a/mqtt-client/mqtt-client.py b/mqtt-client/mqtt-client.py index 18dcd82..37ed8cb 100644 --- a/mqtt-client/mqtt-client.py +++ b/mqtt-client/mqtt-client.py @@ -1,7 +1,9 @@ import paho.mqtt.client as paho from influxdb_client import InfluxDBClient, Point, WritePrecision from influxdb_client.client.write_api import SYNCHRONOUS +import prometheus_client import threading +import asyncio import time import json import sys @@ -9,6 +11,7 @@ import os sys.path.insert(1, os.path.join(os.path.dirname(__file__), "TasmotaCLI")) import tasmotaMQTTClient +import tasmotaHTTPClient class MQTTClient: def __init__(self, mqtt_client_name = "reg.reaweb.uk/mqtt-client", loop_forever = True): @@ -18,6 +21,32 @@ class MQTTClient: org = os.environ["DOCKER_INFLUXDB_INIT_ORG"] ) self.influxc.ping() + self.tasmota_power_prom = prometheus_client.Gauge( + "tasmota_power", + "Power metrics as reported by Tasmota-flashed plugs", + labelnames = ["plug", "field"] + ) + self.humidity_prom = prometheus_client.Gauge( + "humidity", + "Humidity as reported by a zigbee device over MQTT", + labelnames = ["location"] + ) + self.temperature_prom = prometheus_client.Gauge( + "temperature", + "Temperature as reported by a zigbee device over MQTT", + labelnames = ["location"] + ) + self.doorsensor_prom = prometheus_client.Enum( + "door_sensor", + "Door sensor state change as reported by zigbee door sensor over MQTT", + states = ["opened", "closed"], + labelnames = ["location"] + ) + self.door_opened_counter = prometheus_client.Counter( + "door_opened", + "Door sensor opened as reported by zigbee door sensor over MQTT", + labelnames = ["location"] + ) self.mqttc = paho.Client(mqtt_client_name, clean_session = True) if loop_forever: @@ -49,11 +78,15 @@ class MQTTClient: elif type_ == "TasmotaZigbee": self.handle_zigbee(msg_j) + def handle_plug(self, msg_j, location): print("'%s' is using %.1fw @ %s. %.1fkWh so far today, %.1fkWh yesterday" % (location, msg_j["ENERGY"]["Power"], msg_j["Time"], msg_j["ENERGY"]["Today"], msg_j["ENERGY"]["Yesterday"])) fields = {k: v for k, v in msg_j["ENERGY"].items() if k not in {"TotalStartTime"}} self.append_influxdb(fields, "tasmota_power", {"plug": location}) + for k, v in fields.items(): + self.tasmota_power_prom.labels(plug = location, field = k).set(v) + def handle_zigbee(self, msg_j): def toggle_geoffery(): print("Starting thread...") @@ -61,26 +94,46 @@ class MQTTClient: print("Waiting...") time.sleep(8) tasmotaMQTTClient.MQTTClient(MQTT_HOST, "TasmotaGeoffery", os.environ["MQTT_USER"], os.environ["MQTT_PASSWD"], "ON") + print("Toggled again.") zigbee_id = list(msg_j["ZbReceived"].keys())[0] fields = msg_j["ZbReceived"][zigbee_id] friendlyname = fields.pop("Name") del fields["Device"] print("Zigbee device '%s' reported: %s" % (friendlyname, str(fields))) - if "Read" not in fields.keys(): - self.append_influxdb(fields, "zigbee", {"friendlyname": friendlyname, "id": zigbee_id}) if zigbee_id == "0x7327" and friendlyname == "TVButton" and "Power" in fields.keys(): if fields["Power"] == 2: print("TV Zigbee button pressed, toggling TasmotaTV Tasmota Plug") self.toggle_plug("TasmotaTV") threading.Thread(target = toggle_geoffery, args = ()).start() + #loop = asyncio.get_event_loop() + #loop.run_until_complete(tasmotaHTTPClient.main(host = "geoffery.plug", username = "admin", password = os.environ["MQTT_PASSWD"], toggle = True)) + #time.sleep(8) + #loop.run_until_complete(tasmotaHTTPClient.main(host = "geoffery.plug", username = "admin", password = os.environ["MQTT_PASSWD"], toggle = True)) + if zigbee_id == "0x74B3" and friendlyname == "HarveyButton" and "Power" in fields.keys(): if fields["Power"] == 2: print("Harvey's button pressed, toggling TasmotaHarveyPC Plug") self.toggle_plug("TasmotaHarveyPC") + if "Humidity" in fields.keys(): + fields["Humidity"] = float(fields["Humidity"]) + self.humidity_prom.labels(location = friendlyname).set(fields["Humidity"]) + elif "Temperature" in fields.keys(): + fields["Temperature"] = float(fields["Temperature"]) + self.temperature_prom.labels(location = friendlyname).set(fields["Temperature"]) + elif "ZoneStatus" in fields.keys() and "Contact" in fields.keys(): + if fields["ZoneStatus"] == 1 and fields["Contact"] == 1: + self.doorsensor_prom.labels(location = friendlyname).state("opened") + self.door_opened_counter.labels(location = friendlyname).inc() + elif fields["ZoneStatus"] == 0 and fields["Contact"] == 0: + self.doorsensor_prom.labels(location = friendlyname).state("closed") + + if "Read" not in fields.keys(): + self.append_influxdb(fields, "zigbee", {"friendlyname": friendlyname, "id": zigbee_id}) + def set_plug(self, friendlyname, payload): t = "cmnd/TasmotaPlug/%s/Power" % friendlyname self.mqttc.publish(t, payload = payload) @@ -106,8 +159,11 @@ if __name__ == "__main__": dotenv.load_dotenv(dotenv_path = env_path) INFLUXDB_HOST = "dns.athome" MQTT_HOST = "dns.athome" + PROM_HOST = "dns.athome" else: INFLUXDB_HOST = "influxdb" MQTT_HOST = "mqtt" + PROM_HOST = "prometheus" + prometheus_client.start_http_server(8000) mqtt_client = MQTTClient() diff --git a/mqtt-client/requirements.txt b/mqtt-client/requirements.txt index ac151c7..3e6f8db 100644 --- a/mqtt-client/requirements.txt +++ b/mqtt-client/requirements.txt @@ -1,3 +1,4 @@ paho-mqtt==1.6.1 python-dotenv influxdb-client +prometheus-client diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml new file mode 100644 index 0000000..48bd2f6 --- /dev/null +++ b/prometheus/prometheus.yml @@ -0,0 +1,16 @@ +global: + scrape_interval: 10s +scrape_configs: + - job_name: prometheus + static_configs: + - targets: + - prometheus:9090 + - job_name: pushgateway + static_configs: + - targets: + - pushgateway:9091 + - job_name: mqtt + static_configs: + - targets: + - mqtt_client:8000 + diff --git a/switch-snmp/Dockerfile b/switch-snmp/Dockerfile index d09e341..9e12d5e 100644 --- a/switch-snmp/Dockerfile +++ b/switch-snmp/Dockerfile @@ -12,6 +12,6 @@ RUN pip3 install -r requirements.txt RUN rm "privateMibs(20220831).zip" RUN echo "*/1 * * * * root python3 /app/switches.py > /proc/1/fd/1 2>/proc/1/fd/2" > /etc/crontab -RUN echo "*/1 * * * * root sh -c 'sleep 30 && python3 /app/switches.py' > /proc/1/fd/1 2>/proc/1/fd/2" >> /etc/crontab +# RUN echo "*/1 * * * * root sh -c 'sleep 30 && python3 /app/switches.py' > /proc/1/fd/1 2>/proc/1/fd/2" >> /etc/crontab ENTRYPOINT ["bash"] CMD ["entrypoint.sh"] diff --git a/switch-snmp/mikrotik.py b/switch-snmp/mikrotik.py index 29bc797..8493675 100644 --- a/switch-snmp/mikrotik.py +++ b/switch-snmp/mikrotik.py @@ -8,9 +8,6 @@ import time import os import re -from influxdb_client import InfluxDBClient, Point, WritePrecision -from influxdb_client.client.write_api import SYNCHRONOUS - logging.basicConfig( format = "%(levelname)s\t[%(asctime)s]\t%(message)s", level = logging.INFO, @@ -121,30 +118,6 @@ def print_points(points): measurement["fields"]["tpPoeVoltage"], )) -def append(points): - env_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "config.env") - if os.path.exists(env_path): - import dotenv - dotenv.load_dotenv(dotenv_path = env_path) - INFLUXDB_HOST = "dns.athome" - else: - INFLUXDB_HOST = "influxdb" - - influxc = InfluxDBClient( - url = "http://%s:8086" % INFLUXDB_HOST, - token = os.environ["DOCKER_INFLUXDB_INIT_ADMIN_TOKEN"], - org = os.environ["DOCKER_INFLUXDB_INIT_ORG"] - ) - influxc.ping() - - write_api = influxc.write_api(write_options = SYNCHRONOUS) - write_api.write( - os.environ["DOCKER_INFLUXDB_INIT_BUCKET"], - os.environ["DOCKER_INFLUXDB_INIT_ORG"], - points, - write_precision = WritePrecision.S - ) - if __name__ == "__main__": if not os.path.exists(os.path.join(os.path.dirname(__file__), "mikrotik-switches.conf")): raise FileNotFoundError("Couldn't find mikrotik config file") @@ -154,5 +127,4 @@ if __name__ == "__main__": import json points = get_points() print(json.dumps(points, indent = 4)) - append(points) diff --git a/switch-snmp/requirements.txt b/switch-snmp/requirements.txt index aaf744d..aeb5843 100644 --- a/switch-snmp/requirements.txt +++ b/switch-snmp/requirements.txt @@ -1,4 +1,5 @@ python-dotenv influxdb-client pandas -fabric
\ No newline at end of file +fabric +prometheus-client
\ No newline at end of file diff --git a/switch-snmp/switches.py b/switch-snmp/switches.py index aee56d2..1bc43c2 100644 --- a/switch-snmp/switches.py +++ b/switch-snmp/switches.py @@ -1,7 +1,62 @@ +import prometheus_client import snmpOmada import mikrotik +import os + +from influxdb_client import InfluxDBClient, Point, WritePrecision +from influxdb_client.client.write_api import SYNCHRONOUS + +def append(points): + influxc = InfluxDBClient( + url = "http://%s:8086" % INFLUXDB_HOST, + token = os.environ["DOCKER_INFLUXDB_INIT_ADMIN_TOKEN"], + org = os.environ["DOCKER_INFLUXDB_INIT_ORG"] + ) + influxc.ping() + + for measurement in points: + for field in measurement["fields"].keys(): + try: + float(measurement["fields"][field]) + except ValueError: + continue + else: + switch_power.labels( + field = field, + type = measurement["tags"]["type"], + port = str(measurement["tags"]["port"]), + port_name = measurement["tags"]["port_name"], + host = measurement["tags"]["switch_host"] + ).set(float(measurement["fields"][field])) + prometheus_client.push_to_gateway("%s:9091" % PUSHGATEWAY_HOST, job = "switchSNMP", registry = registry) + + write_api = influxc.write_api(write_options = SYNCHRONOUS) + write_api.write( + os.environ["DOCKER_INFLUXDB_INIT_BUCKET"], + os.environ["DOCKER_INFLUXDB_INIT_ORG"], + points, + write_precision = WritePrecision.S + ) if __name__ == "__main__": + env_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "config.env") + if os.path.exists(env_path): + import dotenv + dotenv.load_dotenv(dotenv_path = env_path) + INFLUXDB_HOST = "dns.athome" + PUSHGATEWAY_HOST = "dns.athome" + else: + INFLUXDB_HOST = "influxdb" + PUSHGATEWAY_HOST = "pushgateway" + + registry = prometheus_client.CollectorRegistry() + switch_power = prometheus_client.Gauge( + "switch_power", + "POE switch power usage metrics from Omada and Mikrotik switches, using Omada SNMP names", + labelnames = ["field", "type", "port", "port_name", "host"], + registry = registry + ) + points = snmpOmada.get_points() + mikrotik.get_points() mikrotik.print_points(points) - mikrotik.append(points)
\ No newline at end of file + append(points) |