From 7f5059b4236bdb94205c3ad8291f6f041ab93fb7 Mon Sep 17 00:00:00 2001 From: Pushpit Kamboj Date: Mon, 24 Aug 2026 23:56:45 +0530 Subject: [PATCH] [feature] Added configurable timeseries database backends --- .env | 15 +++ deploy/telegraf.conf | 25 +++++ docker-compose.yml | 45 ++++++++- docs/user/architecture.rst | 3 +- docs/user/settings.rst | 150 ++++++++++++++++++++++++++++- images/common/openwisp/settings.py | 49 ++++++++-- images/openwisp_base/Dockerfile | 14 +++ tests/runtests.py | 36 +++++++ 8 files changed, 324 insertions(+), 13 deletions(-) create mode 100644 deploy/telegraf.conf diff --git a/.env b/.env index 1967becc..c08f7061 100644 --- a/.env +++ b/.env @@ -15,8 +15,23 @@ VPN_DOMAIN=openvpn.openwisp.org EMAIL_DJANGO_DEFAULT=example@example.org DB_USER=admin DB_PASS=admin +TIMESERIES_BACKEND=influxdb +COMPOSE_PROFILES= +TIMESERIES_UDP_WRITES=False +TIMESERIES_UDP_PORT=8089 INFLUXDB_USER=admin INFLUXDB_PASS=admin +INFLUXDB2_USER=admin +INFLUXDB2_PASS=adminadmin +INFLUXDB2_BUCKET=openwisp +INFLUXDB2_HOST=influxdb2 +INFLUXDB2_PORT=8086 +INFLUXDB2_ORG=openwisp +INFLUXDB2_TOKEN=openwisp-token +INFLUXDB2_UDP_HOST=telegraf +ELASTICSEARCH_NAME=openwisp +ELASTICSEARCH_URL=http://elasticsearch:9200 +ELASTICSEARCH_VERSION=9.4.3 # Security DJANGO_SECRET_KEY=default_secret_key # Enable Modules diff --git a/deploy/telegraf.conf b/deploy/telegraf.conf new file mode 100644 index 00000000..7fb9a11e --- /dev/null +++ b/deploy/telegraf.conf @@ -0,0 +1,25 @@ +[agent] + # Keep data series compatible with OpenWISP records written over HTTP. + omit_hostname = true + +[[inputs.socket_listener]] + service_address = "udp://:8089" + data_format = "influx" + read_buffer_size = 8388608 + [inputs.socket_listener.tags] + bucket = "${TIMESERIES_DB}" + +[[inputs.socket_listener]] + service_address = "udp://:8090" + data_format = "influx" + read_buffer_size = 8388608 + [inputs.socket_listener.tags] + bucket = "${TIMESERIES_DB}_short" + +[[outputs.influxdb_v2]] + urls = ["http://influxdb2:8086"] + token = "${INFLUXDB2_TOKEN}" + organization = "${INFLUXDB2_ORG}" + bucket = "${TIMESERIES_DB}" + bucket_tag = "bucket" + exclude_bucket_tag = true diff --git a/docker-compose.yml b/docker-compose.yml index 62265c3e..a0120aa9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,7 +34,7 @@ services: - postgres - redis - postfix - - influxdb + - ${TIMESERIES_BACKEND:-influxdb} api: image: openwisp/openwisp-api:${OPENWISP_VERSION:-edge} @@ -214,6 +214,47 @@ services: volumes: - influxdb_data:/var/lib/influxdb + influxdb2: + image: influxdb:2.9.1-alpine + restart: always + profiles: + - influxdb2 + environment: + - DOCKER_INFLUXDB_INIT_MODE=setup + - DOCKER_INFLUXDB_INIT_USERNAME=$INFLUXDB2_USER + - DOCKER_INFLUXDB_INIT_PASSWORD=$INFLUXDB2_PASS + - DOCKER_INFLUXDB_INIT_ORG=$INFLUXDB2_ORG + - DOCKER_INFLUXDB_INIT_BUCKET=$INFLUXDB2_BUCKET + - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=$INFLUXDB2_TOKEN + volumes: + - influxdb2_data:/var/lib/influxdb2 + + telegraf: + image: telegraf:1.31-alpine + restart: always + profiles: + - influxdb2 + environment: + - TIMESERIES_DB=$INFLUXDB2_BUCKET + - INFLUXDB2_ORG=$INFLUXDB2_ORG + - INFLUXDB2_TOKEN=$INFLUXDB2_TOKEN + volumes: + - ./deploy/telegraf.conf:/etc/telegraf/telegraf.conf:ro + depends_on: + - influxdb2 + + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION:-9.4.3} + restart: always + profiles: + - elasticsearch + environment: + - discovery.type=single-node + - xpack.security.enabled=false + - ES_JAVA_OPTS=-Xms512m -Xmx512m + volumes: + - elasticsearch_data:/usr/share/elasticsearch/data + redis: image: redis:alpine restart: always @@ -222,6 +263,8 @@ services: volumes: influxdb_data: {} + influxdb2_data: {} + elasticsearch_data: {} postgres_data: {} redis_data: {} openwisp_certs: {} diff --git a/docs/user/architecture.rst b/docs/user/architecture.rst index b35ee794..0986475d 100644 --- a/docs/user/architecture.rst +++ b/docs/user/architecture.rst @@ -25,7 +25,8 @@ repository. updating configurations of your device. - **openwisp-celery-monitoring**: Runs background tasks that perform active monitoring checks, e.g. ping checks and configuration checks. It - also executes task for writing monitoring data to the timeseries DB. + also executes tasks for writing monitoring data to the configurable + timeseries DB, which defaults to InfluxDB 1.8. - **openwisp-celerybeat**: Runs periodic background tasks. e.g. revoking all the expired certificates. - **openwisp-nginx**: Internet facing container that facilitates all the diff --git a/docs/user/settings.rst b/docs/user/settings.rst index a2425c8c..4916b9c4 100644 --- a/docs/user/settings.rst +++ b/docs/user/settings.rst @@ -16,7 +16,10 @@ Additionally, you can search for the following prefixes: - ``OPENWISP_``: OpenWISP application settings. - ``DB_``: PostgreSQL Database settings. +- ``TIMESERIES_``: Timeseries database backend settings. - ``INFLUXDB_``: InfluxDB settings. +- ``INFLUXDB2_``: InfluxDB 2.x settings. +- ``ELASTICSEARCH_``: Elasticsearch settings. - ``DJANGO_``: Django settings. - ``EMAIL_``: Email settings (see also ``POSTFIX_``). - ``POSTFIX_``: Postfix settings (see also ``EMAIL_``). @@ -642,11 +645,69 @@ PostgreSQL Database `__. - **Default:** ``django.contrib.gis.db.backends.postgis`` -InfluxDB --------- +Timeseries Database +------------------- + +InfluxDB 1.8 is the default timeseries database used by the +:doc:`Monitoring module `. InfluxDB 2.x and +Elasticsearch are available as opt-in backends. + +To use the default InfluxDB 1.8 backend: + +.. code-block:: bash + + TIMESERIES_BACKEND=influxdb + +To use InfluxDB 2.x: + +.. code-block:: bash + + TIMESERIES_BACKEND=influxdb2 + COMPOSE_PROFILES=influxdb2 + +To use Elasticsearch: + +.. code-block:: bash + + TIMESERIES_BACKEND=elasticsearch + COMPOSE_PROFILES=elasticsearch + +When using an optional backend, both ``TIMESERIES_BACKEND`` and +``COMPOSE_PROFILES`` must be set to the same backend name. + +``COMPOSE_PROFILES`` +~~~~~~~~~~~~~~~~~~~~ + +- **Explanation:** Docker Compose profiles to enable. Set this to + ``influxdb2`` or ``elasticsearch`` when using one of the optional + timeseries backend services. +- **Valid Values:** A comma-separated list of Docker Compose profile names. +- **Default:** ``""`` (empty string). + +``TIMESERIES_BACKEND`` +~~~~~~~~~~~~~~~~~~~~~~ -InfluxDB is the default time series database used by the :doc:`Monitoring -module `. +- **Explanation:** Timeseries database backend used by OpenWISP Monitoring. +- **Valid Values:** ``influxdb``, ``influxdb2``, ``elasticsearch``. +- **Default:** ``influxdb``. + +``TIMESERIES_UDP_WRITES`` +~~~~~~~~~~~~~~~~~~~~~~~~~ + +- **Explanation:** Whether to write timeseries data over UDP. InfluxDB 2.x + UDP writes are handled by the ``telegraf`` container because InfluxDB 2.x + does not support UDP writes natively. Elasticsearch does not support UDP + writes. +- **Valid Values:** ``True``, ``False``. +- **Default:** ``False``. + +``TIMESERIES_UDP_PORT`` +~~~~~~~~~~~~~~~~~~~~~~~ + +- **Explanation:** UDP port used for timeseries writes. For InfluxDB 2.x, + this is the Telegraf listener port. +- **Valid Values:** INTEGER. +- **Default:** ``8089``. ``INFLUXDB_USER`` ~~~~~~~~~~~~~~~~~ @@ -692,6 +753,87 @@ module `. - **Valid Values:** STRING. - **Default:** ``26280h0m0s`` (3 years). +``INFLUXDB2_USER`` +~~~~~~~~~~~~~~~~~~ + +- **Explanation:** Username used to initialize the InfluxDB 2.x Docker + container. +- **Valid Values:** STRING. +- **Default:** ``admin``. + +``INFLUXDB2_PASS`` +~~~~~~~~~~~~~~~~~~ + +- **Explanation:** Password used to initialize the InfluxDB 2.x Docker + container. +- **Valid Values:** STRING. +- **Default:** ``adminadmin``. + +``INFLUXDB2_BUCKET`` +~~~~~~~~~~~~~~~~~~~~ + +- **Explanation:** Bucket used by the InfluxDB 2.x backend. +- **Valid Values:** STRING. +- **Default:** ``openwisp``. + +``INFLUXDB2_HOST`` +~~~~~~~~~~~~~~~~~~ + +- **Explanation:** Host to be used when connecting to InfluxDB 2.x. +- **Valid Values:** any valid hostname or IP address. +- **Default:** ``influxdb2``. + +``INFLUXDB2_PORT`` +~~~~~~~~~~~~~~~~~~ + +- **Explanation:** Port on which InfluxDB 2.x is listening. +- **Valid Values:** INTEGER. +- **Default:** ``8086``. + +``INFLUXDB2_ORG`` +~~~~~~~~~~~~~~~~~ + +- **Explanation:** InfluxDB 2.x organization used by OpenWISP Monitoring. +- **Valid Values:** STRING. +- **Default:** ``openwisp``. + +``INFLUXDB2_TOKEN`` +~~~~~~~~~~~~~~~~~~~ + +- **Explanation:** InfluxDB 2.x API token used by OpenWISP Monitoring. +- **Valid Values:** STRING. +- **Default:** ``openwisp-token``. + +``INFLUXDB2_UDP_HOST`` +~~~~~~~~~~~~~~~~~~~~~~ + +- **Explanation:** Hostname of the Telegraf UDP listener used for InfluxDB + 2.x UDP writes. +- **Valid Values:** any valid hostname or IP address. +- **Default:** ``telegraf``. + +``ELASTICSEARCH_NAME`` +~~~~~~~~~~~~~~~~~~~~~~ + +- **Explanation:** Name used by OpenWISP Monitoring for Elasticsearch data + streams. +- **Valid Values:** STRING. +- **Default:** ``openwisp``. + +``ELASTICSEARCH_URL`` +~~~~~~~~~~~~~~~~~~~~~ + +- **Explanation:** URL used to connect to Elasticsearch. +- **Valid Values:** URL. +- **Default:** ``http://elasticsearch:9200``. + +``ELASTICSEARCH_VERSION`` +~~~~~~~~~~~~~~~~~~~~~~~~~ + +- **Explanation:** Elasticsearch Docker image version. +- **Valid Values:** Any valid Elasticsearch Docker image tag. +- **Default:** ``9.4.3``. + Postfix ------- diff --git a/images/common/openwisp/settings.py b/images/common/openwisp/settings.py index 048de604..b8826f13 100644 --- a/images/common/openwisp/settings.py +++ b/images/common/openwisp/settings.py @@ -5,6 +5,7 @@ from urllib.parse import quote import tldextract +from django.core.exceptions import ImproperlyConfigured from openwisp.utils import ( env_bool, is_string_env_bool, @@ -209,14 +210,48 @@ }, } -TIMESERIES_DATABASE = { - "BACKEND": "openwisp_monitoring.db.backends.influxdb", - "USER": os.environ["INFLUXDB_USER"], - "PASSWORD": os.environ["INFLUXDB_PASS"], - "NAME": os.environ["INFLUXDB_NAME"], - "HOST": os.environ["INFLUXDB_HOST"], - "PORT": os.environ["INFLUXDB_PORT"], +TIMESERIES_BACKEND = os.environ.get("TIMESERIES_BACKEND", "influxdb") +TIMESERIES_UDP_WRITES = env_bool(os.environ.get("TIMESERIES_UDP_WRITES", "False")) +TIMESERIES_UDP_PORT = int(os.environ.get("TIMESERIES_UDP_PORT", 8089)) +TIMESERIES_DATABASE_OPTIONS = { + "udp_writes": TIMESERIES_UDP_WRITES, + "udp_port": TIMESERIES_UDP_PORT, } +if TIMESERIES_BACKEND == "influxdb": + TIMESERIES_DATABASE = { + "BACKEND": "openwisp_monitoring.db.backends.influxdb", + "USER": os.environ["INFLUXDB_USER"], + "PASSWORD": os.environ["INFLUXDB_PASS"], + "NAME": os.environ["INFLUXDB_NAME"], + "HOST": os.environ["INFLUXDB_HOST"], + "PORT": os.environ["INFLUXDB_PORT"], + "OPTIONS": TIMESERIES_DATABASE_OPTIONS, + } +elif TIMESERIES_BACKEND == "influxdb2": + TIMESERIES_DATABASE = { + "BACKEND": "openwisp_monitoring.db.backends.influxdb2", + "NAME": os.environ["INFLUXDB2_BUCKET"], + "USER": os.environ["INFLUXDB2_ORG"], + "PASSWORD": os.environ["INFLUXDB2_TOKEN"], + "URL": os.environ.get( + "INFLUXDB2_URL", + f'http://{os.environ["INFLUXDB2_HOST"]}:{os.environ["INFLUXDB2_PORT"]}', + ), + "OPTIONS": { + **TIMESERIES_DATABASE_OPTIONS, + "udp_host": os.environ["INFLUXDB2_UDP_HOST"], + }, + } +elif TIMESERIES_BACKEND == "elasticsearch": + TIMESERIES_DATABASE = { + "BACKEND": "openwisp_monitoring.db.backends.elasticsearch", + "NAME": os.environ["ELASTICSEARCH_NAME"], + "URL": os.environ["ELASTICSEARCH_URL"], + } +else: + raise ImproperlyConfigured( + f'Unsupported TIMESERIES_BACKEND "{TIMESERIES_BACKEND}".' + ) OPENWISP_MONITORING_DEFAULT_RETENTION_POLICY = os.environ[ "INFLUXDB_DEFAULT_RETENTION_POLICY" ] diff --git a/images/openwisp_base/Dockerfile b/images/openwisp_base/Dockerfile index 04627047..492f67f7 100644 --- a/images/openwisp_base/Dockerfile +++ b/images/openwisp_base/Dockerfile @@ -118,12 +118,26 @@ ENV DASHBOARD_APP_SERVICE=dashboard \ DB_SSLCERT=None \ DB_SSLROOTCERT=None \ DB_OPTIONS={} \ + TIMESERIES_BACKEND=influxdb \ + TIMESERIES_UDP_WRITES=False \ + TIMESERIES_UDP_PORT=8089 \ INFLUXDB_USER=admin \ INFLUXDB_PASS=admin \ INFLUXDB_NAME=openwisp \ INFLUXDB_HOST=influxdb \ INFLUXDB_PORT=8086 \ INFLUXDB_DEFAULT_RETENTION_POLICY=26280h0m0s \ + INFLUXDB2_USER=admin \ + INFLUXDB2_PASS=adminadmin \ + INFLUXDB2_BUCKET=openwisp \ + INFLUXDB2_HOST=influxdb2 \ + INFLUXDB2_PORT=8086 \ + INFLUXDB2_ORG=openwisp \ + INFLUXDB2_TOKEN=openwisp-token \ + INFLUXDB2_UDP_HOST=telegraf \ + ELASTICSEARCH_NAME=openwisp \ + ELASTICSEARCH_URL=http://elasticsearch:9200 \ + ELASTICSEARCH_VERSION=9.4.3 \ EMAIL_BACKEND=djcelery_email.backends.CeleryEmailBackend \ EMAIL_HOST=postfix \ EMAIL_HOST_PORT=25 \ diff --git a/tests/runtests.py b/tests/runtests.py index c78f9e92..90bb6d43 100644 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -494,6 +494,42 @@ def test_containers_down(self): class TestLocalUtils(BaseTestUtils, unittest.TestCase): """Tests for local utilities""" + def _docker_compose_services(self, **environment): + env = os.environ.copy() + env.update(environment) + result = subprocess.run( + ["docker", "compose", "config", "--services"], + cwd=self.root_location, + check=False, + capture_output=True, + text=True, + env=env, + ) + self.assertEqual(result.returncode, 0, result.stderr) + return set(result.stdout.splitlines()) + + def test_default_compose_timeseries_backend(self): + services = self._docker_compose_services( + TIMESERIES_BACKEND="influxdb", COMPOSE_PROFILES="" + ) + self.assertIn("influxdb", services) + self.assertNotIn("influxdb2", services) + self.assertNotIn("telegraf", services) + self.assertNotIn("elasticsearch", services) + + def test_influxdb2_compose_profile(self): + services = self._docker_compose_services( + TIMESERIES_BACKEND="influxdb2", COMPOSE_PROFILES="influxdb2" + ) + self.assertIn("influxdb2", services) + self.assertIn("telegraf", services) + + def test_elasticsearch_compose_profile(self): + services = self._docker_compose_services( + TIMESERIES_BACKEND="elasticsearch", COMPOSE_PROFILES="elasticsearch" + ) + self.assertIn("elasticsearch", services) + def test_update_version_updates_only_version_file(self): repository_root = Path(__file__).resolve().parents[1] makefile_content = (