From 6b7dc1840de958af0315c24d82df8ccbecd13ce4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bit=C3=B3=20Attila?= Date: Wed, 5 Aug 2026 23:56:11 +0200 Subject: [PATCH] Add thermo history Helm chart --- Chart.yaml | 6 ++ README.md | 25 ++++++ templates/_helpers.tpl | 24 ++++++ templates/external-secrets.yaml | 52 +++++++++++ templates/postgres-init-configmap.yaml | 72 ++++++++++++++++ templates/postgres-service.yaml | 14 +++ templates/postgres-statefulset.yaml | 63 ++++++++++++++ templates/writer-configmap.yaml | 115 +++++++++++++++++++++++++ templates/writer-deployment.yaml | 54 ++++++++++++ values.yaml | 48 +++++++++++ 10 files changed, 473 insertions(+) create mode 100644 Chart.yaml create mode 100644 README.md create mode 100644 templates/_helpers.tpl create mode 100644 templates/external-secrets.yaml create mode 100644 templates/postgres-init-configmap.yaml create mode 100644 templates/postgres-service.yaml create mode 100644 templates/postgres-statefulset.yaml create mode 100644 templates/writer-configmap.yaml create mode 100644 templates/writer-deployment.yaml create mode 100644 values.yaml diff --git a/Chart.yaml b/Chart.yaml new file mode 100644 index 0000000..ca07c81 --- /dev/null +++ b/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: thermo +description: Historical storage for home sensor events +type: application +version: 0.1.0 +appVersion: "1.0.0" diff --git a/README.md b/README.md new file mode 100644 index 0000000..0d23e8e --- /dev/null +++ b/README.md @@ -0,0 +1,25 @@ +# Thermo + +Helm chart for storing home sensor events from Kafka in PostgreSQL. + +The chart deploys: + +- PostgreSQL with persistent storage; +- a generic event and measurement schema suitable for WindFree and BLE sensors; +- a read-only database role for Grafana; +- a lightweight Redpanda Connect consumer for `application.thermo.events`; +- External Secrets backed by the existing AWS SSM secret store. + +## Required SSM parameters + +- `/vulcan/platform-core/thermo-postgres/writer-password` +- `/vulcan/platform-core/thermo-postgres/reader-password` + +## Validation + +```shell +helm lint . +helm template thermo . --namespace platform-core +``` + +The Grafana datasource and dashboards are intentionally managed outside this chart because Grafana runs in the `monitoring` namespace. diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl new file mode 100644 index 0000000..6ab37fc --- /dev/null +++ b/templates/_helpers.tpl @@ -0,0 +1,24 @@ +{{- define "thermo.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{- define "thermo.fullname" -}} +{{- default (include "thermo.name" .) .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{- define "thermo.labels" -}} +app.kubernetes.io/name: {{ include "thermo.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" }} +{{- end }} + +{{- define "thermo.postgresSelectorLabels" -}} +app: {{ include "thermo.fullname" . }}-postgres +{{- end }} + +{{- define "thermo.writerSelectorLabels" -}} +app.kubernetes.io/name: {{ include "thermo.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/component: writer +{{- end }} diff --git a/templates/external-secrets.yaml b/templates/external-secrets.yaml new file mode 100644 index 0000000..ac8e2e0 --- /dev/null +++ b/templates/external-secrets.yaml @@ -0,0 +1,52 @@ +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: {{ include "thermo.fullname" . }}-postgres-secret + labels: + {{- include "thermo.labels" . | nindent 4 }} +spec: + refreshInterval: {{ .Values.externalSecrets.refreshInterval }} + secretStoreRef: + name: {{ .Values.externalSecrets.secretStore.name }} + kind: {{ .Values.externalSecrets.secretStore.kind }} + target: + name: {{ include "thermo.fullname" . }}-postgres-secret + creationPolicy: Owner + template: + type: Opaque + data: + POSTGRES_DB: {{ .Values.postgres.database | quote }} + POSTGRES_USER: {{ .Values.postgres.writerUser | quote }} + POSTGRES_PASSWORD: {{ "{{ .writer_password }}" | quote }} + GRAFANA_USER: {{ .Values.postgres.readerUser | quote }} + GRAFANA_PASSWORD: {{ "{{ .reader_password }}" | quote }} + data: + - secretKey: writer_password + remoteRef: + key: {{ .Values.externalSecrets.writerPasswordKey }} + - secretKey: reader_password + remoteRef: + key: {{ .Values.externalSecrets.readerPasswordKey }} +--- +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: {{ include "thermo.fullname" . }}-postgres-redpanda + labels: + {{- include "thermo.labels" . | nindent 4 }} +spec: + refreshInterval: {{ .Values.externalSecrets.refreshInterval }} + secretStoreRef: + name: {{ .Values.externalSecrets.secretStore.name }} + kind: {{ .Values.externalSecrets.secretStore.kind }} + target: + name: {{ include "thermo.fullname" . }}-postgres-redpanda + creationPolicy: Owner + template: + type: Opaque + data: + THERMO_POSTGRES_DSN: {{ printf "postgres://%s:%s@%s-postgres.%s.svc.cluster.local:5432/%s?sslmode=disable" .Values.postgres.writerUser "{{ .writer_password }}" (include "thermo.fullname" .) .Release.Namespace .Values.postgres.database | quote }} + data: + - secretKey: writer_password + remoteRef: + key: {{ .Values.externalSecrets.writerPasswordKey }} diff --git a/templates/postgres-init-configmap.yaml b/templates/postgres-init-configmap.yaml new file mode 100644 index 0000000..47de863 --- /dev/null +++ b/templates/postgres-init-configmap.yaml @@ -0,0 +1,72 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "thermo.fullname" . }}-postgres-init + labels: + {{- include "thermo.labels" . | nindent 4 }} +data: + 01-schema.sql: | + CREATE TABLE sensor_events ( + event_id uuid PRIMARY KEY, + observed_at timestamptz NOT NULL, + ingested_at timestamptz NOT NULL, + source text NOT NULL, + event_type text NOT NULL, + severity text NOT NULL, + collector_id text, + device_id text NOT NULL, + raw_event jsonb NOT NULL, + stored_at timestamptz NOT NULL DEFAULT now() + ); + + CREATE TABLE sensor_measurements ( + event_id uuid NOT NULL REFERENCES sensor_events(event_id) ON DELETE CASCADE, + observed_at timestamptz NOT NULL, + source text NOT NULL, + collector_id text, + device_id text NOT NULL, + metric_name text NOT NULL, + numeric_value double precision, + text_value text, + unit text, + source_updated_at timestamptz, + PRIMARY KEY (event_id, metric_name), + CHECK (numeric_value IS NOT NULL OR text_value IS NOT NULL) + ); + + CREATE INDEX sensor_events_device_time_idx + ON sensor_events (device_id, observed_at DESC); + CREATE INDEX sensor_measurements_metric_device_time_idx + ON sensor_measurements (metric_name, device_id, observed_at DESC); + CREATE INDEX sensor_measurements_time_idx + ON sensor_measurements (observed_at DESC); + + CREATE VIEW grafana_sensor_measurements AS + SELECT + observed_at AS time, + source, + collector_id, + device_id, + metric_name, + numeric_value, + text_value, + unit + FROM sensor_measurements; + + 02-reader.sh: | + #!/bin/sh + set -eu + + psql --set ON_ERROR_STOP=1 \ + --username "$POSTGRES_USER" \ + --dbname "$POSTGRES_DB" \ + --set reader_user="$GRAFANA_USER" \ + --set reader_password="$GRAFANA_PASSWORD" <<'SQL' + SELECT format('CREATE ROLE %I LOGIN PASSWORD %L', :'reader_user', :'reader_password') + WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = :'reader_user') \gexec + + GRANT CONNECT ON DATABASE {{ .Values.postgres.database }} TO :"reader_user"; + GRANT USAGE ON SCHEMA public TO :"reader_user"; + GRANT SELECT ON sensor_events, sensor_measurements, grafana_sensor_measurements TO :"reader_user"; + ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO :"reader_user"; + SQL diff --git a/templates/postgres-service.yaml b/templates/postgres-service.yaml new file mode 100644 index 0000000..090fda0 --- /dev/null +++ b/templates/postgres-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "thermo.fullname" . }}-postgres + labels: + {{- include "thermo.labels" . | nindent 4 }} +spec: + type: ClusterIP + selector: + {{- include "thermo.postgresSelectorLabels" . | nindent 4 }} + ports: + - name: postgres + port: 5432 + targetPort: postgres diff --git a/templates/postgres-statefulset.yaml b/templates/postgres-statefulset.yaml new file mode 100644 index 0000000..5dd2783 --- /dev/null +++ b/templates/postgres-statefulset.yaml @@ -0,0 +1,63 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "thermo.fullname" . }}-postgres + labels: + {{- include "thermo.labels" . | nindent 4 }} +spec: + serviceName: {{ include "thermo.fullname" . }}-postgres + replicas: 1 + selector: + matchLabels: + {{- include "thermo.postgresSelectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "thermo.postgresSelectorLabels" . | nindent 8 }} + spec: + securityContext: + fsGroup: 70 + fsGroupChangePolicy: OnRootMismatch + containers: + - name: postgres + image: "{{ .Values.postgres.image.repository }}:{{ .Values.postgres.image.tag }}" + imagePullPolicy: {{ .Values.postgres.image.pullPolicy }} + ports: + - name: postgres + containerPort: 5432 + envFrom: + - secretRef: + name: {{ include "thermo.fullname" . }}-postgres-secret + volumeMounts: + - name: data + mountPath: /var/lib/postgresql/data + - name: init + mountPath: /docker-entrypoint-initdb.d + readOnly: true + readinessProbe: + exec: + command: ["sh", "-ec", "pg_isready -d \"$POSTGRES_DB\" -U \"$POSTGRES_USER\""] + initialDelaySeconds: 5 + periodSeconds: 5 + livenessProbe: + exec: + command: ["sh", "-ec", "pg_isready -d \"$POSTGRES_DB\" -U \"$POSTGRES_USER\""] + initialDelaySeconds: 20 + periodSeconds: 10 + resources: + {{- toYaml .Values.postgres.resources | nindent 12 }} + volumes: + - name: init + configMap: + name: {{ include "thermo.fullname" . }}-postgres-init + defaultMode: 0555 + volumeClaimTemplates: + - metadata: + name: data + spec: + storageClassName: {{ .Values.postgres.storage.className }} + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.postgres.storage.size }} diff --git a/templates/writer-configmap.yaml b/templates/writer-configmap.yaml new file mode 100644 index 0000000..5ddeeb6 --- /dev/null +++ b/templates/writer-configmap.yaml @@ -0,0 +1,115 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "thermo.fullname" . }}-writer + labels: + {{- include "thermo.labels" . | nindent 4 }} +data: + redpanda.yaml: | + http: + enabled: true + address: 0.0.0.0:4195 + + input: + kafka: + addresses: + {{- toYaml .Values.writer.kafka.brokers | nindent 10 }} + topics: + - {{ .Values.writer.kafka.topic }} + consumer_group: {{ .Values.writer.kafka.consumerGroup }} + start_from_oldest: {{ .Values.writer.kafka.startFromOldest }} + + pipeline: + processors: + - mapping: | + root = if this.event_id.type() != "string" || + this.timestamp.type() != "string" || + this.source.type() != "string" || + this.event_type.type() != "string" || + this.severity.type() != "string" || + this.data.type() != "object" || + this.data.measurements.type() != "object" { + deleted() + } else { + this + } + + output: + sql_raw: + driver: postgres + dsn: ${THERMO_POSTGRES_DSN} + max_in_flight: 1 + query: | + WITH payload AS ( + SELECT $1::jsonb AS document + ), event_data AS ( + SELECT + (document->>'event_id')::uuid AS event_id, + COALESCE(document->>'timestamp', document#>>'{data,observed_at}')::timestamptz AS observed_at, + COALESCE(document->>'ingested_at', document->>'timestamp')::timestamptz AS ingested_at, + document->>'source' AS source, + document->>'event_type' AS event_type, + document->>'severity' AS severity, + document#>>'{data,collector_id}' AS collector_id, + COALESCE( + document#>>'{data,device_id}', + document#>>'{data,sensor_id}', + document#>>'{data,device_address}' + ) AS device_id, + document AS raw_event, + COALESCE(document#>'{data,measurements}', '{}'::jsonb) AS measurements + FROM payload + ), stored_event AS ( + INSERT INTO sensor_events ( + event_id, observed_at, ingested_at, source, event_type, + severity, collector_id, device_id, raw_event + ) + SELECT + event_id, observed_at, ingested_at, source, event_type, + severity, collector_id, device_id, raw_event + FROM event_data + ON CONFLICT (event_id) DO NOTHING + RETURNING event_id + ) + INSERT INTO sensor_measurements ( + event_id, observed_at, source, collector_id, device_id, + metric_name, numeric_value, text_value, unit, source_updated_at + ) + SELECT + event.event_id, + COALESCE( + NULLIF(measurement.value->>'updated_at', '')::timestamptz, + event.observed_at + ), + event.source, + event.collector_id, + event.device_id, + measurement.key, + CASE + WHEN jsonb_typeof(measurement.value) = 'number' + THEN measurement.value::text::double precision + WHEN jsonb_typeof(measurement.value) = 'boolean' + THEN CASE WHEN measurement.value::boolean THEN 1 ELSE 0 END + WHEN jsonb_typeof(measurement.value->'value') = 'number' + THEN (measurement.value->>'value')::double precision + WHEN jsonb_typeof(measurement.value->'value') = 'boolean' + THEN CASE WHEN (measurement.value->>'value')::boolean THEN 1 ELSE 0 END + ELSE NULL + END, + CASE + WHEN jsonb_typeof(measurement.value) = 'string' + THEN measurement.value#>>'{}' + WHEN jsonb_typeof(measurement.value->'value') = 'string' + THEN measurement.value->>'value' + ELSE NULL + END, + measurement.value->>'unit', + NULLIF(measurement.value->>'updated_at', '')::timestamptz + FROM event_data AS event + CROSS JOIN LATERAL jsonb_each(event.measurements) AS measurement + WHERE + jsonb_typeof(measurement.value) IN ('number', 'boolean', 'string') + OR jsonb_typeof(measurement.value->'value') IN ('number', 'boolean', 'string') + ON CONFLICT (event_id, metric_name) DO NOTHING; + args_mapping: | + root = [this.string()] diff --git a/templates/writer-deployment.yaml b/templates/writer-deployment.yaml new file mode 100644 index 0000000..9a8c76d --- /dev/null +++ b/templates/writer-deployment.yaml @@ -0,0 +1,54 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "thermo.fullname" . }}-writer + labels: + {{- include "thermo.labels" . | nindent 4 }} +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + {{- include "thermo.writerSelectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "thermo.writerSelectorLabels" . | nindent 8 }} + spec: + containers: + - name: redpanda-connect + image: "{{ .Values.writer.image.repository }}:{{ .Values.writer.image.tag }}" + imagePullPolicy: {{ .Values.writer.image.pullPolicy }} + args: + - -c + - /config/redpanda.yaml + - --disable-telemetry + envFrom: + - secretRef: + name: {{ include "thermo.fullname" . }}-postgres-redpanda + ports: + - name: http + containerPort: 4195 + readinessProbe: + httpGet: + path: /ready + port: http + initialDelaySeconds: 3 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /ping + port: http + initialDelaySeconds: 10 + periodSeconds: 10 + resources: + {{- toYaml .Values.writer.resources | nindent 12 }} + volumeMounts: + - name: config + mountPath: /config + readOnly: true + volumes: + - name: config + configMap: + name: {{ include "thermo.fullname" . }}-writer diff --git a/values.yaml b/values.yaml new file mode 100644 index 0000000..74301f8 --- /dev/null +++ b/values.yaml @@ -0,0 +1,48 @@ +nameOverride: "" +fullnameOverride: "thermo" + +postgres: + image: + repository: postgres + tag: 16-alpine + pullPolicy: IfNotPresent + database: thermo + writerUser: thermo_writer + readerUser: thermo_reader + storage: + className: microk8s-hostpath + size: 5Gi + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + +writer: + image: + repository: docker.redpanda.com/redpandadata/connect + tag: 4.98.0 + pullPolicy: IfNotPresent + kafka: + brokers: + - kafka.platform-core.svc.cluster.local:9092 + topic: application.thermo.events + consumerGroup: thermo-postgres-writer + startFromOldest: true + resources: + requests: + cpu: 25m + memory: 64Mi + limits: + cpu: 250m + memory: 256Mi + +externalSecrets: + refreshInterval: 24h + secretStore: + name: aws-ssm + kind: ClusterSecretStore + writerPasswordKey: /vulcan/platform-core/thermo-postgres/writer-password + readerPasswordKey: /vulcan/platform-core/thermo-postgres/reader-password