Add thermo history Helm chart

Chart.yaml

@@ -0,0 +1,6 @@
+apiVersion: v2
+name: thermo
+description: Historical storage for home sensor events
+type: application
+version: 0.1.0
+appVersion: "1.0.0"

README.md

@@ -0,0 +1,25 @@
+# Thermo
+
+Helm chart for storing home sensor events from Kafka in PostgreSQL.
+
+The chart deploys:
+
+- PostgreSQL with persistent storage;
+- a generic event and measurement schema suitable for WindFree and BLE sensors;
+- a read-only database role for Grafana;
+- a lightweight Redpanda Connect consumer for `application.thermo.events`;
+- External Secrets backed by the existing AWS SSM secret store.
+
+## Required SSM parameters
+
+- `/vulcan/platform-core/thermo-postgres/writer-password`
+- `/vulcan/platform-core/thermo-postgres/reader-password`
+
+## Validation
+
+```shell
+helm lint .
+helm template thermo . --namespace platform-core
+```
+
+The Grafana datasource and dashboards are intentionally managed outside this chart because Grafana runs in the `monitoring` namespace.

templates/_helpers.tpl

@@ -0,0 +1,24 @@
+{{- define "thermo.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{- define "thermo.fullname" -}}
+{{- default (include "thermo.name" .) .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{- define "thermo.labels" -}}
+app.kubernetes.io/name: {{ include "thermo.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" }}
+{{- end }}
+
+{{- define "thermo.postgresSelectorLabels" -}}
+app: {{ include "thermo.fullname" . }}-postgres
+{{- end }}
+
+{{- define "thermo.writerSelectorLabels" -}}
+app.kubernetes.io/name: {{ include "thermo.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+app.kubernetes.io/component: writer
+{{- end }}

templates/external-secrets.yaml

@@ -0,0 +1,52 @@
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: {{ include "thermo.fullname" . }}-postgres-secret
+  labels:
+    {{- include "thermo.labels" . | nindent 4 }}
+spec:
+  refreshInterval: {{ .Values.externalSecrets.refreshInterval }}
+  secretStoreRef:
+    name: {{ .Values.externalSecrets.secretStore.name }}
+    kind: {{ .Values.externalSecrets.secretStore.kind }}
+  target:
+    name: {{ include "thermo.fullname" . }}-postgres-secret
+    creationPolicy: Owner
+    template:
+      type: Opaque
+      data:
+        POSTGRES_DB: {{ .Values.postgres.database | quote }}
+        POSTGRES_USER: {{ .Values.postgres.writerUser | quote }}
+        POSTGRES_PASSWORD: {{ "{{ .writer_password }}" | quote }}
+        GRAFANA_USER: {{ .Values.postgres.readerUser | quote }}
+        GRAFANA_PASSWORD: {{ "{{ .reader_password }}" | quote }}
+  data:
+    - secretKey: writer_password
+      remoteRef:
+        key: {{ .Values.externalSecrets.writerPasswordKey }}
+    - secretKey: reader_password
+      remoteRef:
+        key: {{ .Values.externalSecrets.readerPasswordKey }}
+---
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: {{ include "thermo.fullname" . }}-postgres-redpanda
+  labels:
+    {{- include "thermo.labels" . | nindent 4 }}
+spec:
+  refreshInterval: {{ .Values.externalSecrets.refreshInterval }}
+  secretStoreRef:
+    name: {{ .Values.externalSecrets.secretStore.name }}
+    kind: {{ .Values.externalSecrets.secretStore.kind }}
+  target:
+    name: {{ include "thermo.fullname" . }}-postgres-redpanda
+    creationPolicy: Owner
+    template:
+      type: Opaque
+      data:
+        THERMO_POSTGRES_DSN: {{ printf "postgres://%s:%s@%s-postgres.%s.svc.cluster.local:5432/%s?sslmode=disable" .Values.postgres.writerUser "{{ .writer_password }}" (include "thermo.fullname" .) .Release.Namespace .Values.postgres.database | quote }}
+  data:
+    - secretKey: writer_password
+      remoteRef:
+        key: {{ .Values.externalSecrets.writerPasswordKey }}

templates/postgres-init-configmap.yaml

@@ -0,0 +1,72 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "thermo.fullname" . }}-postgres-init
+  labels:
+    {{- include "thermo.labels" . | nindent 4 }}
+data:
+  01-schema.sql: |
+    CREATE TABLE sensor_events (
+      event_id uuid PRIMARY KEY,
+      observed_at timestamptz NOT NULL,
+      ingested_at timestamptz NOT NULL,
+      source text NOT NULL,
+      event_type text NOT NULL,
+      severity text NOT NULL,
+      collector_id text,
+      device_id text NOT NULL,
+      raw_event jsonb NOT NULL,
+      stored_at timestamptz NOT NULL DEFAULT now()
+    );
+
+    CREATE TABLE sensor_measurements (
+      event_id uuid NOT NULL REFERENCES sensor_events(event_id) ON DELETE CASCADE,
+      observed_at timestamptz NOT NULL,
+      source text NOT NULL,
+      collector_id text,
+      device_id text NOT NULL,
+      metric_name text NOT NULL,
+      numeric_value double precision,
+      text_value text,
+      unit text,
+      source_updated_at timestamptz,
+      PRIMARY KEY (event_id, metric_name),
+      CHECK (numeric_value IS NOT NULL OR text_value IS NOT NULL)
+    );
+
+    CREATE INDEX sensor_events_device_time_idx
+      ON sensor_events (device_id, observed_at DESC);
+    CREATE INDEX sensor_measurements_metric_device_time_idx
+      ON sensor_measurements (metric_name, device_id, observed_at DESC);
+    CREATE INDEX sensor_measurements_time_idx
+      ON sensor_measurements (observed_at DESC);
+
+    CREATE VIEW grafana_sensor_measurements AS
+      SELECT
+        observed_at AS time,
+        source,
+        collector_id,
+        device_id,
+        metric_name,
+        numeric_value,
+        text_value,
+        unit
+      FROM sensor_measurements;
+
+  02-reader.sh: |
+    #!/bin/sh
+    set -eu
+
+    psql --set ON_ERROR_STOP=1 \
+      --username "$POSTGRES_USER" \
+      --dbname "$POSTGRES_DB" \
+      --set reader_user="$GRAFANA_USER" \
+      --set reader_password="$GRAFANA_PASSWORD" <<'SQL'
+    SELECT format('CREATE ROLE %I LOGIN PASSWORD %L', :'reader_user', :'reader_password')
+    WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = :'reader_user') \gexec
+
+    GRANT CONNECT ON DATABASE {{ .Values.postgres.database }} TO :"reader_user";
+    GRANT USAGE ON SCHEMA public TO :"reader_user";
+    GRANT SELECT ON sensor_events, sensor_measurements, grafana_sensor_measurements TO :"reader_user";
+    ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO :"reader_user";
+    SQL

templates/postgres-service.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "thermo.fullname" . }}-postgres
+  labels:
+    {{- include "thermo.labels" . | nindent 4 }}
+spec:
+  type: ClusterIP
+  selector:
+    {{- include "thermo.postgresSelectorLabels" . | nindent 4 }}
+  ports:
+    - name: postgres
+      port: 5432
+      targetPort: postgres

templates/postgres-statefulset.yaml

@@ -0,0 +1,63 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ include "thermo.fullname" . }}-postgres
+  labels:
+    {{- include "thermo.labels" . | nindent 4 }}
+spec:
+  serviceName: {{ include "thermo.fullname" . }}-postgres
+  replicas: 1
+  selector:
+    matchLabels:
+      {{- include "thermo.postgresSelectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      labels:
+        {{- include "thermo.postgresSelectorLabels" . | nindent 8 }}
+    spec:
+      securityContext:
+        fsGroup: 70
+        fsGroupChangePolicy: OnRootMismatch
+      containers:
+        - name: postgres
+          image: "{{ .Values.postgres.image.repository }}:{{ .Values.postgres.image.tag }}"
+          imagePullPolicy: {{ .Values.postgres.image.pullPolicy }}
+          ports:
+            - name: postgres
+              containerPort: 5432
+          envFrom:
+            - secretRef:
+                name: {{ include "thermo.fullname" . }}-postgres-secret
+          volumeMounts:
+            - name: data
+              mountPath: /var/lib/postgresql/data
+            - name: init
+              mountPath: /docker-entrypoint-initdb.d
+              readOnly: true
+          readinessProbe:
+            exec:
+              command: ["sh", "-ec", "pg_isready -d \"$POSTGRES_DB\" -U \"$POSTGRES_USER\""]
+            initialDelaySeconds: 5
+            periodSeconds: 5
+          livenessProbe:
+            exec:
+              command: ["sh", "-ec", "pg_isready -d \"$POSTGRES_DB\" -U \"$POSTGRES_USER\""]
+            initialDelaySeconds: 20
+            periodSeconds: 10
+          resources:
+            {{- toYaml .Values.postgres.resources | nindent 12 }}
+      volumes:
+        - name: init
+          configMap:
+            name: {{ include "thermo.fullname" . }}-postgres-init
+            defaultMode: 0555
+  volumeClaimTemplates:
+    - metadata:
+        name: data
+      spec:
+        storageClassName: {{ .Values.postgres.storage.className }}
+        accessModes:
+          - ReadWriteOnce
+        resources:
+          requests:
+            storage: {{ .Values.postgres.storage.size }}

templates/writer-configmap.yaml

@@ -0,0 +1,115 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "thermo.fullname" . }}-writer
+  labels:
+    {{- include "thermo.labels" . | nindent 4 }}
+data:
+  redpanda.yaml: |
+    http:
+      enabled: true
+      address: 0.0.0.0:4195
+
+    input:
+      kafka:
+        addresses:
+          {{- toYaml .Values.writer.kafka.brokers | nindent 10 }}
+        topics:
+          - {{ .Values.writer.kafka.topic }}
+        consumer_group: {{ .Values.writer.kafka.consumerGroup }}
+        start_from_oldest: {{ .Values.writer.kafka.startFromOldest }}
+
+    pipeline:
+      processors:
+        - mapping: |
+            root = if this.event_id.type() != "string" ||
+              this.timestamp.type() != "string" ||
+              this.source.type() != "string" ||
+              this.event_type.type() != "string" ||
+              this.severity.type() != "string" ||
+              this.data.type() != "object" ||
+              this.data.measurements.type() != "object" {
+              deleted()
+            } else {
+              this
+            }
+
+    output:
+      sql_raw:
+        driver: postgres
+        dsn: ${THERMO_POSTGRES_DSN}
+        max_in_flight: 1
+        query: |
+          WITH payload AS (
+            SELECT $1::jsonb AS document
+          ), event_data AS (
+            SELECT
+              (document->>'event_id')::uuid AS event_id,
+              COALESCE(document->>'timestamp', document#>>'{data,observed_at}')::timestamptz AS observed_at,
+              COALESCE(document->>'ingested_at', document->>'timestamp')::timestamptz AS ingested_at,
+              document->>'source' AS source,
+              document->>'event_type' AS event_type,
+              document->>'severity' AS severity,
+              document#>>'{data,collector_id}' AS collector_id,
+              COALESCE(
+                document#>>'{data,device_id}',
+                document#>>'{data,sensor_id}',
+                document#>>'{data,device_address}'
+              ) AS device_id,
+              document AS raw_event,
+              COALESCE(document#>'{data,measurements}', '{}'::jsonb) AS measurements
+            FROM payload
+          ), stored_event AS (
+            INSERT INTO sensor_events (
+              event_id, observed_at, ingested_at, source, event_type,
+              severity, collector_id, device_id, raw_event
+            )
+            SELECT
+              event_id, observed_at, ingested_at, source, event_type,
+              severity, collector_id, device_id, raw_event
+            FROM event_data
+            ON CONFLICT (event_id) DO NOTHING
+            RETURNING event_id
+          )
+          INSERT INTO sensor_measurements (
+            event_id, observed_at, source, collector_id, device_id,
+            metric_name, numeric_value, text_value, unit, source_updated_at
+          )
+          SELECT
+            event.event_id,
+            COALESCE(
+              NULLIF(measurement.value->>'updated_at', '')::timestamptz,
+              event.observed_at
+            ),
+            event.source,
+            event.collector_id,
+            event.device_id,
+            measurement.key,
+            CASE
+              WHEN jsonb_typeof(measurement.value) = 'number'
+                THEN measurement.value::text::double precision
+              WHEN jsonb_typeof(measurement.value) = 'boolean'
+                THEN CASE WHEN measurement.value::boolean THEN 1 ELSE 0 END
+              WHEN jsonb_typeof(measurement.value->'value') = 'number'
+                THEN (measurement.value->>'value')::double precision
+              WHEN jsonb_typeof(measurement.value->'value') = 'boolean'
+                THEN CASE WHEN (measurement.value->>'value')::boolean THEN 1 ELSE 0 END
+              ELSE NULL
+            END,
+            CASE
+              WHEN jsonb_typeof(measurement.value) = 'string'
+                THEN measurement.value#>>'{}'
+              WHEN jsonb_typeof(measurement.value->'value') = 'string'
+                THEN measurement.value->>'value'
+              ELSE NULL
+            END,
+            measurement.value->>'unit',
+            NULLIF(measurement.value->>'updated_at', '')::timestamptz
+          FROM event_data AS event
+          CROSS JOIN LATERAL jsonb_each(event.measurements) AS measurement
+          WHERE
+            jsonb_typeof(measurement.value) IN ('number', 'boolean', 'string')
+            OR jsonb_typeof(measurement.value->'value') IN ('number', 'boolean', 'string')
+          ON CONFLICT (event_id, metric_name) DO NOTHING;
+        args_mapping: |
+          root = [this.string()]

templates/writer-deployment.yaml

@@ -0,0 +1,54 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "thermo.fullname" . }}-writer
+  labels:
+    {{- include "thermo.labels" . | nindent 4 }}
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      {{- include "thermo.writerSelectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      labels:
+        {{- include "thermo.writerSelectorLabels" . | nindent 8 }}
+    spec:
+      containers:
+        - name: redpanda-connect
+          image: "{{ .Values.writer.image.repository }}:{{ .Values.writer.image.tag }}"
+          imagePullPolicy: {{ .Values.writer.image.pullPolicy }}
+          args:
+            - -c
+            - /config/redpanda.yaml
+            - --disable-telemetry
+          envFrom:
+            - secretRef:
+                name: {{ include "thermo.fullname" . }}-postgres-redpanda
+          ports:
+            - name: http
+              containerPort: 4195
+          readinessProbe:
+            httpGet:
+              path: /ready
+              port: http
+            initialDelaySeconds: 3
+            periodSeconds: 5
+          livenessProbe:
+            httpGet:
+              path: /ping
+              port: http
+            initialDelaySeconds: 10
+            periodSeconds: 10
+          resources:
+            {{- toYaml .Values.writer.resources | nindent 12 }}
+          volumeMounts:
+            - name: config
+              mountPath: /config
+              readOnly: true
+      volumes:
+        - name: config
+          configMap:
+            name: {{ include "thermo.fullname" . }}-writer

values.yaml

@@ -0,0 +1,48 @@
+nameOverride: ""
+fullnameOverride: "thermo"
+
+postgres:
+  image:
+    repository: postgres
+    tag: 16-alpine
+    pullPolicy: IfNotPresent
+  database: thermo
+  writerUser: thermo_writer
+  readerUser: thermo_reader
+  storage:
+    className: microk8s-hostpath
+    size: 5Gi
+  resources:
+    requests:
+      cpu: 50m
+      memory: 128Mi
+    limits:
+      cpu: 500m
+      memory: 512Mi
+
+writer:
+  image:
+    repository: docker.redpanda.com/redpandadata/connect
+    tag: 4.98.0
+    pullPolicy: IfNotPresent
+  kafka:
+    brokers:
+      - kafka.platform-core.svc.cluster.local:9092
+    topic: application.thermo.events
+    consumerGroup: thermo-postgres-writer
+    startFromOldest: true
+  resources:
+    requests:
+      cpu: 25m
+      memory: 64Mi
+    limits:
+      cpu: 250m
+      memory: 256Mi
+
+externalSecrets:
+  refreshInterval: 24h
+  secretStore:
+    name: aws-ssm
+    kind: ClusterSecretStore
+  writerPasswordKey: /vulcan/platform-core/thermo-postgres/writer-password
+  readerPasswordKey: /vulcan/platform-core/thermo-postgres/reader-password