From 0d3da6bbf36f56fb64ed38a38f40c9b210950ca7 Mon Sep 17 00:00:00 2001
From: alhendrickson <159636032+alhendrickson@users.noreply.github.com.>
Date: Tue, 10 Mar 2026 19:25:23 +0000
Subject: [PATCH 1/4] feat(helm): Add tracing to medcat-trainer

---
 .../cogstack-helm-ce/templates/NOTES.txt      |  2 +-
 .../charts/cogstack-helm-ce/values.yaml       |  3 +-
 .../templates/_helpers.tpl                    | 22 ++++++++++
 .../templates/medcat-trainer-deployment.yaml  | 11 +++++
 .../medcat-trainer-env-configmap.yaml         | 17 ++++++++
 .../charts/medcat-trainer-helm/values.yaml    | 43 +++++++++++++++++++
 6 files changed, 95 insertions(+), 3 deletions(-)

diff --git a/deployment/kubernetes/charts/cogstack-helm-ce/templates/NOTES.txt b/deployment/kubernetes/charts/cogstack-helm-ce/templates/NOTES.txt
index 2d073a1..ab3903d 100644
--- a/deployment/kubernetes/charts/cogstack-helm-ce/templates/NOTES.txt
+++ b/deployment/kubernetes/charts/cogstack-helm-ce/templates/NOTES.txt
@@ -29,8 +29,8 @@ export OPENSEARCH_DASHBOARD_PORT={{ index .Values "opensearch-dashboards" "servi
 kubectl --namespace {{ .Release.Namespace }} port-forward svc/$OPENSEARCH_DASHBOARD_SERVICE 5601:$OPENSEARCH_DASHBOARD_PORT &
 {{- end }}
 
-# 5. Jupyterhub
 {{- if index .Values "cogstack-jupyterhub" "enabled" }}
+# 5. Jupyterhub
 export JUPYTERHUB_PROXY_PUBLIC_SERVICE=proxy-public
 export JUPYTERHUB_PROXY_SERVICE_PORT=$(kubectl get svc $JUPYTERHUB_PROXY_PUBLIC_SERVICE -o jsonpath="{.spec.ports[0].port}" )
 kubectl --namespace {{ .Release.Namespace }} port-forward svc/$JUPYTERHUB_PROXY_PUBLIC_SERVICE 8000:$JUPYTERHUB_PROXY_SERVICE_PORT &
diff --git a/deployment/kubernetes/charts/cogstack-helm-ce/values.yaml b/deployment/kubernetes/charts/cogstack-helm-ce/values.yaml
index 3d1b92b..8febfe0 100644
--- a/deployment/kubernetes/charts/cogstack-helm-ce/values.yaml
+++ b/deployment/kubernetes/charts/cogstack-helm-ce/values.yaml
@@ -24,10 +24,9 @@ medcat-service:
     repository: cogstacksystems/medcat-service
     tag: "1.2.0"
 
-
 medcat-trainer:
   image:
-    tag: "latest@sha256:dcbd9f7c480dcb88e072189d233a146317f210a826573fd0c47d469da5360be0"
+    tag: latest@sha256:103215a7540ad614c32866f4cb00ddd91e7aff37cea9abc25dc226c577f9506d
   provisioning:
     enabled: true
     existingConfigMap:
diff --git a/deployment/kubernetes/charts/medcat-trainer-helm/templates/_helpers.tpl b/deployment/kubernetes/charts/medcat-trainer-helm/templates/_helpers.tpl
index 368e578..430016c 100644
--- a/deployment/kubernetes/charts/medcat-trainer-helm/templates/_helpers.tpl
+++ b/deployment/kubernetes/charts/medcat-trainer-helm/templates/_helpers.tpl
@@ -97,6 +97,28 @@ Return full Solr URL: combines host and port
 http://{{ include "medcat-trainer-helm.solrHost" . }}:{{ include "medcat-trainer-helm.solrPort" . }}
 {{- end }}
 
+{{/*
+Validate tracing.otlp: when otlp.enabled is true, at least one of grpc.enabled or http.enabled must be true.
+*/}}
+{{- define "medcat-trainer-helm.validateTracing" -}}
+{{- if and .Values.tracing .Values.tracing.otlp (eq .Values.tracing.otlp.enabled true) -}}
+{{- if not (or (index .Values.tracing.otlp.grpc "enabled") (index .Values.tracing.otlp.http "enabled")) -}}
+{{- fail "tracing.otlp.enabled is true but neither tracing.otlp.grpc.enabled nor tracing.otlp.http.enabled is true. Enable at least one of tracing.otlp.grpc.enabled or tracing.otlp.http.enabled." -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Convert tracing.resourceAttributes (object) to OTEL CSV format: key1=value1,key2=value2,...
+*/}}
+{{- define "medcat-trainer-helm.tracingResourceAttributesCsv" -}}
+{{- $parts := list -}}
+{{- range $name, $value := .Values.tracing.resourceAttributes -}}
+{{- $parts = append $parts (printf "%s=%s" $name $value) -}}
+{{- end -}}
+{{- join "," $parts | quote -}}
+{{- end -}}
+
 {{- define "postgres.service-name" -}}
 {{ include "postgresql.v1.primary.fullname" (dict "Values" .Values.postgresql "Chart" (dict "Name" "postgresql") "Release" .Release) }}
 {{- end -}}
diff --git a/deployment/kubernetes/charts/medcat-trainer-helm/templates/medcat-trainer-deployment.yaml b/deployment/kubernetes/charts/medcat-trainer-helm/templates/medcat-trainer-deployment.yaml
index 19042f8..296ac5e 100644
--- a/deployment/kubernetes/charts/medcat-trainer-helm/templates/medcat-trainer-deployment.yaml
+++ b/deployment/kubernetes/charts/medcat-trainer-helm/templates/medcat-trainer-deployment.yaml
@@ -1,3 +1,4 @@
+{{- include "medcat-trainer-helm.validateTracing" . -}}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -58,6 +59,16 @@ spec:
                 name: {{ include "medcat-trainer-helm.fullname" . }}-env
             - secretRef:
                 name: {{ include "medcat-trainer-helm.fullname" . }}-secret
+          env:
+          {{- range $key, $value := .Values.envValueFrom }}
+          - name: {{ $key | quote }}
+            valueFrom:
+              {{- tpl (toYaml $value) $ | nindent 14 }}
+          {{- end }}
+          # {{- if .Values.tracing.otlp.enabled }}
+          # - name: OTEL_RESOURCE_ATTRIBUTES  
+          #   value: {{ include "medcat-trainer-helm.tracingResourceAttributesCsv" . }}
+          # {{- end }}
           {{- with .Values.livenessProbe }}
           livenessProbe:
             {{- toYaml . | nindent 12 }}
diff --git a/deployment/kubernetes/charts/medcat-trainer-helm/templates/medcat-trainer-env-configmap.yaml b/deployment/kubernetes/charts/medcat-trainer-helm/templates/medcat-trainer-env-configmap.yaml
index 733b2b1..5cf4c2d 100644
--- a/deployment/kubernetes/charts/medcat-trainer-helm/templates/medcat-trainer-env-configmap.yaml
+++ b/deployment/kubernetes/charts/medcat-trainer-helm/templates/medcat-trainer-env-configmap.yaml
@@ -14,6 +14,23 @@ data:
   DB_PATH: "/home/api/db/db.sqlite3"
   MCT_VERSION: {{ .Values.image.tag | default .Chart.AppVersion }}
   API_URL: http://localhost:{{ .Values.service.apiPort }}/api/
+  REMOTE_MODEL_SERVICE_TYPE: "medcat" # TODO: Remove this for REMOTE_MODEL_SERVICE_TYPE after resolving pr-363 in cogstack-nlp
+
+  {{- if .Values.tracing.otlp.enabled }}
+    # Tracing configuration
+  MCT_ENABLE_TRACING: {{ .Values.tracing.otlp.enabled | quote }}
+  OTEL_EXPORTER_OTLP_ENDPOINT: {{ .Values.tracing.otlp.grpc.endpoint | quote }}
+  OTEL_TRACES_EXPORTER: otlp
+  OTEL_SERVICE_NAME: {{ .Values.tracing.serviceName | quote }}
+  OTEL_EXPORTER_OTLP_PROTOCOL: {{ if .Values.tracing.otlp.grpc.enabled }}"grpc"{{ else if .Values.tracing.otlp.http.enabled }}"http/protobuf"{{ else }}""{{ end }}
+  OTEL_METRICS_EXPORTER: none
+  OTEL_LOGS_EXPORTER: none
+  OTEL_PYTHON_DJANGO_EXCLUDED_URLS: /api/health/*,/metrics
+  OTEL_EXPERIMENTAL_RESOURCE_DETECTORS: {{ .Values.tracing.experimentalResourceDetectors | quote }}
+  OTEL_PYTHON_DISABLED_INSTRUMENTATIONS: {{ .Values.tracing.disabledInstrumentations | quote }}
+  OTEL_RESOURCE_ATTRIBUTES: {{ include "medcat-trainer-helm.tracingResourceAttributesCsv" . }}
+  {{- end }}
+
   {{- if and .Values.provisioning .Values.provisioning.enabled }}
   LOAD_EXAMPLES: "1"
   PROVISIONING_CONFIG_PATH: "/home/configs/provisioning.yaml"
diff --git a/deployment/kubernetes/charts/medcat-trainer-helm/values.yaml b/deployment/kubernetes/charts/medcat-trainer-helm/values.yaml
index 77161de..f7a9851 100644
--- a/deployment/kubernetes/charts/medcat-trainer-helm/values.yaml
+++ b/deployment/kubernetes/charts/medcat-trainer-helm/values.yaml
@@ -39,6 +39,21 @@ env:
   DB_ENGINE: "postgresql"
   DB_PORT: "5432"
 
+# Allow setting env values from field/configmap/secret references. Defaults to include k8s details for observability.
+envValueFrom:
+  K8S_NODE_NAME:
+    fieldRef:
+      fieldPath: spec.nodeName
+  K8S_POD_NAME:
+    fieldRef:
+      fieldPath: metadata.name
+  K8S_POD_UID:
+    fieldRef:
+      fieldPath: metadata.uid
+  K8S_POD_NAMESPACE:
+    fieldRef:
+      fieldPath: metadata.namespace
+
 provisioning:
   # -- Set to true to enable provisioning of projects and models on startup..
   enabled: false
@@ -61,6 +76,34 @@ provisioning:
           description: "Example project for medcat trainer"
           annotationGuidelineLink: "https://docs.google.com/document/d/1xxelBOYbyVzJ7vLlztP2q1Kw9F5Vr1pRwblgrXPS7QM/edit?usp=sharing"
 
+# Enable tracing with opentelemetry for the medcat-trainer backend
+tracing:
+  # The name of the service in the tracing system
+  serviceName: "medcat-trainer"
+  # Resource attributes to add to the traces
+  resourceAttributes:
+    k8s.pod.uid: "$(K8S_POD_UID)"
+    k8s.pod.name: "$(K8S_POD_NAME)"
+    k8s.namespace.name: "$(K8S_POD_NAMESPACE)"
+    k8s.node.name: "$(K8S_NODE_NAME)" 
+  experimentalResourceDetectors: "containerid,os"
+  # Optionally disable the db instrumentations due to noise.
+  disabledInstrumentations: "psycopg,sqlite3"
+  otlp:
+    # Set to true to enable tracing
+    enabled: false
+    grpc:
+      # Set to true to enable grpc tracing
+      enabled: false
+      # The endpoint to send the traces to
+      endpoint: "http://<unused>:4317"
+    http:
+      # Set to true to enable http tracing over http/protobug
+      enabled: false
+      # The endpoint to send the traces to
+      endpoint: "http://<unused>:4318"
+
+
 postgresql:
   enabled: true
   # TODO: Support custom DB overrides

From 03d8d3feafb60dd2d0647a53ab9d723a3c0bbe02 Mon Sep 17 00:00:00 2001
From: alhendrickson <159636032+alhendrickson@users.noreply.github.com.>
Date: Tue, 10 Mar 2026 19:29:06 +0000
Subject: [PATCH 2/4] feat(helm): Add tracing to medcat-trainer - format

---
 .../templates/medcat-trainer-env-configmap.yaml          | 9 +++++++--
 .../kubernetes/charts/medcat-trainer-helm/values.yaml    | 3 +--
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/deployment/kubernetes/charts/medcat-trainer-helm/templates/medcat-trainer-env-configmap.yaml b/deployment/kubernetes/charts/medcat-trainer-helm/templates/medcat-trainer-env-configmap.yaml
index 5cf4c2d..341d53d 100644
--- a/deployment/kubernetes/charts/medcat-trainer-helm/templates/medcat-trainer-env-configmap.yaml
+++ b/deployment/kubernetes/charts/medcat-trainer-helm/templates/medcat-trainer-env-configmap.yaml
@@ -19,10 +19,15 @@ data:
   {{- if .Values.tracing.otlp.enabled }}
     # Tracing configuration
   MCT_ENABLE_TRACING: {{ .Values.tracing.otlp.enabled | quote }}
-  OTEL_EXPORTER_OTLP_ENDPOINT: {{ .Values.tracing.otlp.grpc.endpoint | quote }}
   OTEL_TRACES_EXPORTER: otlp
   OTEL_SERVICE_NAME: {{ .Values.tracing.serviceName | quote }}
-  OTEL_EXPORTER_OTLP_PROTOCOL: {{ if .Values.tracing.otlp.grpc.enabled }}"grpc"{{ else if .Values.tracing.otlp.http.enabled }}"http/protobuf"{{ else }}""{{ end }}
+  {{- if .Values.tracing.otlp.grpc.enabled }}
+  OTEL_EXPORTER_OTLP_ENDPOINT: {{ .Values.tracing.otlp.grpc.endpoint | quote }}
+  OTEL_EXPORTER_OTLP_PROTOCOL: "grpc"
+  {{- else if .Values.tracing.otlp.http.enabled }}
+  OTEL_EXPORTER_OTLP_ENDPOINT: {{ .Values.tracing.otlp.http.endpoint | quote }}
+  OTEL_EXPORTER_OTLP_PROTOCOL: "http/protobuf"
+  {{- end }}
   OTEL_METRICS_EXPORTER: none
   OTEL_LOGS_EXPORTER: none
   OTEL_PYTHON_DJANGO_EXCLUDED_URLS: /api/health/*,/metrics
diff --git a/deployment/kubernetes/charts/medcat-trainer-helm/values.yaml b/deployment/kubernetes/charts/medcat-trainer-helm/values.yaml
index f7a9851..8b3c6a8 100644
--- a/deployment/kubernetes/charts/medcat-trainer-helm/values.yaml
+++ b/deployment/kubernetes/charts/medcat-trainer-helm/values.yaml
@@ -85,7 +85,7 @@ tracing:
     k8s.pod.uid: "$(K8S_POD_UID)"
     k8s.pod.name: "$(K8S_POD_NAME)"
     k8s.namespace.name: "$(K8S_POD_NAMESPACE)"
-    k8s.node.name: "$(K8S_NODE_NAME)" 
+    k8s.node.name: "$(K8S_NODE_NAME)"
   experimentalResourceDetectors: "containerid,os"
   # Optionally disable the db instrumentations due to noise.
   disabledInstrumentations: "psycopg,sqlite3"
@@ -103,7 +103,6 @@ tracing:
       # The endpoint to send the traces to
       endpoint: "http://<unused>:4318"
 
-
 postgresql:
   enabled: true
   # TODO: Support custom DB overrides

From 22aaaa3b2ee61af3488e1ccbac3378fe24e6e391 Mon Sep 17 00:00:00 2001
From: alhendrickson <159636032+alhendrickson@users.noreply.github.com.>
Date: Wed, 11 Mar 2026 08:45:39 +0000
Subject: [PATCH 3/4] feat(helm): Add tracing to medcat-trainer - testing

---
 .../templates/medcat-trainer-deployment.yaml              | 8 ++++----
 .../templates/medcat-trainer-env-configmap.yaml           | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/deployment/kubernetes/charts/medcat-trainer-helm/templates/medcat-trainer-deployment.yaml b/deployment/kubernetes/charts/medcat-trainer-helm/templates/medcat-trainer-deployment.yaml
index 296ac5e..2739fa3 100644
--- a/deployment/kubernetes/charts/medcat-trainer-helm/templates/medcat-trainer-deployment.yaml
+++ b/deployment/kubernetes/charts/medcat-trainer-helm/templates/medcat-trainer-deployment.yaml
@@ -65,10 +65,10 @@ spec:
             valueFrom:
               {{- tpl (toYaml $value) $ | nindent 14 }}
           {{- end }}
-          # {{- if .Values.tracing.otlp.enabled }}
-          # - name: OTEL_RESOURCE_ATTRIBUTES  
-          #   value: {{ include "medcat-trainer-helm.tracingResourceAttributesCsv" . }}
-          # {{- end }}
+          {{- if .Values.tracing.otlp.enabled }}
+          - name: OTEL_RESOURCE_ATTRIBUTES  
+            value: {{ include "medcat-trainer-helm.tracingResourceAttributesCsv" . }}
+          {{- end }}
           {{- with .Values.livenessProbe }}
           livenessProbe:
             {{- toYaml . | nindent 12 }}
diff --git a/deployment/kubernetes/charts/medcat-trainer-helm/templates/medcat-trainer-env-configmap.yaml b/deployment/kubernetes/charts/medcat-trainer-helm/templates/medcat-trainer-env-configmap.yaml
index 341d53d..d7e32a6 100644
--- a/deployment/kubernetes/charts/medcat-trainer-helm/templates/medcat-trainer-env-configmap.yaml
+++ b/deployment/kubernetes/charts/medcat-trainer-helm/templates/medcat-trainer-env-configmap.yaml
@@ -19,7 +19,7 @@ data:
   {{- if .Values.tracing.otlp.enabled }}
     # Tracing configuration
   MCT_ENABLE_TRACING: {{ .Values.tracing.otlp.enabled | quote }}
-  OTEL_TRACES_EXPORTER: otlp
+  OTEL_TRACES_EXPORTER: console
   OTEL_SERVICE_NAME: {{ .Values.tracing.serviceName | quote }}
   {{- if .Values.tracing.otlp.grpc.enabled }}
   OTEL_EXPORTER_OTLP_ENDPOINT: {{ .Values.tracing.otlp.grpc.endpoint | quote }}
@@ -33,7 +33,7 @@ data:
   OTEL_PYTHON_DJANGO_EXCLUDED_URLS: /api/health/*,/metrics
   OTEL_EXPERIMENTAL_RESOURCE_DETECTORS: {{ .Values.tracing.experimentalResourceDetectors | quote }}
   OTEL_PYTHON_DISABLED_INSTRUMENTATIONS: {{ .Values.tracing.disabledInstrumentations | quote }}
-  OTEL_RESOURCE_ATTRIBUTES: {{ include "medcat-trainer-helm.tracingResourceAttributesCsv" . }}
+
   {{- end }}
 
   {{- if and .Values.provisioning .Values.provisioning.enabled }}

From 82a78d0a2e4b21c368dac99db8d4da0902c36233 Mon Sep 17 00:00:00 2001
From: alhendrickson <159636032+alhendrickson@users.noreply.github.com.>
Date: Wed, 11 Mar 2026 12:13:13 +0000
Subject: [PATCH 4/4] feat(medcat-trainer): Add semantic convention resource
 attributes

---
 .../charts/medcat-trainer-helm/templates/_helpers.tpl         | 4 +++-
 .../templates/medcat-trainer-deployment.yaml                  | 2 ++
 .../templates/medcat-trainer-env-configmap.yaml               | 2 +-
 deployment/kubernetes/charts/medcat-trainer-helm/values.yaml  | 4 +++-
 4 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/deployment/kubernetes/charts/medcat-trainer-helm/templates/_helpers.tpl b/deployment/kubernetes/charts/medcat-trainer-helm/templates/_helpers.tpl
index 430016c..cce2199 100644
--- a/deployment/kubernetes/charts/medcat-trainer-helm/templates/_helpers.tpl
+++ b/deployment/kubernetes/charts/medcat-trainer-helm/templates/_helpers.tpl
@@ -110,11 +110,13 @@ Validate tracing.otlp: when otlp.enabled is true, at least one of grpc.enabled o
 
 {{/*
 Convert tracing.resourceAttributes (object) to OTEL CSV format: key1=value1,key2=value2,...
+Values in the map are templated (e.g. "{{ .Release.Name }}") so they are evaluated at render time.
 */}}
 {{- define "medcat-trainer-helm.tracingResourceAttributesCsv" -}}
+{{- $root := . -}}
 {{- $parts := list -}}
 {{- range $name, $value := .Values.tracing.resourceAttributes -}}
-{{- $parts = append $parts (printf "%s=%s" $name $value) -}}
+{{- $parts = append $parts (printf "%s=%s" $name (tpl (toString $value) $root)) -}}
 {{- end -}}
 {{- join "," $parts | quote -}}
 {{- end -}}
diff --git a/deployment/kubernetes/charts/medcat-trainer-helm/templates/medcat-trainer-deployment.yaml b/deployment/kubernetes/charts/medcat-trainer-helm/templates/medcat-trainer-deployment.yaml
index 2739fa3..7de9501 100644
--- a/deployment/kubernetes/charts/medcat-trainer-helm/templates/medcat-trainer-deployment.yaml
+++ b/deployment/kubernetes/charts/medcat-trainer-helm/templates/medcat-trainer-deployment.yaml
@@ -66,6 +66,8 @@ spec:
               {{- tpl (toYaml $value) $ | nindent 14 }}
           {{- end }}
           {{- if .Values.tracing.otlp.enabled }}
+          - name: K8S_CONTAINER_NAME
+            value: medcat-trainer
           - name: OTEL_RESOURCE_ATTRIBUTES  
             value: {{ include "medcat-trainer-helm.tracingResourceAttributesCsv" . }}
           {{- end }}
diff --git a/deployment/kubernetes/charts/medcat-trainer-helm/templates/medcat-trainer-env-configmap.yaml b/deployment/kubernetes/charts/medcat-trainer-helm/templates/medcat-trainer-env-configmap.yaml
index d7e32a6..df95647 100644
--- a/deployment/kubernetes/charts/medcat-trainer-helm/templates/medcat-trainer-env-configmap.yaml
+++ b/deployment/kubernetes/charts/medcat-trainer-helm/templates/medcat-trainer-env-configmap.yaml
@@ -19,7 +19,7 @@ data:
   {{- if .Values.tracing.otlp.enabled }}
     # Tracing configuration
   MCT_ENABLE_TRACING: {{ .Values.tracing.otlp.enabled | quote }}
-  OTEL_TRACES_EXPORTER: console
+  OTEL_TRACES_EXPORTER: "otlp"
   OTEL_SERVICE_NAME: {{ .Values.tracing.serviceName | quote }}
   {{- if .Values.tracing.otlp.grpc.enabled }}
   OTEL_EXPORTER_OTLP_ENDPOINT: {{ .Values.tracing.otlp.grpc.endpoint | quote }}
diff --git a/deployment/kubernetes/charts/medcat-trainer-helm/values.yaml b/deployment/kubernetes/charts/medcat-trainer-helm/values.yaml
index 8b3c6a8..dcf7324 100644
--- a/deployment/kubernetes/charts/medcat-trainer-helm/values.yaml
+++ b/deployment/kubernetes/charts/medcat-trainer-helm/values.yaml
@@ -80,12 +80,14 @@ provisioning:
 tracing:
   # The name of the service in the tracing system
   serviceName: "medcat-trainer"
-  # Resource attributes to add to the traces
+  # Resource attributes to add to the traces. Can be templated
   resourceAttributes:
     k8s.pod.uid: "$(K8S_POD_UID)"
     k8s.pod.name: "$(K8S_POD_NAME)"
     k8s.namespace.name: "$(K8S_POD_NAMESPACE)"
     k8s.node.name: "$(K8S_NODE_NAME)"
+    service.version: "{{ .Values.image.tag | default .Chart.AppVersion }}"
+    service.instance.id: "$(K8S_POD_NAMESPACE)-$(K8S_POD_NAME)-$(K8S_CONTAINER_NAME)"
   experimentalResourceDetectors: "containerid,os"
   # Optionally disable the db instrumentations due to noise.
   disabledInstrumentations: "psycopg,sqlite3"