Grafana incident-response Implementation Plan¶

package v1alpha1

import "testing"

func TestGrafanaSpec_Fields(t *testing.T) {
    g := GrafanaSpec{Enabled: true, URL: "http://grafana:3000", SecretRef: "proj-grafana", CooldownSeconds: 1800}
    if !g.Enabled || g.URL == "" || g.SecretRef == "" || g.CooldownSeconds != 1800 {
        t.Fatalf("GrafanaSpec fields not wired: %+v", g)
    }
    p := ProjectSpec{Grafana: &g}
    if p.Grafana == nil || !p.Grafana.Enabled {
        t.Fatalf("ProjectSpec.Grafana not wired")
    }
    st := ProjectStatus{Grafana: &GrafanaStatus{Phase: "Ready", Endpoint: "http://grafana-mcp-x.ns.svc:8000"}}
    if st.Grafana == nil || st.Grafana.Phase != "Ready" {
        t.Fatalf("ProjectStatus.Grafana not wired")
    }
}

// GrafanaSpec configures the optional per-project Grafana incident-response
// feature: an operator-provisioned read-only grafana-mcp and an alert-webhook
// receiver. The feature is inert unless Enabled.
type GrafanaSpec struct {
    Enabled bool `json:"enabled"`
    // URL is the Grafana base URL grafana-mcp queries (non-secret).
    URL string `json:"url"`
    // SecretRef names a Secret holding the Grafana credentials. Keys:
    //   serviceAccountToken - Grafana Viewer SA token (mounted into grafana-mcp)
    //   webhookSecret       - static bearer the alert webhook must present
    SecretRef string `json:"secretRef"`
    // CooldownSeconds is the per-alert-group refire window (default 3600).
    // +kubebuilder:default=3600
    // +optional
    CooldownSeconds int `json:"cooldownSeconds,omitempty"`
}

// GrafanaStatus reports the observed state of the per-Project grafana-mcp.
type GrafanaStatus struct {
    // +optional
    Phase string `json:"phase,omitempty"`
    // +optional
    Endpoint string `json:"endpoint,omitempty"`
}

    // +optional
    Grafana *GrafanaSpec `json:"grafana,omitempty"`

    // +optional
    Grafana *GrafanaStatus `json:"grafana,omitempty"`

func TestValidateTaskSpec_Incident(t *testing.T) {
    if err := ValidateTaskSpec(TaskSpec{Kind: "incident"}); err != nil {
        t.Fatalf("incident with empty repositoryRef must be valid: %v", err)
    }
    if err := ValidateTaskSpec(TaskSpec{Kind: "incident", RepositoryRef: "r"}); err == nil {
        t.Fatalf("incident with a repositoryRef must be rejected (project-scoped)")
    }
}

var projectScopedKinds = map[string]bool{
    "brainstorm":  true,
    "healthCheck": true,
    "incident":    true,
}

    // +kubebuilder:validation:Enum=implement;review;selfImprove;triageIssue;brainstorm;issueLifecycle;incident

package grafanamcp

import (
    "strings"
    "testing"

    tatarav1alpha1 "github.com/szymonrychu/tatara-operator/api/v1alpha1"
    metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func proj() *tatarav1alpha1.Project {
    p := &tatarav1alpha1.Project{ObjectMeta: metav1.ObjectMeta{Name: "acme", Namespace: "tatara"}}
    p.Spec.Grafana = &tatarav1alpha1.GrafanaSpec{Enabled: true, URL: "http://grafana:3000", SecretRef: "acme-grafana"}
    return p
}

func TestDeployment_ReadOnlyStreamableHTTP(t *testing.T) {
    d := Deployment(proj(), Config{Namespace: "tatara", Image: "grafana/mcp-grafana:v0.1.0"})
    if d.Name != "grafana-mcp-acme" {
        t.Fatalf("name: %s", d.Name)
    }
    c := d.Spec.Template.Spec.Containers[0]
    args := strings.Join(c.Args, " ")
    if !strings.Contains(args, "streamable-http") || !strings.Contains(args, "--disable-write") {
        t.Fatalf("args must be read-only streamable-http: %v", c.Args)
    }
    if c.Ports[0].ContainerPort != 8000 {
        t.Fatalf("port: %d", c.Ports[0].ContainerPort)
    }
    var url, tokenFile string
    for _, e := range c.Env {
        if e.Name == "GRAFANA_URL" {
            url = e.Value
        }
        if e.Name == "GRAFANA_SERVICE_ACCOUNT_TOKEN_FILE" {
            tokenFile = e.Value
        }
    }
    if url != "http://grafana:3000" {
        t.Fatalf("GRAFANA_URL: %q", url)
    }
    if tokenFile != "/etc/grafana/token" {
        t.Fatalf("token file env: %q", tokenFile)
    }
    if c.SecurityContext == nil || c.SecurityContext.RunAsUser == nil {
        t.Fatalf("container needs a numeric runAsUser (runAsNonRoot incident)")
    }
    // token mounted from the project's grafana secret, key serviceAccountToken.
    vol := d.Spec.Template.Spec.Volumes[0]
    if vol.Secret == nil || vol.Secret.SecretName != "acme-grafana" {
        t.Fatalf("token volume must project secret acme-grafana: %+v", vol)
    }
    if vol.Secret.Items[0].Key != "serviceAccountToken" || vol.Secret.Items[0].Path != "token" {
        t.Fatalf("token volume item must be serviceAccountToken->token: %+v", vol.Secret.Items)
    }
}

func TestService_ClusterIP8000(t *testing.T) {
    s := Service(proj(), Config{Namespace: "tatara"})
    if s.Name != "grafana-mcp-acme" || s.Spec.Ports[0].Port != 8000 {
        t.Fatalf("service: %s :%d", s.Name, s.Spec.Ports[0].Port)
    }
}

func TestEndpointAndMCPURL(t *testing.T) {
    if Endpoint("acme", "tatara") != "http://grafana-mcp-acme.tatara.svc:8000" {
        t.Fatalf("endpoint: %s", Endpoint("acme", "tatara"))
    }
    if MCPURL("acme", "tatara") != "http://grafana-mcp-acme.tatara.svc:8000/mcp" {
        t.Fatalf("mcp url: %s", MCPURL("acme", "tatara"))
    }
}

// Package grafanamcp holds pure builder functions producing the per-Project
// read-only grafana-mcp workload (grafana/mcp-grafana, streamable-http,
// --disable-write). No function performs a client call; the ProjectReconciler
// server-side-applies the returned objects (mirrors internal/memory).
package grafanamcp

import (
    "fmt"

    tatarav1alpha1 "github.com/szymonrychu/tatara-operator/api/v1alpha1"
    metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// Config is the operator-level (non-per-Project) input the builders need.
type Config struct {
    Namespace       string
    Image           string
    ImagePullSecret string
}

// Name returns the Deployment/Service name for a project.
func Name(project string) string { return "grafana-mcp-" + project }

// Endpoint is the in-cluster base URL of a project's grafana-mcp.
func Endpoint(project, namespace string) string {
    return fmt.Sprintf("http://grafana-mcp-%s.%s.svc:8000", project, namespace)
}

// MCPURL is the streamable-http MCP endpoint the agent connects to.
func MCPURL(project, namespace string) string { return Endpoint(project, namespace) + "/mcp" }

func labels(project string) map[string]string {
    return map[string]string{
        "app.kubernetes.io/name":     "grafana-mcp",
        "app.kubernetes.io/instance": Name(project),
        "tatara.dev/project":         project,
    }
}

func ownerRef(p *tatarav1alpha1.Project) metav1.OwnerReference {
    return *metav1.NewControllerRef(p, tatarav1alpha1.GroupVersion.WithKind("Project"))
}

func objectMeta(p *tatarav1alpha1.Project, cfg Config, name string) metav1.ObjectMeta {
    return metav1.ObjectMeta{
        Name:            name,
        Namespace:       cfg.Namespace,
        Labels:          labels(p.Name),
        OwnerReferences: []metav1.OwnerReference{ownerRef(p)},
    }
}

package grafanamcp

import (
    tatarav1alpha1 "github.com/szymonrychu/tatara-operator/api/v1alpha1"
    appsv1 "k8s.io/api/apps/v1"
    corev1 "k8s.io/api/core/v1"
    metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    "k8s.io/apimachinery/pkg/util/intstr"
)

const grafanaRunAsUser int64 = 65532 // distroless nonroot; image runs as nonroot

func imagePullSecrets(cfg Config) []corev1.LocalObjectReference {
    if cfg.ImagePullSecret == "" {
        return nil
    }
    return []corev1.LocalObjectReference{{Name: cfg.ImagePullSecret}}
}

// Deployment builds the per-Project read-only grafana-mcp Deployment.
// streamable-http on :8000, --disable-write, Grafana Viewer token mounted from
// the project's grafana secret (key serviceAccountToken) and read via
// GRAFANA_SERVICE_ACCOUNT_TOKEN_FILE (re-read per request).
func Deployment(p *tatarav1alpha1.Project, cfg Config) *appsv1.Deployment {
    name := Name(p.Name)
    replicas := int32(1)
    runAsNonRoot := true
    noPrivEsc := false
    runAsUser := grafanaRunAsUser
    sel := map[string]string{"app.kubernetes.io/instance": name}
    podLabels := labels(p.Name)

    return &appsv1.Deployment{
        TypeMeta:   metav1.TypeMeta{APIVersion: "apps/v1", Kind: "Deployment"},
        ObjectMeta: objectMeta(p, cfg, name),
        Spec: appsv1.DeploymentSpec{
            Replicas: &replicas,
            Selector: &metav1.LabelSelector{MatchLabels: sel},
            Template: corev1.PodTemplateSpec{
                ObjectMeta: metav1.ObjectMeta{Labels: podLabels},
                Spec: corev1.PodSpec{
                    ImagePullSecrets: imagePullSecrets(cfg),
                    Volumes: []corev1.Volume{{
                        Name: "grafana-token",
                        VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{
                            SecretName: p.Spec.Grafana.SecretRef,
                            Items:      []corev1.KeyToPath{{Key: "serviceAccountToken", Path: "token"}},
                        }},
                    }},
                    Containers: []corev1.Container{{
                        Name:  "grafana-mcp",
                        Image: cfg.Image,
                        Args:  []string{"-t", "streamable-http", "--disable-write"},
                        Ports: []corev1.ContainerPort{{Name: "http", ContainerPort: 8000, Protocol: corev1.ProtocolTCP}},
                        Env: []corev1.EnvVar{
                            {Name: "GRAFANA_URL", Value: p.Spec.Grafana.URL},
                            {Name: "GRAFANA_SERVICE_ACCOUNT_TOKEN_FILE", Value: "/etc/grafana/token"},
                        },
                        VolumeMounts: []corev1.VolumeMount{{
                            Name: "grafana-token", MountPath: "/etc/grafana", ReadOnly: true,
                        }},
                        SecurityContext: &corev1.SecurityContext{
                            RunAsNonRoot:             &runAsNonRoot,
                            RunAsUser:                &runAsUser,
                            AllowPrivilegeEscalation: &noPrivEsc,
                        },
                    }},
                },
            },
        },
    }
}

// Service exposes grafana-mcp on 8000 (ClusterIP).
func Service(p *tatarav1alpha1.Project, cfg Config) *corev1.Service {
    name := Name(p.Name)
    return &corev1.Service{
        TypeMeta:   metav1.TypeMeta{APIVersion: "v1", Kind: "Service"},
        ObjectMeta: objectMeta(p, cfg, name),
        Spec: corev1.ServiceSpec{
            Type:     corev1.ServiceTypeClusterIP,
            Selector: map[string]string{"app.kubernetes.io/instance": name},
            Ports:    []corev1.ServicePort{{Name: "http", Port: 8000, TargetPort: intstr.FromString("http"), Protocol: corev1.ProtocolTCP}},
        },
    }
}

package controller

import (
    "context"
    "testing"

    tatarav1alpha1 "github.com/szymonrychu/tatara-operator/api/v1alpha1"
    appsv1 "k8s.io/api/apps/v1"
    corev1 "k8s.io/api/core/v1"
    apierrors "k8s.io/apimachinery/pkg/api/errors"
    metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    "k8s.io/apimachinery/pkg/types"
)

func TestReconcileGrafanaMCP_AppliesWhenEnabled(t *testing.T) {
    ctx := context.Background()
    p := &tatarav1alpha1.Project{}
    p.Name = "gmcp-on"
    p.Namespace = testNS
    p.Spec.ScmSecretRef = "gmcp-on-scm"
    p.Spec.Grafana = &tatarav1alpha1.GrafanaSpec{Enabled: true, URL: "http://grafana:3000", SecretRef: "gmcp-on-grafana"}
    if err := k8sClient.Create(ctx, p); err != nil {
        t.Fatalf("create project: %v", err)
    }

    r := &ProjectReconciler{Client: k8sClient, Scheme: k8sClient.Scheme()}
    r.GrafanaConfig.Namespace = testNS
    r.GrafanaConfig.Image = "grafana/mcp-grafana:test"

    if _, err := r.reconcileGrafanaMCP(ctx, p); err != nil {
        t.Fatalf("reconcileGrafanaMCP: %v", err)
    }

    var d appsv1.Deployment
    if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: testNS, Name: "grafana-mcp-gmcp-on"}, &d); err != nil {
        t.Fatalf("expected grafana-mcp deployment: %v", err)
    }
    var svc corev1.Service
    if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: testNS, Name: "grafana-mcp-gmcp-on"}, &svc); err != nil {
        t.Fatalf("expected grafana-mcp service: %v", err)
    }
    if p.Status.Grafana == nil || p.Status.Grafana.Endpoint == "" {
        t.Fatalf("status.grafana not set: %+v", p.Status.Grafana)
    }
}

func TestReconcileGrafanaMCP_TeardownWhenDisabled(t *testing.T) {
    ctx := context.Background()
    p := &tatarav1alpha1.Project{}
    p.Name = "gmcp-off"
    p.Namespace = testNS
    p.Spec.ScmSecretRef = "gmcp-off-scm"
    // No Grafana spec -> feature off.
    if err := k8sClient.Create(ctx, p); err != nil {
        t.Fatalf("create project: %v", err)
    }
    r := &ProjectReconciler{Client: k8sClient, Scheme: k8sClient.Scheme()}
    r.GrafanaConfig.Namespace = testNS
    r.GrafanaConfig.Image = "grafana/mcp-grafana:test"

    if _, err := r.reconcileGrafanaMCP(ctx, p); err != nil {
        t.Fatalf("reconcile (disabled): %v", err)
    }
    var d appsv1.Deployment
    err := k8sClient.Get(ctx, types.NamespacedName{Namespace: testNS, Name: "grafana-mcp-gmcp-off"}, &d)
    if err == nil || !apierrors.IsNotFound(err) {
        t.Fatalf("disabled project must have NO grafana-mcp deployment; got err=%v", err)
    }
    _ = metav1.Now()
}

package controller

import (
    "context"
    "fmt"
    "time"

    tataradevv1alpha1 "github.com/szymonrychu/tatara-operator/api/v1alpha1"
    "github.com/szymonrychu/tatara-operator/internal/grafanamcp"
    appsv1 "k8s.io/api/apps/v1"
    corev1 "k8s.io/api/core/v1"
    apierrors "k8s.io/apimachinery/pkg/api/errors"
    "k8s.io/apimachinery/pkg/types"
    "sigs.k8s.io/controller-runtime/pkg/client"
    "sigs.k8s.io/controller-runtime/pkg/log"
)

const grafanaFieldOwner = "tatara-operator"
const grafanaRequeue = 10 * time.Second

// reconcileGrafanaMCP provisions (or tears down) the per-Project read-only
// grafana-mcp workload, gated on Spec.Grafana.Enabled. It mirrors
// reconcileMemory: SSA-apply when enabled, status roll-up, teardown when off.
// Failure is isolated and does not block other reconciles.
func (r *ProjectReconciler) reconcileGrafanaMCP(ctx context.Context, p *tataradevv1alpha1.Project) (time.Duration, error) {
    l := log.FromContext(ctx)
    ns := r.GrafanaConfig.Namespace
    name := grafanamcp.Name(p.Name)

    enabled := p.Spec.Grafana != nil && p.Spec.Grafana.Enabled
    if !enabled {
        // Teardown: best-effort delete of a previously-applied workload.
        _ = r.Delete(ctx, &appsv1.Deployment{ObjectMeta: objMeta(ns, name)})
        _ = r.Delete(ctx, &corev1.Service{ObjectMeta: objMeta(ns, name)})
        p.Status.Grafana = nil
        return 0, nil
    }

    objs := []client.Object{
        grafanamcp.Deployment(p, r.GrafanaConfig),
        grafanamcp.Service(p, r.GrafanaConfig),
    }
    for _, obj := range objs {
        if err := r.Patch(ctx, obj, client.Apply, //nolint:staticcheck
            client.FieldOwner(grafanaFieldOwner), client.ForceOwnership); err != nil {
            p.Status.Grafana = &tataradevv1alpha1.GrafanaStatus{Phase: "Failed", Endpoint: grafanamcp.Endpoint(p.Name, ns)}
            l.Error(err, "grafana-mcp apply failed", "action", "grafana_apply", "resource_id", p.Name)
            return 0, fmt.Errorf("apply %T %s: %w", obj, obj.GetName(), err)
        }
    }

    phase := "Provisioning"
    var d appsv1.Deployment
    if err := r.Get(ctx, types.NamespacedName{Namespace: ns, Name: name}, &d); err == nil {
        if d.Status.AvailableReplicas >= 1 {
            phase = "Ready"
        }
    } else if !apierrors.IsNotFound(err) {
        return grafanaRequeue, nil // transient cache blip
    }
    p.Status.Grafana = &tataradevv1alpha1.GrafanaStatus{Phase: phase, Endpoint: grafanamcp.Endpoint(p.Name, ns)}
    if phase != "Ready" {
        return grafanaRequeue, nil
    }
    return 0, nil
}

func objMeta(ns, name string) metav1ObjectMeta { return metav1ObjectMeta{Name: name, Namespace: ns} }

func objMeta(ns, name string) metav1.ObjectMeta { return metav1.ObjectMeta{Name: name, Namespace: ns} }

    GrafanaConfig grafanamcp.Config

    grafanaRequeueAfter, grafErr := r.reconcileGrafanaMCP(ctx, &project)
    if grafErr != nil {
        l.Error(grafErr, "grafana-mcp reconcile failed (non-blocking)", "resource_id", project.Name)
    }
    if grafanaRequeueAfter > 0 && (requeueAfter == 0 || grafanaRequeueAfter < requeueAfter) {
        requeueAfter = grafanaRequeueAfter
    }

        GrafanaConfig: grafanamcp.Config{
            Namespace:       cfg.Namespace,
            Image:           cfg.GrafanaMCPImage,
            ImagePullSecret: cfg.ImagePullSecret, // reuse the same field memoryConfigFromConfig uses
        },

package agent

import (
    "testing"

    tatarav1alpha1 "github.com/szymonrychu/tatara-operator/api/v1alpha1"
    metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func grafanaEnv(pod interface{ getEnv() map[string]string }) {}

func podEnvMap(t *testing.T, project *tatarav1alpha1.Project) map[string]string {
    t.Helper()
    task := &tatarav1alpha1.Task{ObjectMeta: metav1.ObjectMeta{Name: "t1"}, Spec: tatarav1alpha1.TaskSpec{ProjectRef: project.Name, Kind: "incident"}}
    pod := BuildPod(project, nil, task, nil, "http://mem", PodConfig{Namespace: "tatara"})
    m := map[string]string{}
    for _, e := range pod.Spec.Containers[0].Env {
        m[e.Name] = e.Value
    }
    return m
}

func TestBuildPod_GrafanaMCPURL_WhenEnabled(t *testing.T) {
    p := &tatarav1alpha1.Project{ObjectMeta: metav1.ObjectMeta{Name: "acme"}}
    p.Spec.Grafana = &tatarav1alpha1.GrafanaSpec{Enabled: true, URL: "http://g", SecretRef: "s"}
    m := podEnvMap(t, p)
    if m["TATARA_GRAFANA_MCP_URL"] != "http://grafana-mcp-acme.tatara.svc:8000/mcp" {
        t.Fatalf("grafana mcp url env wrong/missing: %q", m["TATARA_GRAFANA_MCP_URL"])
    }
}

func TestBuildPod_NoGrafanaMCPURL_WhenDisabled(t *testing.T) {
    p := &tatarav1alpha1.Project{ObjectMeta: metav1.ObjectMeta{Name: "acme"}}
    m := podEnvMap(t, p)
    if _, ok := m["TATARA_GRAFANA_MCP_URL"]; ok {
        t.Fatalf("grafana mcp url env must be absent when feature off")
    }
}

    if project.Spec.Grafana != nil && project.Spec.Grafana.Enabled {
        // Per-project read-only grafana-mcp endpoint. The wrapper registers this
        // as an HTTP MCP server so the agent can query Grafana for live debugging.
        env = append(env, corev1.EnvVar{
            Name:  "TATARA_GRAFANA_MCP_URL",
            Value: grafanamcp.MCPURL(project.Name, cfg.Namespace),
        })
    }

package incident

import (
    "strings"
    "testing"
)

func TestGoalProject(t *testing.T) {
    g := GoalProject("groupKey=abc status=firing commonLabels={alertname=HighCPU}", []string{"o/api", "o/web"})
    for _, kw := range []string{"o/api", "o/web", "groupKey=abc", "grafana", "propose_issue", "read-only"} {
        if !strings.Contains(g, kw) {
            t.Fatalf("incident goal missing %q:\n%s", kw, g)
        }
    }
    // Must forbid remediation/write actions.
    if !strings.Contains(g, "Do NOT") || !strings.Contains(strings.ToLower(g), "remediat") {
        t.Fatalf("incident goal must forbid remediation:\n%s", g)
    }
}

// Package incident builds the turn-0 goal for a Grafana-fired incident Task.
// It lives in its own package so both the webhook receiver and the controller
// can import it without a cycle.
package incident

import "strings"

// GoalProject returns the turn-0 goal for a project-scoped incident Task fired
// by a Grafana alert. The agent investigates live (read-only) via the Grafana
// MCP server, then files exactly one evidence issue via propose_issue, choosing
// the repo the evidence implicates. alertCtx is a pre-rendered compact block of
// the alert (group key, status, labels, annotations, generator/external URLs).
func GoalProject(alertCtx string, slugs []string) string {
    repoList := strings.Join(slugs, ", ")
    return "A Grafana alert is FIRING for this project. Investigate it and hand a well-evidenced issue " +
        "to the team. Repositories in this project: " + repoList + ".\n\n" +
        "ALERT:\n" + alertCtx + "\n\n" +
        "Investigate LIVE using the `grafana` MCP server (read-only): query the relevant Prometheus/Loki " +
        "datasources, read the firing alert rule (follow its generatorURL), and inspect related dashboards. " +
        "Form a diagnosis backed by the queries you ran and their results.\n\n" +
        "Then call propose_issue(repo, body) EXACTLY ONCE. Choose the `repo` (from the list above) that the " +
        "evidence implicates. The body MUST contain: the alert summary, the queries/tools you ran and their " +
        "results, your diagnosis, and the Grafana links (generatorURL/externalURL). The issue lands with the " +
        "brainstorming label and the normal triage/brainstorm flow takes over.\n\n" +
        "If the grafana MCP server is unreachable, still file the issue with the raw alert and note the MCP was " +
        "unavailable. If after investigation this is a confirmed false positive, finish with a one-line note and " +
        "do NOT open an issue.\n\n" +
        "This is a READ-ONLY investigation. Do NOT take any remediation, write, or corrective action on any " +
        "system. Your only output is the issue (or the false-positive note)."
}

package webhook

import (
    "strings"
    "testing"
)

const grafanaFiring = `{"status":"firing","groupKey":"{}/{alertname=\"HighCPU\"}","commonLabels":{"alertname":"HighCPU","severity":"critical"},"commonAnnotations":{"summary":"CPU high"},"externalURL":"http://grafana:3000","alerts":[{"status":"firing","labels":{"alertname":"HighCPU","instance":"node1"},"annotations":{"summary":"CPU high on node1"},"startsAt":"2026-06-19T00:00:00Z","generatorURL":"http://grafana:3000/alerting/rule","fingerprint":"abc123"}]}`

const grafanaResolved = `{"status":"resolved","groupKey":"g","alerts":[]}`

func TestParseGrafanaAlert(t *testing.T) {
    a, err := parseGrafanaAlert([]byte(grafanaFiring))
    if err != nil {
        t.Fatalf("parse: %v", err)
    }
    if a.Status != "firing" || a.GroupKey == "" || len(a.Alerts) != 1 {
        t.Fatalf("parsed wrong: %+v", a)
    }
    if a.Alerts[0].GeneratorURL == "" || a.CommonLabels["severity"] != "critical" {
        t.Fatalf("fields missing: %+v", a)
    }
}

func TestRenderAlertContext(t *testing.T) {
    a, _ := parseGrafanaAlert([]byte(grafanaFiring))
    ctx := renderAlertContext(a)
    for _, kw := range []string{"firing", "HighCPU", "generatorURL", "http://grafana:3000"} {
        if !strings.Contains(ctx, kw) {
            t.Fatalf("rendered ctx missing %q:\n%s", kw, ctx)
        }
    }
}

func TestAlertGroupHash_StableAndDistinct(t *testing.T) {
    a, _ := parseGrafanaAlert([]byte(grafanaFiring))
    b, _ := parseGrafanaAlert([]byte(grafanaResolved))
    if alertGroupHash(a) == "" || len(alertGroupHash(a)) != 16 {
        t.Fatalf("hash must be 16 hex: %q", alertGroupHash(a))
    }
    if alertGroupHash(a) == alertGroupHash(b) {
        t.Fatalf("distinct groupKeys must hash differently")
    }
}

package webhook

import (
    "crypto/sha256"
    "encoding/hex"
    "encoding/json"
    "fmt"
    "sort"
    "strings"
)

// GrafanaAlert is the subset of the Grafana unified-alerting webhook payload
// (Alertmanager-compatible) the receiver needs.
type GrafanaAlert struct {
    Status            string            `json:"status"`
    GroupKey          string            `json:"groupKey"`
    CommonLabels      map[string]string `json:"commonLabels"`
    CommonAnnotations map[string]string `json:"commonAnnotations"`
    ExternalURL       string            `json:"externalURL"`
    Alerts            []GrafanaAlertItem `json:"alerts"`
}

type GrafanaAlertItem struct {
    Status       string            `json:"status"`
    Labels       map[string]string `json:"labels"`
    Annotations  map[string]string `json:"annotations"`
    StartsAt     string            `json:"startsAt"`
    GeneratorURL string            `json:"generatorURL"`
    Fingerprint  string            `json:"fingerprint"`
}

func parseGrafanaAlert(body []byte) (GrafanaAlert, error) {
    var a GrafanaAlert
    if err := json.Unmarshal(body, &a); err != nil {
        return GrafanaAlert{}, fmt.Errorf("parse grafana alert: %w", err)
    }
    if a.Status == "" {
        return GrafanaAlert{}, fmt.Errorf("grafana alert missing status")
    }
    return a, nil
}

func sortedKV(m map[string]string) string {
    keys := make([]string, 0, len(m))
    for k := range m {
        keys = append(keys, k)
    }
    sort.Strings(keys)
    parts := make([]string, 0, len(keys))
    for _, k := range keys {
        parts = append(parts, k+"="+m[k])
    }
    return strings.Join(parts, ", ")
}

// renderAlertContext produces the compact alert block embedded in the incident
// goal (one line per fact; per-alert generatorURL/labels included).
func renderAlertContext(a GrafanaAlert) string {
    var b strings.Builder
    fmt.Fprintf(&b, "status=%s groupKey=%s\n", a.Status, a.GroupKey)
    fmt.Fprintf(&b, "commonLabels: {%s}\n", sortedKV(a.CommonLabels))
    fmt.Fprintf(&b, "commonAnnotations: {%s}\n", sortedKV(a.CommonAnnotations))
    fmt.Fprintf(&b, "externalURL: %s\n", a.ExternalURL)
    for i, it := range a.Alerts {
        fmt.Fprintf(&b, "alert[%d]: status=%s labels={%s} annotations={%s} startsAt=%s generatorURL=%s\n",
            i, it.Status, sortedKV(it.Labels), sortedKV(it.Annotations), it.StartsAt, it.GeneratorURL)
    }
    return strings.TrimRight(b.String(), "\n")
}

// alertGroupHash is the dedup key for an alert group (16 hex of groupKey).
func alertGroupHash(a GrafanaAlert) string {
    h := sha256.Sum256([]byte(a.GroupKey))
    return hex.EncodeToString(h[:])[:16]
}

package webhook

import (
    "net/http"
    "net/http/httptest"
    "strings"
    "testing"

    "github.com/go-chi/chi/v5"
    tatarav1 "github.com/szymonrychu/tatara-operator/api/v1alpha1"
    "github.com/szymonrychu/tatara-operator/internal/obs"
    corev1 "k8s.io/api/core/v1"
    metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    "k8s.io/apimachinery/pkg/runtime"
    "github.com/prometheus/client_golang/prometheus"
    "sigs.k8s.io/controller-runtime/pkg/client"
    "sigs.k8s.io/controller-runtime/pkg/client/fake"
)

func grafanaRouter(t *testing.T, objs ...client.Object) (*chi.Mux, client.Client) {
    t.Helper()
    sch := runtime.NewScheme()
    _ = tatarav1.AddToScheme(sch)
    _ = corev1.AddToScheme(sch)
    fc := fake.NewClientBuilder().WithScheme(sch).WithObjects(objs...).
        WithStatusSubresource(&tatarav1.Project{}, &tatarav1.Task{}).Build()
    s := NewServer(Config{Client: fc, Namespace: "tatara", Metrics: obs.NewOperatorMetrics(prometheus.NewRegistry())})
    r := chi.NewRouter()
    s.Mount(r)
    return r, fc
}

func grafanaProject(name string) *tatarav1.Project {
    p := &tatarav1.Project{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "tatara"}}
    p.Spec.Grafana = &tatarav1.GrafanaSpec{Enabled: true, URL: "http://g", SecretRef: name + "-grafana", CooldownSeconds: 3600}
    return p
}

func grafanaSecret(name string) *corev1.Secret {
    return &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name + "-grafana", Namespace: "tatara"},
        Data: map[string][]byte{"webhookSecret": []byte("tok"), "serviceAccountToken": []byte("sa")}}
}

func postGrafana(r *chi.Mux, project, bearer, body string) *httptest.ResponseRecorder {
    req := httptest.NewRequest(http.MethodPost, "/operator/webhooks/"+project+"/grafana", strings.NewReader(body))
    if bearer != "" {
        req.Header.Set("Authorization", "Bearer "+bearer)
    }
    req.Header.Set("Content-Type", "application/json")
    w := httptest.NewRecorder()
    r.ServeHTTP(w, req)
    return w
}

func listIncidentTasks(t *testing.T, fc client.Client) []tatarav1.Task {
    t.Helper()
    var tl tatarav1.TaskList
    _ = fc.List(t.Context(), &tl)
    var out []tatarav1.Task
    for _, x := range tl.Items {
        if x.Spec.Kind == "incident" {
            out = append(out, x)
        }
    }
    return out
}

func TestGrafana_FiringCreatesIncident(t *testing.T) {
    r, fc := grafanaRouter(t, grafanaProject("p1"), grafanaSecret("p1"))
    w := postGrafana(r, "p1", "tok", grafanaFiring)
    if w.Code != http.StatusAccepted {
        t.Fatalf("want 202, got %d: %s", w.Code, w.Body.String())
    }
    if n := len(listIncidentTasks(t, fc)); n != 1 {
        t.Fatalf("want 1 incident task, got %d", n)
    }
}

func TestGrafana_BadBearer401(t *testing.T) {
    r, _ := grafanaRouter(t, grafanaProject("p2"), grafanaSecret("p2"))
    if w := postGrafana(r, "p2", "wrong", grafanaFiring); w.Code != http.StatusUnauthorized {
        t.Fatalf("want 401, got %d", w.Code)
    }
}

func TestGrafana_ResolvedIgnored(t *testing.T) {
    r, fc := grafanaRouter(t, grafanaProject("p3"), grafanaSecret("p3"))
    w := postGrafana(r, "p3", "tok", grafanaResolved)
    if w.Code != http.StatusAccepted {
        t.Fatalf("want 202, got %d", w.Code)
    }
    if n := len(listIncidentTasks(t, fc)); n != 0 {
        t.Fatalf("resolved must create no task, got %d", n)
    }
}

func TestGrafana_DedupInFlight(t *testing.T) {
    r, fc := grafanaRouter(t, grafanaProject("p4"), grafanaSecret("p4"))
    _ = postGrafana(r, "p4", "tok", grafanaFiring)
    _ = postGrafana(r, "p4", "tok", grafanaFiring) // same groupKey, in-flight
    if n := len(listIncidentTasks(t, fc)); n != 1 {
        t.Fatalf("dedup failed: want 1 incident task, got %d", n)
    }
}

func TestGrafana_DisabledProject(t *testing.T) {
    p := &tatarav1.Project{ObjectMeta: metav1.ObjectMeta{Name: "p5", Namespace: "tatara"}} // no Grafana
    r, _ := grafanaRouter(t, p)
    if w := postGrafana(r, "p5", "tok", grafanaFiring); w.Code != http.StatusNotFound {
        t.Fatalf("disabled project must 404, got %d", w.Code)
    }
}