MultiCluster on OpenShift

# Name for the spoke cluster in ACM (must be a valid Kubernetes resource name)
export SPOKE_CLUSTER_NAME="spoke"

# Istio version to install. Must be a version supported by the installed OSSM operator.
# After installing the operator (Phase 3.2), you can list supported versions with:
#   oc --context=ossm-kiali-spoke get crd istios.sailoperator.io \
#     -o jsonpath='{.spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.version.enum}'
# Must be >= 1.30 for ambient cross-cluster traffic routing.
export ISTIO_VERSION="1.30.1"

# meshID - arbitrary identifier for this mesh
export MESH_ID="mesh1"

# SeaweedFS S3 credentials for in-cluster Thanos object storage (ACM Observability)
# These are only used inside the cluster — no external storage account is required
export SEAWEEDFS_ACCESS_KEY="seaweedfs"
export SEAWEEDFS_SECRET_KEY="seaweedfs123"

oc --context=ossm-kiali-hub   whoami --show-server
oc --context=ossm-kiali-spoke whoami --show-server

ACM_CHANNEL=$(oc --context=ossm-kiali-hub get packagemanifest advanced-cluster-management \
  -n openshift-marketplace \
  -o jsonpath='{.status.channels[*].name}' | \
  tr ' ' '\n' | sort -V | tail -1)
echo "Using ACM channel: ${ACM_CHANNEL}"

oc --context=ossm-kiali-hub create namespace open-cluster-management 2>/dev/null || true

oc --context=ossm-kiali-hub apply -f - <<'EOF'
apiVersion: operators.coreos.com/v1
kind: OperatorGroup
metadata:
  name: open-cluster-management
  namespace: open-cluster-management
spec:
  targetNamespaces:
  - open-cluster-management
EOF

oc --context=ossm-kiali-hub apply -f - <<EOF
apiVersion: operators.coreos.com/v1alpha1
kind: Subscription
metadata:
  name: acm-operator-subscription
  namespace: open-cluster-management
spec:
  sourceNamespace: openshift-marketplace
  source: redhat-operators
  channel: ${ACM_CHANNEL}
  installPlanApproval: Automatic
  name: advanced-cluster-management
EOF

until oc --context=ossm-kiali-hub get crd \
  multiclusterhubs.operator.open-cluster-management.io &>/dev/null; do
  echo "Waiting for MCH CRD to appear..."
  sleep 5
done

oc --context=ossm-kiali-hub wait crd/multiclusterhubs.operator.open-cluster-management.io \
  --for=condition=Established \
  --timeout=300s

until oc --context=ossm-kiali-hub get pods \
  -l name=multiclusterhub-operator \
  -n open-cluster-management \
  --no-headers 2>/dev/null | grep -q .; do
  sleep 5
done

oc --context=ossm-kiali-hub wait pod \
  -l name=multiclusterhub-operator \
  -n open-cluster-management \
  --for=condition=Ready \
  --timeout=300s

oc --context=ossm-kiali-hub apply -f - <<'EOF'
apiVersion: operator.open-cluster-management.io/v1
kind: MultiClusterHub
metadata:
  name: multiclusterhub
  namespace: open-cluster-management
spec: {}
EOF

echo "Waiting for MultiClusterHub to reach Running status..."
while true; do
  PHASE=$(oc --context=ossm-kiali-hub get mch multiclusterhub \
    -n open-cluster-management \
    -o jsonpath='{.status.phase}' 2>/dev/null)
  if [ "${PHASE}" = "Running" ]; then
    echo "MultiClusterHub is Running"
    break
  fi
  echo "  Current phase: ${PHASE} — waiting..."
  sleep 15
done

oc --context=ossm-kiali-hub get multiclusterhub multiclusterhub -n open-cluster-management \
  -o jsonpath='{.status.phase}{"\n"}'
# Expected: Running

oc --context=ossm-kiali-hub create namespace open-cluster-management-observability 2>/dev/null || true

oc --context=ossm-kiali-hub apply -f - <<EOF
apiVersion: apps/v1
kind: Deployment
metadata:
  name: seaweedfs
  namespace: open-cluster-management-observability
spec:
  replicas: 1
  selector:
    matchLabels:
      app: seaweedfs
  template:
    metadata:
      labels:
        app: seaweedfs
    spec:
      containers:
      - name: seaweedfs
        image: ghcr.io/chrislusf/seaweedfs:4.47
        args:
        - mini
        - -dir=/data
        - -admin.port=12646
        - -master.telemetry=false
        env:
        - name: AWS_ACCESS_KEY_ID
          value: "${SEAWEEDFS_ACCESS_KEY}"
        - name: AWS_SECRET_ACCESS_KEY
          value: "${SEAWEEDFS_SECRET_KEY}"
        - name: S3_BUCKET
          value: thanos
        ports:
        - containerPort: 8333
          name: s3
        volumeMounts:
        - name: data
          mountPath: /data
        readinessProbe:
          tcpSocket:
            port: 8333
          initialDelaySeconds: 10
          periodSeconds: 5
        livenessProbe:
          tcpSocket:
            port: 8333
          initialDelaySeconds: 10
          periodSeconds: 5
      volumes:
      - name: data
        emptyDir: {}
---
apiVersion: v1
kind: Service
metadata:
  name: seaweedfs
  namespace: open-cluster-management-observability
spec:
  ports:
  - port: 8333
    name: s3
    targetPort: 8333
  selector:
    app: seaweedfs
EOF

oc --context=ossm-kiali-hub rollout status deployment/seaweedfs \
  -n open-cluster-management-observability \
  --timeout=120s

oc --context=ossm-kiali-hub apply -f - <<EOF
apiVersion: v1
kind: Secret
metadata:
  name: thanos-object-storage
  namespace: open-cluster-management-observability
type: Opaque
stringData:
  thanos.yaml: |
    type: s3
    config:
      bucket: thanos
      endpoint: seaweedfs.open-cluster-management-observability.svc:8333
      insecure: true
      access_key: ${SEAWEEDFS_ACCESS_KEY}
      secret_key: ${SEAWEEDFS_SECRET_KEY}
EOF

oc --context=ossm-kiali-hub apply -f - <<'EOF'
apiVersion: observability.open-cluster-management.io/v1beta2
kind: MultiClusterObservability
metadata:
  name: observability
spec:
  capabilities:
    platform:
      metrics:
        default:
          enabled: true
    userWorkloads:
      metrics:
        default:
          enabled: true
  observabilityAddonSpec: {}
  storageConfig:
    metricObjectStorage:
      name: thanos-object-storage
      key: thanos.yaml
    alertmanagerStorageSize: 1Gi
    compactStorageSize: 10Gi
    receiveStorageSize: 10Gi
    ruleStorageSize: 1Gi
    storeStorageSize: 10Gi
  advanced:
    retentionConfig:
      retentionInLocal: 24h
      retentionResolution1h: 365d
      retentionResolution5m: 365d
      retentionResolutionRaw: 365d
    alertmanager:
      replicas: 1
      resources:
        requests:
          cpu: 20m
          memory: 64Mi
    compact:
      resources:
        requests:
          cpu: 50m
          memory: 128Mi
    grafana:
      replicas: 1
      resources:
        requests:
          cpu: 20m
          memory: 64Mi
    observatoriumAPI:
      replicas: 1
      resources:
        requests:
          cpu: 20m
          memory: 64Mi
    query:
      replicas: 1
      resources:
        requests:
          cpu: 50m
          memory: 128Mi
    queryFrontend:
      replicas: 1
      resources:
        requests:
          cpu: 50m
          memory: 64Mi
    queryFrontendMemcached:
      replicas: 1
      resources:
        requests:
          cpu: 20m
          memory: 64Mi
    rbacQueryProxy:
      replicas: 1
      resources:
        requests:
          cpu: 20m
          memory: 64Mi
    receive:
      resources:
        requests:
          cpu: 50m
          memory: 128Mi
    rule:
      replicas: 1
      resources:
        requests:
          cpu: 50m
          memory: 128Mi
    store:
      replicas: 1
      resources:
        requests:
          cpu: 50m
          memory: 128Mi
    storeMemcached:
      replicas: 1
      resources:
        requests:
          cpu: 20m
          memory: 64Mi
EOF

echo "Waiting for MultiClusterObservability to be ready..."
while true; do
  READY=$(oc --context=ossm-kiali-hub get mco observability \
    -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)
  READY=${READY:-Unknown}
  if [ "${READY}" = "True" ]; then
    echo "MultiClusterObservability is Ready"
    break
  fi
  echo "  Ready=${READY} — waiting..."
  sleep 15
done

oc --context=ossm-kiali-hub get route observatorium-api \
  -n open-cluster-management-observability \
  -o jsonpath='{.spec.host}{"\n"}'

ADDON_JSON=$(oc --context=ossm-kiali-hub get clustermanagementaddon \
  multicluster-observability-addon -o json)

PLACEMENT_COUNT=$(echo "${ADDON_JSON}" | \
  jq '(.spec.installStrategy.placements // []) | length')

if [ "${PLACEMENT_COUNT}" -eq 1 ]; then
  MCOA_PLACEMENT_NAME=$(echo "${ADDON_JSON}" | \
    jq -r '.spec.installStrategy.placements[0].name')
  MCOA_PLACEMENT_NS=$(echo "${ADDON_JSON}" | \
    jq -r '.spec.installStrategy.placements[0].namespace')
  echo "Using placement: ${MCOA_PLACEMENT_NS}/${MCOA_PLACEMENT_NAME}"
elif [ "${PLACEMENT_COUNT}" -gt 1 ]; then
  echo "Multiple placements found — set MCOA_PLACEMENT_NAME and MCOA_PLACEMENT_NS manually:"
  echo "${ADDON_JSON}" | jq -r \
    '(.spec.installStrategy.placements // [])[] | "  name=\(.name) namespace=\(.namespace)"'
else
  echo "ERROR: no MCOA placements found; ensure MCO is Ready"
fi

oc --context=ossm-kiali-hub apply -f - <<'EOF'
apiVersion: monitoring.rhobs/v1alpha1
kind: ScrapeConfig
metadata:
  labels:
    app.kubernetes.io/component: user-workload-metrics-collector
    app.kubernetes.io/managed-by: kiali-mcoa-federation
  name: kiali-istio-federation
  namespace: open-cluster-management-observability
spec:
  honorLabels: true
  jobName: kiali-istio-federation
  metricRelabelings:
  - action: replace
    regex: 'workload:(.*)'
    replacement: '${1}'
    sourceLabels: [__name__]
    targetLabel: __name__
  - action: replace
    regex: 'kiali:(.*)'
    replacement: '${1}'
    sourceLabels: [__name__]
    targetLabel: __name__
  metricsPath: /federate
  params:
    match[]:
    # CORE TIER
    # Istio traffic (pod-aggregated on edge via recording rules)
    - '{__name__=~"workload:istio_requests_total"}'
    - '{__name__=~"workload:istio_request_bytes_(bucket|count|sum)"}'
    - '{__name__=~"workload:istio_request_duration_milliseconds_(bucket|count|sum)"}'
    - '{__name__=~"workload:istio_request_messages_total"}'
    - '{__name__=~"workload:istio_response_bytes_(bucket|count|sum)"}'
    - '{__name__=~"workload:istio_response_messages_total"}'
    - '{__name__=~"workload:istio_tcp_connections_(opened|closed)_total"}'
    - '{__name__=~"workload:istio_tcp_(received|sent)_bytes_total"}'
    # Istio info (not aggregated)
    - '{__name__=~"istio_build"}'
    # Control plane overview (Kiali mesh page)
    - '{__name__=~"process_cpu_seconds_total"}'
    - '{__name__=~"process_resident_memory_bytes"}'
    - '{__name__=~"pilot_info"}'
    - '{__name__=~"pilot_proxy_convergence_time_(sum|count)"}'
    - '{__name__=~"pilot_services"}'
    - '{__name__=~"pilot_xds$"}'
    - '{__name__=~"pilot_xds_pushes"}'
    - '{__name__=~"workload_manager_active_proxy_count"}'
    # Envoy workload details (all metrics used by Kiali Envoy tab)
    - '{__name__=~"envoy_cluster_upstream_cx_active"}'
    - '{__name__=~"envoy_cluster_upstream_rq_total"}'
    - '{__name__=~"envoy_listener_downstream_cx_active"}'
    - '{__name__=~"envoy_listener_http_downstream_rq"}'
    - '{__name__=~"envoy_server_memory_allocated"}'
    - '{__name__=~"envoy_server_memory_heap_size"}'
    - '{__name__=~"envoy_server_uptime"}'
    # KIALI TIER
    # Kiali self-monitoring metrics
    - '{__name__=~"kiali:kiali_.*"}'
    # ISTIO DASHBOARD TIER
    # istio-control-plane + istio-performance (Istiod runtime)
    - '{__name__=~"pilot_k8s_(reg|cfg)_events"}'
    - '{__name__=~"pilot_push_triggers"}'
    - '{__name__=~"pilot_total_xds_(rejects|internal_errors)"}'
    - '{__name__=~"pilot_xds_push_time_bucket"}'
    - '{__name__=~"pilot_xds_config_size_bytes_bucket"}'
    - '{__name__=~"go_goroutines"}'
    - '{__name__=~"go_memstats_(alloc_bytes|alloc_bytes_total|heap_alloc_bytes|heap_inuse_bytes|heap_sys_bytes|mallocs_total|stack_inuse_bytes)"}'
    # istio-performance (container/process detail)
    - '{__name__=~"container_fs_usage_bytes"}'
    - '{__name__=~"process_open_fds"}'
    - '{__name__=~"process_virtual_memory_bytes"}'
    # istio-ztunnel-dashboard (ambient)
    - '{__name__=~"istio_dns_requests_total"}'
    - '{__name__=~"istio_tcp_sockets_open"}'
    - '{__name__=~"istio_xds_connection_terminations_total"}'
    - '{__name__=~"istio_xds_message_total"}'
    - '{__name__=~"workload_manager_pending_proxy_count"}'
    # istio-extension-dashboard (WASM)
    - '{__name__=~"envoy_wasm_.*"}'
  scrapeInterval: 5m
EOF

oc --context=ossm-kiali-hub apply -f - <<'EOF'
apiVersion: monitoring.rhobs/v1alpha1
kind: ScrapeConfig
metadata:
  labels:
    app.kubernetes.io/component: platform-metrics-collector
    app.kubernetes.io/managed-by: kiali-mcoa-federation
  name: kiali-istio-platform-federation
  namespace: open-cluster-management-observability
spec:
  jobName: kiali-istio-platform-federation
  metricsPath: /federate
  params:
    match[]:
    - '{__name__=~"container_cpu_usage_seconds_total|container_memory_working_set_bytes"}'
  scrapeInterval: 5m
EOF

oc --context=ossm-kiali-spoke create namespace mesh-observability --dry-run=client -o yaml | \
  oc --context=ossm-kiali-spoke apply -f -
oc --context=ossm-kiali-spoke apply -f - <<'EOF'
apiVersion: v1
kind: ConfigMap
metadata:
  name: user-workload-monitoring-config
  namespace: openshift-user-workload-monitoring
data:
  config.yaml: |
    namespacesWithoutLabelEnforcement:
    - mesh-observability
EOF

oc --context=ossm-kiali-hub apply -f - <<'EOF'
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  annotations:
    observability.open-cluster-management.io/target-namespace: mesh-observability
  labels:
    app.kubernetes.io/component: user-workload-metrics-collector
    app.kubernetes.io/managed-by: kiali-mcoa-federation
    openshift.io/prometheus-rule-evaluation-scope: leaf-prometheus
  name: kiali-istio-aggregation
  namespace: open-cluster-management-observability
spec:
  groups:
  - interval: 30s
    name: istio.workload-aggregation
    rules:
    # CORE TIER
    - expr: sum without (pod, pod_template_hash, instance, job, node) (istio_requests_total)
      record: workload:istio_requests_total
    - expr: sum without (pod, pod_template_hash, instance, job, node) (istio_request_messages_total)
      record: workload:istio_request_messages_total
    - expr: sum without (pod, pod_template_hash, instance, job, node) (istio_response_messages_total)
      record: workload:istio_response_messages_total
    - expr: sum without (pod, pod_template_hash, instance, job, node) (istio_tcp_sent_bytes_total)
      record: workload:istio_tcp_sent_bytes_total
    - expr: sum without (pod, pod_template_hash, instance, job, node) (istio_tcp_received_bytes_total)
      record: workload:istio_tcp_received_bytes_total
    - expr: sum without (pod, pod_template_hash, instance, job, node) (istio_tcp_connections_opened_total)
      record: workload:istio_tcp_connections_opened_total
    - expr: sum without (pod, pod_template_hash, instance, job, node) (istio_tcp_connections_closed_total)
      record: workload:istio_tcp_connections_closed_total
    - expr: sum without (pod, pod_template_hash, instance, job, node) (istio_request_duration_milliseconds_bucket)
      record: workload:istio_request_duration_milliseconds_bucket
    - expr: sum without (pod, pod_template_hash, instance, job, node) (istio_request_duration_milliseconds_sum)
      record: workload:istio_request_duration_milliseconds_sum
    - expr: sum without (pod, pod_template_hash, instance, job, node) (istio_request_duration_milliseconds_count)
      record: workload:istio_request_duration_milliseconds_count
    - expr: sum without (pod, pod_template_hash, instance, job, node) (istio_request_bytes_bucket)
      record: workload:istio_request_bytes_bucket
    - expr: sum without (pod, pod_template_hash, instance, job, node) (istio_request_bytes_sum)
      record: workload:istio_request_bytes_sum
    - expr: sum without (pod, pod_template_hash, instance, job, node) (istio_request_bytes_count)
      record: workload:istio_request_bytes_count
    - expr: sum without (pod, pod_template_hash, instance, job, node) (istio_response_bytes_bucket)
      record: workload:istio_response_bytes_bucket
    - expr: sum without (pod, pod_template_hash, instance, job, node) (istio_response_bytes_sum)
      record: workload:istio_response_bytes_sum
    - expr: sum without (pod, pod_template_hash, instance, job, node) (istio_response_bytes_count)
      record: workload:istio_response_bytes_count
    # KIALI TIER
    # Counters
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_ai_requests_total)
      record: kiali:kiali_ai_requests_total
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_ai_store_evictions_total)
      record: kiali:kiali_ai_store_evictions_total
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_api_failures_total)
      record: kiali:kiali_api_failures_total
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_cache_hits_total)
      record: kiali:kiali_cache_hits_total
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_cache_requests_total)
      record: kiali:kiali_cache_requests_total
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_graph_cache_evictions_total)
      record: kiali:kiali_graph_cache_evictions_total
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_graph_cache_hits_total)
      record: kiali:kiali_graph_cache_hits_total
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_graph_cache_misses_total)
      record: kiali:kiali_graph_cache_misses_total
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_health_cache_hits_total)
      record: kiali:kiali_health_cache_hits_total
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_health_cache_misses_total)
      record: kiali:kiali_health_cache_misses_total
    # Gauges
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_ai_store_conversations_total)
      record: kiali:kiali_ai_store_conversations_total
    - expr: max without (pod, pod_template_hash, instance, job, node) (kiali_graph_nodes)
      record: kiali:kiali_graph_nodes
    - expr: max without (pod, pod_template_hash, instance, job, node) (kiali_kubernetes_clients)
      record: kiali:kiali_kubernetes_clients
    - expr: max without (pod, pod_template_hash, instance, job, node) (kiali_health_status)
      record: kiali:kiali_health_status
    # Histograms
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_ai_request_duration_seconds_bucket)
      record: kiali:kiali_ai_request_duration_seconds_bucket
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_ai_request_duration_seconds_sum)
      record: kiali:kiali_ai_request_duration_seconds_sum
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_ai_request_duration_seconds_count)
      record: kiali:kiali_ai_request_duration_seconds_count
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_api_processing_duration_seconds_bucket)
      record: kiali:kiali_api_processing_duration_seconds_bucket
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_api_processing_duration_seconds_sum)
      record: kiali:kiali_api_processing_duration_seconds_sum
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_api_processing_duration_seconds_count)
      record: kiali:kiali_api_processing_duration_seconds_count
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_checker_processing_duration_seconds_bucket)
      record: kiali:kiali_checker_processing_duration_seconds_bucket
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_checker_processing_duration_seconds_sum)
      record: kiali:kiali_checker_processing_duration_seconds_sum
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_checker_processing_duration_seconds_count)
      record: kiali:kiali_checker_processing_duration_seconds_count
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_graph_appender_duration_seconds_bucket)
      record: kiali:kiali_graph_appender_duration_seconds_bucket
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_graph_appender_duration_seconds_sum)
      record: kiali:kiali_graph_appender_duration_seconds_sum
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_graph_appender_duration_seconds_count)
      record: kiali:kiali_graph_appender_duration_seconds_count
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_graph_generation_duration_seconds_bucket)
      record: kiali:kiali_graph_generation_duration_seconds_bucket
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_graph_generation_duration_seconds_sum)
      record: kiali:kiali_graph_generation_duration_seconds_sum
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_graph_generation_duration_seconds_count)
      record: kiali:kiali_graph_generation_duration_seconds_count
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_graph_marshal_duration_seconds_bucket)
      record: kiali:kiali_graph_marshal_duration_seconds_bucket
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_graph_marshal_duration_seconds_sum)
      record: kiali:kiali_graph_marshal_duration_seconds_sum
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_graph_marshal_duration_seconds_count)
      record: kiali:kiali_graph_marshal_duration_seconds_count
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_health_refresh_duration_seconds_bucket)
      record: kiali:kiali_health_refresh_duration_seconds_bucket
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_health_refresh_duration_seconds_sum)
      record: kiali:kiali_health_refresh_duration_seconds_sum
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_health_refresh_duration_seconds_count)
      record: kiali:kiali_health_refresh_duration_seconds_count
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_prometheus_processing_duration_seconds_bucket)
      record: kiali:kiali_prometheus_processing_duration_seconds_bucket
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_prometheus_processing_duration_seconds_sum)
      record: kiali:kiali_prometheus_processing_duration_seconds_sum
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_prometheus_processing_duration_seconds_count)
      record: kiali:kiali_prometheus_processing_duration_seconds_count
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_single_validation_processing_duration_seconds_bucket)
      record: kiali:kiali_single_validation_processing_duration_seconds_bucket
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_single_validation_processing_duration_seconds_sum)
      record: kiali:kiali_single_validation_processing_duration_seconds_sum
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_single_validation_processing_duration_seconds_count)
      record: kiali:kiali_single_validation_processing_duration_seconds_count
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_tracing_processing_duration_seconds_bucket)
      record: kiali:kiali_tracing_processing_duration_seconds_bucket
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_tracing_processing_duration_seconds_sum)
      record: kiali:kiali_tracing_processing_duration_seconds_sum
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_tracing_processing_duration_seconds_count)
      record: kiali:kiali_tracing_processing_duration_seconds_count
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_validation_processing_duration_seconds_bucket)
      record: kiali:kiali_validation_processing_duration_seconds_bucket
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_validation_processing_duration_seconds_sum)
      record: kiali:kiali_validation_processing_duration_seconds_sum
    - expr: sum without (pod, pod_template_hash, instance, job, node) (kiali_validation_processing_duration_seconds_count)
      record: kiali:kiali_validation_processing_duration_seconds_count
EOF

add_mcoa_ref() {
  local group=$1 resource=$2 name=$3
  local patch exists
  exists=$(oc --context=ossm-kiali-hub get clustermanagementaddon \
    multicluster-observability-addon -o json | jq -r \
    --arg g "${group}" --arg r "${resource}" --arg n "${name}" \
    --arg ns "open-cluster-management-observability" \
    --arg pn "${MCOA_PLACEMENT_NAME}" --arg pns "${MCOA_PLACEMENT_NS}" \
    '[.spec.installStrategy.placements[] |
      select(.name == $pn and .namespace == $pns) |
      .configs[]? |
      select(.group == $g and .resource == $r and .name == $n and .namespace == $ns)] | length')
  if [ "${exists}" -eq 0 ]; then
    local idx
    idx=$(oc --context=ossm-kiali-hub get clustermanagementaddon \
      multicluster-observability-addon -o json | jq -r \
      --arg pn "${MCOA_PLACEMENT_NAME}" --arg pns "${MCOA_PLACEMENT_NS}" \
      '.spec.installStrategy.placements | to_entries[] |
        select(.value.name == $pn and .value.namespace == $pns) | .key')
    local configs
    configs=$(oc --context=ossm-kiali-hub get clustermanagementaddon \
      multicluster-observability-addon -o json | jq -r \
      --argjson i "${idx}" \
      '.spec.installStrategy.placements[$i].configs | type == "array"')
    local op path
    if [ "${configs}" = true ]; then
      op="add"; path="/spec/installStrategy/placements/${idx}/configs/-"
    else
      op="add"; path="/spec/installStrategy/placements/${idx}/configs"
    fi
    oc --context=ossm-kiali-hub patch clustermanagementaddon \
      multicluster-observability-addon --type=json -p="[{
        \"op\": \"${op}\",
        \"path\": \"${path}\",
        \"value\": {\"group\": \"${group}\", \"resource\": \"${resource}\",
                    \"name\": \"${name}\",
                    \"namespace\": \"open-cluster-management-observability\"}
      }]"
    echo "Added ${resource}/${name} to placement ${MCOA_PLACEMENT_NS}/${MCOA_PLACEMENT_NAME}"
  else
    echo "Reference ${resource}/${name} already present — skipping"
  fi
}

# User-workload ScrapeConfig (shared)
add_mcoa_ref monitoring.rhobs scrapeconfigs kiali-istio-federation

# Cluster-wide platform ScrapeConfig
add_mcoa_ref monitoring.rhobs scrapeconfigs kiali-istio-platform-federation

# Shared aggregation PrometheusRule
add_mcoa_ref monitoring.coreos.com prometheusrules kiali-istio-aggregation

oc --context=ossm-kiali-hub get clustermanagementaddon multicluster-observability-addon \
  -o jsonpath-as-json='{.spec.installStrategy.placements[*].configs}'

oc --context=ossm-kiali-hub create namespace "${SPOKE_CLUSTER_NAME}" 2>/dev/null || true

oc --context=ossm-kiali-hub apply -f - <<EOF
apiVersion: cluster.open-cluster-management.io/v1
kind: ManagedCluster
metadata:
  name: ${SPOKE_CLUSTER_NAME}
  labels:
    cloud: auto-detect
    vendor: auto-detect
spec:
  hubAcceptsClient: true
EOF

oc config view --context=ossm-kiali-spoke --minify --flatten \
  > /tmp/spoke-kubeconfig.yaml

# Verify it connects to the spoke
oc --kubeconfig=/tmp/spoke-kubeconfig.yaml whoami --show-server

oc --context=ossm-kiali-hub create secret generic auto-import-secret \
  -n "${SPOKE_CLUSTER_NAME}" \
  --from-file=kubeconfig=/tmp/spoke-kubeconfig.yaml

oc --context=ossm-kiali-hub apply -f - <<EOF
apiVersion: agent.open-cluster-management.io/v1
kind: KlusterletAddonConfig
metadata:
  name: ${SPOKE_CLUSTER_NAME}
  namespace: ${SPOKE_CLUSTER_NAME}
spec:
  applicationManager:
    enabled: true
  certPolicyController:
    enabled: true
  policyController:
    enabled: true
  searchCollector:
    enabled: true
EOF

echo "Waiting for ${SPOKE_CLUSTER_NAME} to join and become available..."
while true; do
  STATUS=$(oc --context=ossm-kiali-hub get managedcluster "${SPOKE_CLUSTER_NAME}" \
    -o jsonpath='{range .status.conditions[*]}{.type}={.status}{" "}{end}' 2>/dev/null)
  echo "  Status: ${STATUS}"
  if echo "${STATUS}" | grep -q "ManagedClusterJoined=True" && \
     echo "${STATUS}" | grep -q "ManagedClusterConditionAvailable=True"; then
    echo "${SPOKE_CLUSTER_NAME} is joined and available"
    break
  fi
  sleep 15
done

rm -f /tmp/spoke-kubeconfig.yaml

oc --context=ossm-kiali-hub get managedclusters
# Should show local-cluster and ${SPOKE_CLUSTER_NAME} both with JOINED=True, AVAILABLE=True

oc --context=ossm-kiali-spoke get configmap cluster-monitoring-config \
  -n openshift-monitoring \
  -o jsonpath='{.data.config\.yaml}' 2>/dev/null | \
  grep -q "enableUserWorkload: true" && \
  echo "Already enabled" || echo "Not enabled"

if oc --context=ossm-kiali-spoke get configmap cluster-monitoring-config \
    -n openshift-monitoring &>/dev/null 2>&1; then
  # ConfigMap already exists — patch only the enableUserWorkload key
  EXISTING=$(oc --context=ossm-kiali-spoke get configmap cluster-monitoring-config \
    -n openshift-monitoring -o jsonpath='{.data.config\.yaml}' 2>/dev/null || true)
  if echo "${EXISTING}" | grep -q "enableUserWorkload:"; then
    oc --context=ossm-kiali-spoke get configmap cluster-monitoring-config \
      -n openshift-monitoring -o json \
      | jq '.data["config.yaml"] |= sub("enableUserWorkload:\\s*\\w+"; "enableUserWorkload: true")' \
      | oc --context=ossm-kiali-spoke apply -f -
  else
    oc --context=ossm-kiali-spoke get configmap cluster-monitoring-config \
      -n openshift-monitoring -o json \
      | jq '.data["config.yaml"] = ((.data["config.yaml"] // "") + "\nenableUserWorkload: true\n")' \
      | oc --context=ossm-kiali-spoke apply -f -
  fi
else
  # ConfigMap does not exist — create it and label it as tutorial-owned
  oc --context=ossm-kiali-spoke create configmap cluster-monitoring-config \
    -n openshift-monitoring \
    --from-literal=config.yaml="enableUserWorkload: true"
  oc --context=ossm-kiali-spoke label configmap cluster-monitoring-config \
    -n openshift-monitoring kiali.io/tutorial-owned=true
fi

until oc --context=ossm-kiali-spoke get pods \
  -l app.kubernetes.io/name=prometheus \
  -n openshift-user-workload-monitoring \
  --no-headers 2>/dev/null | grep -q .; do
  sleep 5
done

oc --context=ossm-kiali-spoke wait pod \
  --for=condition=Ready \
  -l app.kubernetes.io/name=prometheus \
  -n openshift-user-workload-monitoring \
  --timeout=300s

oc --context=ossm-kiali-spoke apply -f - <<'EOF'
apiVersion: operators.coreos.com/v1alpha1
kind: Subscription
metadata:
  name: openshift-service-mesh-operator
  namespace: openshift-operators
spec:
  channel: stable
  installPlanApproval: Automatic
  name: servicemeshoperator3
  source: redhat-operators
  sourceNamespace: openshift-marketplace
EOF

until oc --context=ossm-kiali-spoke get pods \
  -l app.kubernetes.io/created-by=servicemeshoperator3 \
  -n openshift-operators \
  --no-headers 2>/dev/null | grep -q .; do
  sleep 5
done

oc --context=ossm-kiali-spoke wait pod \
  --for=condition=Ready \
  -l app.kubernetes.io/created-by=servicemeshoperator3 \
  -n openshift-operators \
  --timeout=300s

oc --context=ossm-kiali-spoke create namespace istio-system 2>/dev/null || true
oc --context=ossm-kiali-spoke create namespace istio-cni 2>/dev/null || true
oc --context=ossm-kiali-spoke create namespace ztunnel 2>/dev/null || true

oc --context=ossm-kiali-spoke label namespace ztunnel istio-discovery=enabled

mkdir -p /tmp/istio-certs && cd /tmp/istio-certs

# Root CA key and certificate
openssl genrsa -out root-key.pem 4096

cat > root-ca.conf <<'CONF'
encrypt_key = no
prompt = no
utf8 = yes
default_md = sha256
default_bits = 4096
req_extensions = req_ext
x509_extensions = req_ext
distinguished_name = req_dn
[ req_ext ]
subjectKeyIdentifier = hash
basicConstraints = critical, CA:true
keyUsage = critical, digitalSignature, nonRepudiation, keyEncipherment, keyCertSign
[ req_dn ]
O = Istio
CN = Root CA
CONF

openssl req -sha256 -new \
  -key root-key.pem \
  -config root-ca.conf \
  -out root-cert.csr

openssl x509 -req -sha256 -days 3650 \
  -signkey root-key.pem \
  -extensions req_ext -extfile root-ca.conf \
  -in root-cert.csr \
  -out root-cert.pem

# Intermediate CA for the spoke
cat > intermediate.conf <<'CONF'
[ req ]
encrypt_key = no
prompt = no
utf8 = yes
default_md = sha256
default_bits = 4096
req_extensions = req_ext
x509_extensions = req_ext
distinguished_name = req_dn
[ req_ext ]
subjectKeyIdentifier = hash
basicConstraints = critical, CA:true, pathlen:0
keyUsage = critical, digitalSignature, nonRepudiation, keyEncipherment, keyCertSign
subjectAltName=@san
[ san ]
DNS.1 = istiod.istio-system.svc
[ req_dn ]
O = Istio
CN = Intermediate CA
L = spoke
CONF

openssl genrsa -out ca-key.pem 4096

openssl req -new \
  -config intermediate.conf \
  -key ca-key.pem \
  -out cluster-ca.csr

openssl x509 -req -sha256 -days 3650 \
  -CA root-cert.pem \
  -CAkey root-key.pem -CAcreateserial \
  -extensions req_ext -extfile intermediate.conf \
  -in cluster-ca.csr \
  -out ca-cert.pem

cat ca-cert.pem root-cert.pem > cert-chain.pem

cd -

oc --context=ossm-kiali-spoke get secret cacerts -n istio-system &>/dev/null || \
oc --context=ossm-kiali-spoke create secret generic cacerts -n istio-system \
  --from-file=ca-cert.pem=/tmp/istio-certs/ca-cert.pem \
  --from-file=ca-key.pem=/tmp/istio-certs/ca-key.pem \
  --from-file=root-cert.pem=/tmp/istio-certs/root-cert.pem \
  --from-file=cert-chain.pem=/tmp/istio-certs/cert-chain.pem

oc --context=ossm-kiali-spoke apply -f - <<EOF
apiVersion: sailoperator.io/v1
kind: IstioCNI
metadata:
  name: default
spec:
  namespace: istio-cni
  profile: openshift-ambient
  version: v${ISTIO_VERSION}
EOF

oc --context=ossm-kiali-spoke wait istiocni default \
  --for=condition=Ready \
  --timeout=300s

oc --context=ossm-kiali-spoke apply -f - <<EOF
apiVersion: sailoperator.io/v1
kind: Istio
metadata:
  name: default
spec:
  namespace: istio-system
  profile: openshift-ambient
  updateStrategy:
    type: InPlace
  values:
    global:
      meshID: ${MESH_ID}
    meshConfig:
      discoverySelectors:
      - matchLabels:
          istio-discovery: enabled
    pilot:
      trustedZtunnelNamespace: ztunnel
  version: v${ISTIO_VERSION}
EOF

oc --context=ossm-kiali-spoke wait istio default \
  --for=condition=Ready \
  --timeout=300s

oc --context=ossm-kiali-spoke apply -f - <<EOF
apiVersion: sailoperator.io/v1
kind: ZTunnel
metadata:
  name: default
spec:
  namespace: ztunnel
  version: v${ISTIO_VERSION}
EOF

oc --context=ossm-kiali-spoke wait ztunnel default \
  --for=condition=Ready \
  --timeout=300s

# Confirm ztunnel pods are running on all nodes
oc --context=ossm-kiali-spoke get pods -n ztunnel -l app=ztunnel

for NS in mesh-observability istio-system ztunnel ambient-demo bookinfo; do
  oc --context=ossm-kiali-spoke create namespace "${NS}" --dry-run=client -o yaml | \
    oc --context=ossm-kiali-spoke apply -f -
done

oc --context=ossm-kiali-spoke apply -f - <<'EOF'
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: istiod-monitor
  namespace: istio-system
spec:
  targetLabels:
  - app
  selector:
    matchLabels:
      istio: pilot
  endpoints:
  - port: http-monitoring
    interval: 30s
EOF

oc --context=ossm-kiali-spoke apply -f - <<EOF
apiVersion: monitoring.coreos.com/v1
kind: PodMonitor
metadata:
  name: ztunnel-monitor
  namespace: ztunnel
spec:
  selector:
    matchExpressions:
    - key: istio-prometheus-ignore
      operator: DoesNotExist
  podMetricsEndpoints:
  - path: /stats/prometheus
    interval: 30s
    relabelings:
    - action: keep
      sourceLabels: ["__meta_kubernetes_pod_container_name"]
      regex: "istio-proxy"
    - action: keep
      sourceLabels: ["__meta_kubernetes_pod_annotationpresent_prometheus_io_scrape"]
    - action: replace
      regex: (\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})
      replacement: '[\$2]:\$1'
      sourceLabels: ["__meta_kubernetes_pod_annotation_prometheus_io_port","__meta_kubernetes_pod_ip"]
      targetLabel: "__address__"
    - action: replace
      regex: (\d+);((([0-9]+?)(\.|$)){4})
      replacement: '\$2:\$1'
      sourceLabels: ["__meta_kubernetes_pod_annotation_prometheus_io_port","__meta_kubernetes_pod_ip"]
      targetLabel: "__address__"
    - sourceLabels: ["__meta_kubernetes_pod_label_app_kubernetes_io_name","__meta_kubernetes_pod_label_app"]
      separator: ";"
      targetLabel: "app"
      action: replace
      regex: "(.+);.*|.*;(.+)"
      replacement: "\${1}\${2}"
    - sourceLabels: ["__meta_kubernetes_pod_label_app_kubernetes_io_version","__meta_kubernetes_pod_label_version"]
      separator: ";"
      targetLabel: "version"
      action: replace
      regex: "(.+);.*|.*;(.+)"
      replacement: "\${1}\${2}"
    - action: replace
      regex: "(.+)"
      replacement: "\${1}"
      sourceLabels: ["__meta_kubernetes_pod_label_app_kubernetes_io_name"]
      targetLabel: "app_kubernetes_io_name"
    - action: replace
      regex: "(.+)"
      replacement: "\${1}"
      sourceLabels: ["__meta_kubernetes_pod_label_app_kubernetes_io_version"]
      targetLabel: "app_kubernetes_io_version"
    - sourceLabels: ["__meta_kubernetes_namespace"]
      action: replace
      targetLabel: namespace
    - action: replace
      replacement: "${MESH_ID}"
      targetLabel: mesh_id
EOF

export OBSERVATORIUM_URL=$(oc --context=ossm-kiali-hub get route observatorium-api \
  -n open-cluster-management-observability \
  -o jsonpath='https://{.spec.host}/api/metrics/v1/default')
echo "Observatorium URL: ${OBSERVATORIUM_URL}"

oc --context=ossm-kiali-hub get secret observability-grafana-certs \
  -n open-cluster-management-observability \
  -o jsonpath='{.data.tls\.crt}' | base64 -d > /tmp/obs-tls.crt

oc --context=ossm-kiali-hub get secret observability-grafana-certs \
  -n open-cluster-management-observability \
  -o jsonpath='{.data.tls\.key}' | base64 -d > /tmp/obs-tls.key

HOST=$(oc --context=ossm-kiali-hub get route observatorium-api \
  -n open-cluster-management-observability \
  -o jsonpath='{.spec.host}')

echo | openssl s_client \
  -connect "${HOST}:443" \
  -servername "${HOST}" \
  -showcerts 2>/dev/null | openssl x509 -noout -issuer

oc --context=ossm-kiali-hub get secret observability-server-ca-certs \
  -n open-cluster-management-observability \
  -o jsonpath='{.data.ca\.crt}' | base64 -d > /tmp/obs-server-ca.crt

oc --context=ossm-kiali-hub get secret observability-client-ca-certs \
  -n open-cluster-management-observability \
  -o jsonpath='{.data.ca\.crt}' | base64 -d > /tmp/obs-server-ca.crt

oc --context=ossm-kiali-spoke create secret generic acm-observability-certs \
  -n istio-system \
  --from-file=tls.crt=/tmp/obs-tls.crt \
  --from-file=tls.key=/tmp/obs-tls.key

oc --context=ossm-kiali-spoke create configmap kiali-cabundle \
  -n istio-system \
  --from-file=additional-ca-bundle.pem=/tmp/obs-server-ca.crt

oc --context=ossm-kiali-spoke apply -f - <<'EOF'
apiVersion: operators.coreos.com/v1alpha1
kind: Subscription
metadata:
  name: kiali-ossm
  namespace: openshift-operators
spec:
  channel: stable
  installPlanApproval: Automatic
  name: kiali-ossm
  source: redhat-operators
  sourceNamespace: openshift-marketplace
EOF

until oc --context=ossm-kiali-spoke get pods \
  -l app.kubernetes.io/name=kiali-operator \
  -n openshift-operators \
  --no-headers 2>/dev/null | grep -q .; do
  sleep 5
done

oc --context=ossm-kiali-spoke wait pod \
  --for=condition=Ready \
  -l app.kubernetes.io/name=kiali-operator \
  -n openshift-operators \
  --timeout=300s

oc --context=ossm-kiali-spoke apply -f - <<EOF
apiVersion: kiali.io/v1alpha1
kind: Kiali
metadata:
  name: kiali
  namespace: istio-system
spec:
  auth:
    strategy: openshift
    openshift:
      impersonation:
        enabled: false
  deployment:
    cluster_wide_access: true
    instance_name: kiali
    logger:
      log_level: debug
    namespace: istio-system
    replicas: 1
  external_services:
    grafana:
      enabled: false
    prometheus:
      auth:
        cert_file: secret:acm-observability-certs:tls.crt
        key_file: secret:acm-observability-certs:tls.key
        type: none
        use_kiali_token: false
      thanos_proxy:
        enabled: true
        retention_period: "365d"
        scrape_interval: "5m"
      url: "${OBSERVATORIUM_URL}"
  version: default
EOF

oc --context=ossm-kiali-spoke wait kiali kiali \
  -n istio-system \
  --for=condition=Successful \
  --timeout=300s

oc --context=ossm-kiali-spoke rollout status deployment/kiali -n istio-system

oc --context=ossm-kiali-spoke apply -f - <<'EOF'
apiVersion: kiali.io/v1alpha1
kind: OSSMConsole
metadata:
  name: ossmconsole
  namespace: istio-system
spec: {}
EOF

until oc --context=ossm-kiali-spoke get ossmconsole ossmconsole \
  -n istio-system \
  -o jsonpath='{.status.conditions[?(@.type=="Successful")].status}' 2>/dev/null | grep -q "True"; do
  echo "Waiting for OSSMConsole reconciliation..."
  sleep 10
done
echo "OSSMConsole ready"

oc --context=ossm-kiali-spoke apply -f - <<'EOF'
apiVersion: v1
kind: Namespace
metadata:
  name: ambient-demo
  labels:
    istio.io/dataplane-mode: ambient
    istio-discovery: enabled
EOF

ISTIO_MINOR=$(echo "${ISTIO_VERSION}" | cut -d. -f1-2)

# Download the helloworld manifest once to avoid GitHub rate limits on repeated requests
curl -sL "https://raw.githubusercontent.com/openshift-service-mesh/istio/release-${ISTIO_MINOR}/samples/helloworld/helloworld.yaml" \
  -o /tmp/helloworld.yaml

oc --context=ossm-kiali-spoke apply -n ambient-demo -l service=helloworld -f /tmp/helloworld.yaml
oc --context=ossm-kiali-spoke apply -n ambient-demo -l version=v1 -f /tmp/helloworld.yaml
oc --context=ossm-kiali-spoke apply -n ambient-demo -l version=v2 -f /tmp/helloworld.yaml
rm -f /tmp/helloworld.yaml

oc --context=ossm-kiali-spoke wait deployment/helloworld-v1 \
  -n ambient-demo --for=condition=Available --timeout=120s
oc --context=ossm-kiali-spoke wait deployment/helloworld-v2 \
  -n ambient-demo --for=condition=Available --timeout=120s

oc --context=ossm-kiali-spoke apply -f - <<'EOF'
apiVersion: apps/v1
kind: Deployment
metadata:
  name: traffic-gen
  namespace: ambient-demo
spec:
  replicas: 1
  selector:
    matchLabels:
      app: traffic-gen
  template:
    metadata:
      labels:
        app: traffic-gen
    spec:
      containers:
      - name: client
        image: registry.access.redhat.com/ubi9/ubi-minimal:latest
        command: ["/bin/sh", "-c"]
        args:
        - |
          while true; do
            curl -s --max-time 5 http://helloworld:5000/hello || echo "failed"
            sleep 2
          done
EOF

oc --context=ossm-kiali-spoke get pods -n ambient-demo
# Each pod should show 1/1 READY (no istio-proxy sidecar)

oc --context=ossm-kiali-spoke apply -f - <<'EOF'
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
  name: waypoint
  namespace: ambient-demo
  labels:
    istio.io/waypoint-for: service
spec:
  gatewayClassName: istio-waypoint
  listeners:
  - name: mesh
    port: 15008
    protocol: HBONE
EOF

oc --context=ossm-kiali-spoke wait gateway/waypoint \
  -n ambient-demo \
  --for=condition=Programmed=True \
  --timeout=120s

oc --context=ossm-kiali-spoke label namespace ambient-demo \
  istio.io/use-waypoint=waypoint

# Confirm the Gateway has istio.io/waypoint-for: service
oc --context=ossm-kiali-spoke get gateway waypoint -n ambient-demo \
  -o jsonpath='waypoint-for={.metadata.labels.istio\.io/waypoint-for}{"\n"}'
# Expected: waypoint-for=service

# Confirm the namespace is enrolled
oc --context=ossm-kiali-spoke get namespace ambient-demo \
  -o jsonpath='use-waypoint={.metadata.labels.istio\.io/use-waypoint}{"\n"}'
# Expected: use-waypoint=waypoint

# Confirm the waypoint pod is running
oc --context=ossm-kiali-spoke get pods -n ambient-demo \
  -l gateway.networking.k8s.io/gateway-name=waypoint

oc --context=ossm-kiali-spoke apply -f - <<EOF
apiVersion: monitoring.coreos.com/v1
kind: PodMonitor
metadata:
  name: istio-proxies-monitor
  namespace: ambient-demo
spec:
  selector:
    matchExpressions:
    - key: istio-prometheus-ignore
      operator: DoesNotExist
  podMetricsEndpoints:
  - path: /stats/prometheus
    interval: 30s
    relabelings:
    - action: keep
      sourceLabels: ["__meta_kubernetes_pod_container_name"]
      regex: "istio-proxy"
    - action: keep
      sourceLabels: ["__meta_kubernetes_pod_annotationpresent_prometheus_io_scrape"]
    - action: replace
      regex: (\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})
      replacement: '[\$2]:\$1'
      sourceLabels: ["__meta_kubernetes_pod_annotation_prometheus_io_port","__meta_kubernetes_pod_ip"]
      targetLabel: "__address__"
    - action: replace
      regex: (\d+);((([0-9]+?)(\.|$)){4})
      replacement: '\$2:\$1'
      sourceLabels: ["__meta_kubernetes_pod_annotation_prometheus_io_port","__meta_kubernetes_pod_ip"]
      targetLabel: "__address__"
    - sourceLabels: ["__meta_kubernetes_pod_label_app_kubernetes_io_name","__meta_kubernetes_pod_label_app"]
      separator: ";"
      targetLabel: "app"
      action: replace
      regex: "(.+);.*|.*;(.+)"
      replacement: "\${1}\${2}"
    - sourceLabels: ["__meta_kubernetes_pod_label_app_kubernetes_io_version","__meta_kubernetes_pod_label_version"]
      separator: ";"
      targetLabel: "version"
      action: replace
      regex: "(.+);.*|.*;(.+)"
      replacement: "\${1}\${2}"
    - action: replace
      regex: "(.+)"
      replacement: "\${1}"
      sourceLabels: ["__meta_kubernetes_pod_label_app_kubernetes_io_name"]
      targetLabel: "app_kubernetes_io_name"
    - action: replace
      regex: "(.+)"
      replacement: "\${1}"
      sourceLabels: ["__meta_kubernetes_pod_label_app_kubernetes_io_version"]
      targetLabel: "app_kubernetes_io_version"
    - sourceLabels: ["__meta_kubernetes_namespace"]
      action: replace
      targetLabel: namespace
    - action: replace
      replacement: "${MESH_ID}"
      targetLabel: mesh_id
EOF

oc --context=ossm-kiali-hub get --raw \
  "/api/v1/namespaces/open-cluster-management-observability/services/http:observability-thanos-query-frontend:9090/proxy/api/v1/query?query=istio_requests_total%7Breporter%3D%22waypoint%22%7D" \
  | jq '.data.result | length'
# Returns the count of waypoint reporter timeseries — must be > 0

oc --context=ossm-kiali-spoke create namespace bookinfo 2>/dev/null || true

oc --context=ossm-kiali-spoke label namespace bookinfo \
  istio-injection=enabled \
  istio-discovery=enabled

ISTIO_MINOR=$(echo "${ISTIO_VERSION}" | cut -d. -f1-2)

oc --context=ossm-kiali-spoke apply -n bookinfo \
  -f "https://raw.githubusercontent.com/openshift-service-mesh/istio/release-${ISTIO_MINOR}/samples/bookinfo/platform/kube/bookinfo.yaml"

oc --context=ossm-kiali-spoke wait pods \
  --for=condition=Ready \
  --all \
  -n bookinfo \
  --timeout=180s

oc --context=ossm-kiali-spoke get pods -n bookinfo

oc --context=ossm-kiali-spoke exec \
  "$(oc --context=ossm-kiali-spoke get pod -l app=ratings -n bookinfo \
     -o jsonpath='{.items[0].metadata.name}')" \
  -c ratings -n bookinfo -- \
  curl -sS productpage:9080/productpage | grep -o "<title>.*</title>"
# Expected: <title>Simple Bookstore App</title>

oc --context=ossm-kiali-spoke apply -n bookinfo \
  -f "https://raw.githubusercontent.com/openshift-service-mesh/istio/release-${ISTIO_MINOR}/samples/bookinfo/gateway-api/bookinfo-gateway.yaml"

oc --context=ossm-kiali-spoke wait \
  --for=condition=Programmed \
  gateway/bookinfo-gateway \
  -n bookinfo \
  --timeout=120s

export BOOKINFO_URL="http://$(oc --context=ossm-kiali-spoke get gateway bookinfo-gateway \
  -n bookinfo \
  -o jsonpath='{.status.addresses[0].value}'):$(oc --context=ossm-kiali-spoke get gateway bookinfo-gateway \
  -n bookinfo \
  -o jsonpath='{.spec.listeners[?(@.name=="http")].port}')/productpage"
echo "Bookinfo URL: ${BOOKINFO_URL}"

oc --context=ossm-kiali-spoke apply -f - <<'EOF'
apiVersion: apps/v1
kind: Deployment
metadata:
  name: traffic-gen
  namespace: bookinfo
spec:
  replicas: 1
  selector:
    matchLabels:
      app: traffic-gen
  template:
    metadata:
      labels:
        app: traffic-gen
    spec:
      containers:
      - name: client
        image: registry.access.redhat.com/ubi9/ubi-minimal:latest
        command: ["/bin/sh", "-c"]
        args:
        - |
          while true; do
            curl -s -o /dev/null -w "%{http_code}\n" --max-time 5 http://productpage:9080/productpage || echo "failed"
            sleep 2
          done
EOF

oc --context=ossm-kiali-spoke apply -f - <<EOF
apiVersion: monitoring.coreos.com/v1
kind: PodMonitor
metadata:
  name: istio-proxies-monitor
  namespace: bookinfo
spec:
  selector:
    matchExpressions:
    - key: istio-prometheus-ignore
      operator: DoesNotExist
  podMetricsEndpoints:
  - path: /stats/prometheus
    interval: 30s
    relabelings:
    - action: keep
      sourceLabels: ["__meta_kubernetes_pod_container_name"]
      regex: "istio-proxy"
    - action: keep
      sourceLabels: ["__meta_kubernetes_pod_annotationpresent_prometheus_io_scrape"]
    - action: replace
      regex: (\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})
      replacement: '[\$2]:\$1'
      sourceLabels: ["__meta_kubernetes_pod_annotation_prometheus_io_port","__meta_kubernetes_pod_ip"]
      targetLabel: "__address__"
    - action: replace
      regex: (\d+);((([0-9]+?)(\.|$)){4})
      replacement: '\$2:\$1'
      sourceLabels: ["__meta_kubernetes_pod_annotation_prometheus_io_port","__meta_kubernetes_pod_ip"]
      targetLabel: "__address__"
    - sourceLabels: ["__meta_kubernetes_pod_label_app_kubernetes_io_name","__meta_kubernetes_pod_label_app"]
      separator: ";"
      targetLabel: "app"
      action: replace
      regex: "(.+);.*|.*;(.+)"
      replacement: "\${1}\${2}"
    - sourceLabels: ["__meta_kubernetes_pod_label_app_kubernetes_io_version","__meta_kubernetes_pod_label_version"]
      separator: ";"
      targetLabel: "version"
      action: replace
      regex: "(.+);.*|.*;(.+)"
      replacement: "\${1}\${2}"
    - action: replace
      regex: "(.+)"
      replacement: "\${1}"
      sourceLabels: ["__meta_kubernetes_pod_label_app_kubernetes_io_name"]
      targetLabel: "app_kubernetes_io_name"
    - action: replace
      regex: "(.+)"
      replacement: "\${1}"
      sourceLabels: ["__meta_kubernetes_pod_label_app_kubernetes_io_version"]
      targetLabel: "app_kubernetes_io_version"
    - sourceLabels: ["__meta_kubernetes_namespace"]
      action: replace
      targetLabel: namespace
    - action: replace
      replacement: "${MESH_ID}"
      targetLabel: mesh_id
EOF

oc --context=ossm-kiali-spoke get istio default
# Expect READY=1, IN USE=1, and STATUS=Healthy.

oc --context=ossm-kiali-spoke get istiocni default
# Expect READY=True and STATUS=Healthy.

oc --context=ossm-kiali-spoke get ztunnel default
# Expect READY=True and STATUS=Healthy.

oc --context=ossm-kiali-spoke get pods -n istio-system
# The istiod pod should be 1/1 Ready and Running.

oc --context=ossm-kiali-spoke get pods -n istio-cni
# Each istio-cni-node DaemonSet pod should be 1/1 Ready and Running.

oc --context=ossm-kiali-spoke get pods -n ztunnel
# Each ztunnel DaemonSet pod should be 1/1 Ready and Running.

# Ambient demo (expect "Hello version: v1" or "Hello version: v2" responses)
oc --context=ossm-kiali-spoke logs -n ambient-demo deployment/traffic-gen --tail=5

# Sidecar demo (expect HTTP 200 status codes)
oc --context=ossm-kiali-spoke logs -n bookinfo deployment/traffic-gen --tail=5

# Locate the spoke's user-workload Prometheus pod
PROM_POD=$(oc --context=ossm-kiali-spoke \
  -n openshift-user-workload-monitoring \
  get pods -l app.kubernetes.io/name=prometheus \
  -o jsonpath='{.items[0].metadata.name}')

# Edge UWM: raw counters scraped from Istio proxies
oc --context=ossm-kiali-spoke \
  -n openshift-user-workload-monitoring \
  exec -c prometheus "${PROM_POD}" -- \
  wget -qO- \
  'http://localhost:9090/api/v1/query?query=istio_requests_total' \
  | jq '.data.result'

# Edge UWM: aggregates produced by the recording rules
oc --context=ossm-kiali-spoke \
  -n openshift-user-workload-monitoring \
  exec -c prometheus "${PROM_POD}" -- \
  wget -qO- \
  'http://localhost:9090/api/v1/query?query=workload%3Aistio_requests_total' \
  | jq '.data.result'

# Hub Thanos: federated aggregates, relabeled back to istio_requests_total
oc --context=ossm-kiali-hub get --raw \
  "/api/v1/namespaces/open-cluster-management-observability/services/http:observability-thanos-query-frontend:9090/proxy/api/v1/query?query=istio_requests_total%7Bcluster%3D%22${SPOKE_CLUSTER_NAME}%22%7D" \
  | jq '.data.result'

# List all Istio metric names present in hub Thanos
oc --context=ossm-kiali-hub get --raw \
  "/api/v1/namespaces/open-cluster-management-observability/services/http:observability-thanos-query-frontend:9090/proxy/api/v1/label/__name__/values" \
  | jq -r '.data[]' | grep "^istio_"

oc --context=ossm-kiali-spoke \
  -n openshift-user-workload-monitoring \
  exec -c prometheus "${PROM_POD}" -- \
  wget -qO- \
  'http://localhost:9090/api/v1/label/__name__/values' \
  | jq -r '
  [.data[] | select(test("istio|envoy"; "i"))] | unique
  | .[] ,
  "===\nTOTAL COUNT OF istio_ AND envoy_ METRICS: \(length)"'

oc --context=ossm-kiali-hub get --raw \
  "/api/v1/namespaces/open-cluster-management-observability/services/http:observability-thanos-query-frontend:9090/proxy/api/v1/label/__name__/values" \
  | jq -r '
  [.data[] | select(test("istio|envoy"; "i"))] | unique
  | .[] ,
  "===\nTOTAL COUNT OF istio_ AND envoy_ METRICS: \(length)"'

# Confirm MCOA capabilities are enabled on the MCO
oc --context=ossm-kiali-hub get mco observability \
  -o jsonpath='{.spec.capabilities}' | jq .

# Confirm hub-side ScrapeConfigs exist
oc --context=ossm-kiali-hub get scrapeconfig \
  -n open-cluster-management-observability | grep kiali

# Confirm hub-side PrometheusRules exist
oc --context=ossm-kiali-hub get prometheusrule \
  -n open-cluster-management-observability | grep kiali

# Confirm the MCOA add-on is Available on the spoke
oc --context=ossm-kiali-hub get managedclusteraddon \
  multicluster-observability-addon -n "${SPOKE_CLUSTER_NAME}"

# Confirm the MCOA PrometheusAgent exists on the spoke
oc --context=ossm-kiali-spoke get prometheusagent \
  -n open-cluster-management-agent-addon

# Confirm the shared aggregation PrometheusRule propagated into the exempt namespace
oc --context=ossm-kiali-spoke get prometheusrule \
  kiali-istio-aggregation -n mesh-observability

# Confirm PodMonitors and ServiceMonitors are in place
oc --context=ossm-kiali-spoke get podmonitor,servicemonitor -A | \
  grep -E "ztunnel|istiod|bookinfo|ambient"

# Confirm UWM Prometheus pods are running
oc --context=ossm-kiali-spoke get pods -n openshift-user-workload-monitoring

oc --context=ossm-kiali-hub get managedcluster "${SPOKE_CLUSTER_NAME}"
oc --context=ossm-kiali-hub get managedclusteraddons -n "${SPOKE_CLUSTER_NAME}"

oc --context=ossm-kiali-spoke get route kiali -n istio-system -o jsonpath='https://{.spec.host}{"\n"}'

oc --context=ossm-kiali-spoke get route console -n openshift-console \
  -o jsonpath='https://{.spec.host}{"\n"}'

oc --context=ossm-kiali-spoke delete gateway waypoint -n ambient-demo --ignore-not-found
oc --context=ossm-kiali-spoke delete namespace ambient-demo bookinfo --ignore-not-found
oc --context=ossm-kiali-spoke delete ossmconsole ossmconsole -n istio-system --ignore-not-found
oc --context=ossm-kiali-spoke delete kiali kiali -n istio-system --ignore-not-found
oc --context=ossm-kiali-spoke delete secret acm-observability-certs cacerts -n istio-system --ignore-not-found
oc --context=ossm-kiali-spoke delete configmap kiali-cabundle -n istio-system --ignore-not-found
oc --context=ossm-kiali-spoke delete ztunnel default --ignore-not-found
oc --context=ossm-kiali-spoke delete istio default --ignore-not-found
oc --context=ossm-kiali-spoke delete istiocni default --ignore-not-found
oc --context=ossm-kiali-spoke delete namespace ztunnel istio-system istio-cni --ignore-not-found

# Read the placement so we know which index to remove from
ADDON_JSON=$(oc --context=ossm-kiali-hub get clustermanagementaddon \
  multicluster-observability-addon -o json 2>/dev/null || true)

if [ -n "${ADDON_JSON}" ]; then
  remove_mcoa_ref() {
    local group=$1 resource=$2 name=$3
    local patch
    patch=$(echo "${ADDON_JSON}" | jq -c \
      --arg g "${group}" --arg r "${resource}" --arg n "${name}" \
      --arg ns "open-cluster-management-observability" \
      '[(.spec.installStrategy.placements // []) | to_entries[] as $p |
        ($p.value.configs // []) | to_entries[] |
        select(.value.group == $g and .value.resource == $r and
               .value.name == $n and .value.namespace == $ns) |
        {op:"remove",
         path:("/spec/installStrategy/placements/"+($p.key|tostring)+"/configs/"+(.key|tostring))}]
      | sort_by(.path) | reverse')
    [ "${patch}" = "[]" ] || \
      oc --context=ossm-kiali-hub patch clustermanagementaddon \
        multicluster-observability-addon --type=json -p="${patch}"
    ADDON_JSON=$(oc --context=ossm-kiali-hub get clustermanagementaddon \
      multicluster-observability-addon -o json 2>/dev/null || echo "${ADDON_JSON}")
  }

  # Remove in reverse order: the PrometheusRule first, then the platform and shared ScrapeConfigs
  remove_mcoa_ref monitoring.coreos.com prometheusrules kiali-istio-aggregation
  remove_mcoa_ref monitoring.rhobs scrapeconfigs kiali-istio-platform-federation
  remove_mcoa_ref monitoring.rhobs scrapeconfigs kiali-istio-federation
fi

# Delete the hub-side MCOA configuration resources
oc --context=ossm-kiali-hub delete prometheusrule kiali-istio-aggregation \
  -n open-cluster-management-observability --ignore-not-found
oc --context=ossm-kiali-hub delete scrapeconfig kiali-istio-platform-federation \
  -n open-cluster-management-observability --ignore-not-found
oc --context=ossm-kiali-hub delete scrapeconfig kiali-istio-federation \
  -n open-cluster-management-observability --ignore-not-found

oc --context=ossm-kiali-hub delete mco observability --ignore-not-found
oc --context=ossm-kiali-hub wait mco observability --for=delete --timeout=120s 2>/dev/null || true
oc --context=ossm-kiali-hub delete deployment seaweedfs -n open-cluster-management-observability --ignore-not-found
oc --context=ossm-kiali-hub delete service seaweedfs -n open-cluster-management-observability --ignore-not-found
oc --context=ossm-kiali-hub delete secret thanos-object-storage -n open-cluster-management-observability --ignore-not-found
oc --context=ossm-kiali-hub delete namespace open-cluster-management-observability --ignore-not-found

# Delete the Klusterlet on the spoke and wait for it to be gone
oc --context=ossm-kiali-spoke delete klusterlet klusterlet --ignore-not-found --wait=false 2>/dev/null || true
until ! oc --context=ossm-kiali-spoke get klusterlet klusterlet &>/dev/null 2>&1; do
  echo "Waiting for Klusterlet removal..."
  sleep 10
done

# Remove ACM agent namespaces from spoke (may already be gone after Klusterlet removal)
oc --context=ossm-kiali-spoke delete namespace \
  open-cluster-management-agent \
  open-cluster-management-agent-addon \
  open-cluster-management-policies \
  --ignore-not-found --wait=false 2>/dev/null || true

# Delete the ManagedCluster on the hub
oc --context=ossm-kiali-hub delete managedcluster "${SPOKE_CLUSTER_NAME}" \
  --ignore-not-found --wait=false
until ! oc --context=ossm-kiali-hub get managedcluster "${SPOKE_CLUSTER_NAME}" &>/dev/null 2>&1; do
  echo "Waiting for ManagedCluster removal..."
  sleep 10
done
oc --context=ossm-kiali-hub delete namespace "${SPOKE_CLUSTER_NAME}" --ignore-not-found

# Remove residual AppliedManifestWork objects
if oc --context=ossm-kiali-spoke get crd appliedmanifestworks.work.open-cluster-management.io &>/dev/null; then
  AMWS=$(oc --context=ossm-kiali-spoke get appliedmanifestwork -o name 2>/dev/null || true)
  if [ -n "${AMWS}" ]; then
    echo "${AMWS}" | xargs oc --context=ossm-kiali-spoke delete --ignore-not-found
  fi
fi

# Release orphaned ConfigurationPolicy finalizers on the spoke. The stranded
# policies live in the namespace named after the spoke cluster. Scope to that
# namespace, the cluster-name label, terminating state, and the specific ACM
# finalizer to avoid touching unrelated policies.
SPOKE_NAME="ossm-kiali-spoke"   # substitute your SPOKE_CLUSTER_NAME if different
if oc --context=ossm-kiali-spoke get crd \
    configurationpolicies.policy.open-cluster-management.io &>/dev/null 2>&1; then
  oc --context=ossm-kiali-spoke get configurationpolicy \
    -n "${SPOKE_NAME}" -o json 2>/dev/null | \
    jq -r --arg cluster "${SPOKE_NAME}" '.items[] |
      select(.metadata.deletionTimestamp != null) |
      select(.metadata.labels["policy.open-cluster-management.io/cluster-name"] == $cluster) |
      select(any(.metadata.finalizers[]?;
        . == "policy.open-cluster-management.io/delete-related-objects")) |
      .metadata.name' | \
  while read -r policy; do
    echo "Releasing finalizer: ${SPOKE_NAME}/${policy}"
    oc --context=ossm-kiali-spoke patch configurationpolicy "${policy}" \
      -n "${SPOKE_NAME}" \
      --type=merge -p '{"metadata":{"finalizers":[]}}' 2>/dev/null || true
  done
fi

# Remove ACM platform recording rules left in openshift-monitoring
PROM_RULES=$(oc --context=ossm-kiali-spoke get prometheusrule \
  -n openshift-monitoring -o name 2>/dev/null | grep '/acm-rs-' || true)
if [ -n "${PROM_RULES}" ]; then
  echo "${PROM_RULES}" | xargs oc --context=ossm-kiali-spoke \
    -n openshift-monitoring delete --ignore-not-found
fi

# Remove residual ACM RBAC from spoke
RBAC=$(oc --context=ossm-kiali-spoke get clusterrole,clusterrolebinding -o name 2>/dev/null | \
  grep -E '/(ocm:|.*open-cluster-management|.*multicluster-observability|.*observability.*mco)' || true)
if [ -n "${RBAC}" ]; then echo "${RBAC}" | xargs oc --context=ossm-kiali-spoke delete --ignore-not-found; fi

# Remove residual ACM admission webhooks from spoke
WEBHOOKS=$(oc --context=ossm-kiali-spoke get \
  validatingwebhookconfiguration,mutatingwebhookconfiguration -o name 2>/dev/null | \
  grep -E '/.*(open-cluster-management|multicluster|observability)' || true)
if [ -n "${WEBHOOKS}" ]; then echo "${WEBHOOKS}" | xargs oc --context=ossm-kiali-spoke delete --ignore-not-found; fi

# Remove residual ACM and Observatorium APIServices from spoke
APIS=$(oc --context=ossm-kiali-spoke get apiservice -o name 2>/dev/null | \
  grep -E '\.(open-cluster-management\.io|multicluster\.openshift\.io|multicluster\.x-k8s\.io|observatorium\.io)$' || true)
if [ -n "${APIS}" ]; then echo "${APIS}" | xargs oc --context=ossm-kiali-spoke delete --ignore-not-found; fi

# Remove residual ACM-installed Hive APIServices from spoke (only when no live Hive workloads)
HIVE_WORKLOADS=$(oc --context=ossm-kiali-spoke get deploy,statefulset,daemonset,pod \
  -n hive -o name 2>/dev/null || true)
if ! oc --context=ossm-kiali-spoke get namespace hive &>/dev/null || \
   [ -z "${HIVE_WORKLOADS}" ]; then
  HIVE_APIS=$(oc --context=ossm-kiali-spoke get apiservice -o name 2>/dev/null | \
    grep -E '\.(hive\.openshift\.io|hiveinternal\.openshift\.io)$' || true)
  if [ -n "${HIVE_APIS}" ]; then echo "${HIVE_APIS}" | xargs oc --context=ossm-kiali-spoke delete --ignore-not-found; fi
fi

# Remove residual ACM, Observatorium, and Hive CRDs from spoke (last — the cleanup above still needs their APIs)
ACM_CRDS=$(oc --context=ossm-kiali-spoke get crd -o name 2>/dev/null | \
  grep -E '\.(open-cluster-management\.io|multicluster\.openshift\.io|multicluster\.x-k8s\.io|observatorium\.io)$' || true)
if [ -n "${ACM_CRDS}" ]; then echo "${ACM_CRDS}" | xargs oc --context=ossm-kiali-spoke delete --ignore-not-found; fi

HIVE_WORKLOADS=$(oc --context=ossm-kiali-spoke get deploy,statefulset,daemonset,pod \
  -n hive -o name 2>/dev/null || true)
if ! oc --context=ossm-kiali-spoke get namespace hive &>/dev/null || \
   [ -z "${HIVE_WORKLOADS}" ]; then
  HIVE_CRDS=$(oc --context=ossm-kiali-spoke get crd -o name 2>/dev/null | \
    grep -E '\.(hive\.openshift\.io|hiveinternal\.openshift\.io)$' || true)
  if [ -n "${HIVE_CRDS}" ]; then echo "${HIVE_CRDS}" | xargs oc --context=ossm-kiali-spoke delete --ignore-not-found; fi
fi

oc --context=ossm-kiali-hub delete multiclusterhub multiclusterhub \
  -n open-cluster-management --ignore-not-found

echo "Waiting for MultiClusterHub deletion (5–15 minutes)..."
while oc --context=ossm-kiali-hub get multiclusterhub multiclusterhub \
  -n open-cluster-management &>/dev/null 2>&1; do
  echo "  Still deleting..."
  sleep 15
done
echo "MultiClusterHub deleted"

oc --context=ossm-kiali-hub delete subscriptions.operators.coreos.com acm-operator-subscription \
  -n open-cluster-management --ignore-not-found
oc --context=ossm-kiali-hub delete csv \
  -n open-cluster-management --all --ignore-not-found
oc --context=ossm-kiali-hub delete namespace open-cluster-management --ignore-not-found --timeout=300s

# Hub RBAC
HUB_RBAC=$(oc --context=ossm-kiali-hub get clusterrole,clusterrolebinding -o name 2>/dev/null | \
  grep -E '/(ocm:|.*open-cluster-management|.*multiclusterengine|.*multicluster-observability|.*observability.*mco)' || true)
[ -n "${HUB_RBAC}" ] && echo "${HUB_RBAC}" | xargs oc --context=ossm-kiali-hub delete --ignore-not-found

# Hub admission webhooks
HUB_WEBHOOKS=$(oc --context=ossm-kiali-hub get \
  validatingwebhookconfiguration,mutatingwebhookconfiguration -o name 2>/dev/null | \
  grep -E '/.*(open-cluster-management|multicluster|observability)' || true)
[ -n "${HUB_WEBHOOKS}" ] && echo "${HUB_WEBHOOKS}" | xargs oc --context=ossm-kiali-hub delete --ignore-not-found

# Hub APIServices: ACM/MCE + Observatorium; Hive only when no live workloads
HUB_APIS=$(oc --context=ossm-kiali-hub get apiservice -o name 2>/dev/null | \
  grep -E '\.(open-cluster-management\.io|multicluster\.openshift\.io|multicluster\.x-k8s\.io|observatorium\.io)$' || true)
[ -n "${HUB_APIS}" ] && echo "${HUB_APIS}" | xargs oc --context=ossm-kiali-hub delete --ignore-not-found
HIVE_WORKLOADS=$(oc --context=ossm-kiali-hub get deploy,statefulset,daemonset,pod \
  -n hive -o name 2>/dev/null || true)
if ! oc --context=ossm-kiali-hub get namespace hive &>/dev/null || \
   [ -z "${HIVE_WORKLOADS}" ]; then
  HIVE_APIS=$(oc --context=ossm-kiali-hub get apiservice -o name 2>/dev/null | \
    grep -E '\.(hive\.openshift\.io|hiveinternal\.openshift\.io)$' || true)
  [ -n "${HIVE_APIS}" ] && echo "${HIVE_APIS}" | xargs oc --context=ossm-kiali-hub delete --ignore-not-found
fi

# Hub CRDs last — the cleanup above still needs their APIs
HUB_CRDS=$(oc --context=ossm-kiali-hub get crd -o name 2>/dev/null | \
  grep -E '\.(open-cluster-management\.io|multicluster\.openshift\.io|multicluster\.x-k8s\.io|observatorium\.io)$' || true)
[ -n "${HUB_CRDS}" ] && echo "${HUB_CRDS}" | xargs oc --context=ossm-kiali-hub delete --ignore-not-found
HIVE_WORKLOADS=$(oc --context=ossm-kiali-hub get deploy,statefulset,daemonset,pod \
  -n hive -o name 2>/dev/null || true)
if ! oc --context=ossm-kiali-hub get namespace hive &>/dev/null || \
   [ -z "${HIVE_WORKLOADS}" ]; then
  HIVE_CRDS=$(oc --context=ossm-kiali-hub get crd -o name 2>/dev/null | \
    grep -E '\.(hive\.openshift\.io|hiveinternal\.openshift\.io)$' || true)
  [ -n "${HIVE_CRDS}" ] && echo "${HIVE_CRDS}" | xargs oc --context=ossm-kiali-hub delete --ignore-not-found
  # Remove the empty hive namespace ACM installed Hive into
  oc --context=ossm-kiali-hub delete namespace hive \
    --ignore-not-found --timeout=120s 2>/dev/null || true
fi

# Remove Subscriptions
oc --context=ossm-kiali-spoke delete subscriptions.operators.coreos.com \
  kiali-ossm openshift-service-mesh-operator \
  -n openshift-operators --ignore-not-found

# Delete this tutorial's pending InstallPlans before removing CSVs — otherwise OLM may recreate CSVs from in-flight plans
for SUBSCRIPTION_LABEL in \
  operators.coreos.com/kiali-ossm.openshift-operators \
  operators.coreos.com/servicemeshoperator3.openshift-operators
do
  oc --context=ossm-kiali-spoke delete installplan -n openshift-operators \
    -l "${SUBSCRIPTION_LABEL}" --ignore-not-found
done

# Remove ALL CSVs — delete the CSV in the operator namespace only; OLM cascades deletion to all copied namespaces automatically
CSV=$(oc --context=ossm-kiali-spoke get csv -n openshift-operators \
  -l operators.coreos.com/kiali-ossm.openshift-operators \
  --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null | head -1)
if [ -n "${CSV}" ]; then oc --context=ossm-kiali-spoke delete csv "${CSV}" -n openshift-operators --ignore-not-found; fi
CSV=$(oc --context=ossm-kiali-spoke get csv -n openshift-operators \
  -l operators.coreos.com/servicemeshoperator3.openshift-operators \
  --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null | head -1)
if [ -n "${CSV}" ]; then oc --context=ossm-kiali-spoke delete csv "${CSV}" -n openshift-operators --ignore-not-found; fi

# Remove ALL CRDs — you must remove every CRD installed by the OSSM and Kiali operators or reinstallation will conflict
for suffix in sailoperator.io istio.io kiali.io; do
  CRDS=$(oc --context=ossm-kiali-spoke get crd \
    --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null \
    | grep "\.${suffix}$")
  if [ -n "${CRDS}" ]; then
    echo "${CRDS}" | xargs oc --context=ossm-kiali-spoke delete crd --ignore-not-found
  fi
done

# Remove cluster-scoped and cross-namespace resources left by Istio
oc --context=ossm-kiali-spoke delete gatewayclass \
  istio istio-remote istio-waypoint istio-east-west --ignore-not-found
for NS in $(oc --context=ossm-kiali-spoke get configmap -A \
  -o custom-columns='NS:.metadata.namespace,NAME:.metadata.name' --no-headers 2>/dev/null \
  | grep -E 'istio-ca-root-cert|istio-ca-crl' | awk '{print $1}' | sort -u); do
  oc --context=ossm-kiali-spoke delete configmap istio-ca-root-cert istio-ca-crl \
    -n "${NS}" --ignore-not-found
done
CRDS=$(oc --context=ossm-kiali-spoke get crd -o name 2>/dev/null \
  | grep -E '\.(gateway\.networking\.k8s\.io|inference\.networking\.(k8s|x-k8s)\.io)$' || true)
[ -z "${CRDS}" ] || echo "${CRDS}" \
  | xargs oc --context=ossm-kiali-spoke delete --ignore-not-found

# Remove the ClusterRole installed by the OSSM operator
oc --context=ossm-kiali-spoke delete clusterrole servicemeshoperator3-metrics-reader --ignore-not-found

# Remove the spoke ConfigMap only if the tutorial created it
OWNED=$(oc --context=ossm-kiali-spoke get configmap cluster-monitoring-config \
  -n openshift-monitoring \
  -o jsonpath='{.metadata.labels.kiali\.io/tutorial-owned}' 2>/dev/null || true)
[ "${OWNED}" = "true" ] && oc --context=ossm-kiali-spoke delete configmap \
  cluster-monitoring-config -n openshift-monitoring --ignore-not-found

# The hub ConfigMap is never created by this tutorial — do not touch it.

spec:
  deployment:
    cluster_wide_access: true   # keep ClusterRole for performance
    discovery_selectors:
      default:
      - matchLabels:
          istio-discovery: enabled

oc --context=ossm-kiali-spoke patch kiali kiali -n istio-system --type=merge -p '{
  "spec": {
    "deployment": {
      "discovery_selectors": {
        "default": [{"matchLabels": {"istio-discovery": "enabled"}}]
      }
    }
  }
}'