Prometheus and ELK (Elastic Search, Logstash, Kibana) are similar tools used to gather, display, and analyze data. This is sometimes referred to a "scraping metrics". For example, both Prometheus and ELK can be used to display data about servers, virtual machines (VMs), databases, containers (e.g. Docker, OpenShift), messaging (e.g. IBM MQ, RabbitMQ), and the list goes on.

The oc get pods command can be used to list the Prometheus pods, often in the openshift-monitoring namespace.

~]$ oc get pods --namespace openshift-monitoring
NAME                                                     READY   STATUS    RESTARTS        AGE
prometheus-adapter-6b98c646c7-m4g76                      1/1     Running   0               8d
prometheus-adapter-6b98c646c7-tczr2                      1/1     Running   0               8d
prometheus-k8s-0                                         6/6     Running   0               11d
prometheus-k8s-1                                         6/6     Running   0               11d
prometheus-operator-6766f68555-mkfv9                     2/2     Running   0               11d
prometheus-operator-admission-webhook-8589888cbc-mq2jx   1/1     Running   0               11d
prometheus-operator-admission-webhook-8589888cbc-t62mt   1/1     Running   0               11d

The oc exec and curl commands can be used to issue a GET request to the Prometheus /api/v1/targets endpoint inside of your Prometheus pod. This will often output a slew of data, too much to parse as stdout, so almost always the output will be redirected to a JSON file.

oc exec pod/prometheus-k8s-0 --container prometheus --namespace openshift-monitoring -- curl -s 'http://localhost:9090/api/v1/targets' | python -m json.tool > prometheus-k8s-0-targets.json

The output JSON should have something like this.

{
    "data": {
        "activeTargets": [
            {
                "discoveredLabels": {
                    "__address__": "10.130.0.127:8443",
                    "__meta_kubernetes_endpoint_address_target_kind": "Pod",
                    "__meta_kubernetes_endpoint_address_target_name": "openshift-apiserver-operator-69bf9465f4-p6cm8",
                    "__meta_kubernetes_endpoint_node_name": "stg001-master-0",
                    "__meta_kubernetes_endpoint_port_name": "https",
                    "__meta_kubernetes_endpoint_port_protocol": "TCP",
                    "__meta_kubernetes_endpoint_ready": "true",
                    "__meta_kubernetes_endpoints_label_app": "openshift-apiserver-operator",
                    "__meta_kubernetes_endpoints_labelpresent_app": "true",
                    "__meta_kubernetes_endpoints_name": "metrics",
                    "__meta_kubernetes_namespace": "openshift-apiserver-operator",
                    "__meta_kubernetes_pod_annotation_k8s_v1_cni_cncf_io_network_status": "[{\n    \"name\": \"openshift-sdn\",\n    \"interface\": \"eth0\",\n    \"ips\": [\n        \"10.130.0.127\"\n    ],\n    \"default\": true,\n    \"dns\": {}\n}]",
                    "__meta_kubernetes_pod_annotation_k8s_v1_cni_cncf_io_networks_status": "[{\n    \"name\": \"openshift-sdn\",\n    \"interface\": \"eth0\",\n    \"ips\": [\n        \"10.130.0.127\"\n    ],\n    \"default\": true,\n    \"dns\": {}\n}]",
                    "__meta_kubernetes_pod_annotation_openshift_io_scc": "nonroot-v2",
                    "__meta_kubernetes_pod_annotation_seccomp_security_alpha_kubernetes_io_pod": "runtime/default",
                    "__meta_kubernetes_pod_annotationpresent_k8s_v1_cni_cncf_io_network_status": "true",
                    "__meta_kubernetes_pod_annotationpresent_k8s_v1_cni_cncf_io_networks_status": "true",
                    "__meta_kubernetes_pod_annotationpresent_openshift_io_scc": "true",
                    "__meta_kubernetes_pod_annotationpresent_seccomp_security_alpha_kubernetes_io_pod": "true",
                    "__meta_kubernetes_pod_container_image": "quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c0a6add7c15fdad18bcf3c6151f8364108028385c145cfe78cf8dac88610bf9a",
                    "__meta_kubernetes_pod_container_name": "openshift-apiserver-operator",
                    "__meta_kubernetes_pod_container_port_name": "metrics",
                    "__meta_kubernetes_pod_container_port_number": "8443",
                    "__meta_kubernetes_pod_container_port_protocol": "TCP",
                    "__meta_kubernetes_pod_controller_kind": "ReplicaSet",
                    "__meta_kubernetes_pod_controller_name": "openshift-apiserver-operator-69bf9465f4",
                    "__meta_kubernetes_pod_host_ip": "10.85.188.119",
                    "__meta_kubernetes_pod_ip": "10.130.0.127",
                    "__meta_kubernetes_pod_label_app": "openshift-apiserver-operator",
                    "__meta_kubernetes_pod_label_pod_template_hash": "69bf9465f4",
                    "__meta_kubernetes_pod_labelpresent_app": "true",
                    "__meta_kubernetes_pod_labelpresent_pod_template_hash": "true",
                    "__meta_kubernetes_pod_name": "openshift-apiserver-operator-69bf9465f4-p6cm8",
                    "__meta_kubernetes_pod_node_name": "stg001-master-0",
                    "__meta_kubernetes_pod_phase": "Running",
                    "__meta_kubernetes_pod_ready": "true",
                    "__meta_kubernetes_pod_uid": "8bc06d66-fff6-4682-a94b-25274ad4615f",
                    "__meta_kubernetes_service_annotation_include_release_openshift_io_ibm_cloud_managed": "true",
                    "__meta_kubernetes_service_annotation_include_release_openshift_io_self_managed_high_availability": "true",
                    "__meta_kubernetes_service_annotation_include_release_openshift_io_single_node_developer": "true",
                    "__meta_kubernetes_service_annotation_service_alpha_openshift_io_serving_cert_secret_name": "openshift-apiserver-operator-serving-cert",
                    "__meta_kubernetes_service_annotation_service_alpha_openshift_io_serving_cert_signed_by": "openshift-service-serving-signer@1602707828",
                    "__meta_kubernetes_service_annotation_service_beta_openshift_io_serving_cert_signed_by": "openshift-service-serving-signer@1602707828",
                    "__meta_kubernetes_service_annotationpresent_include_release_openshift_io_ibm_cloud_managed": "true",
                    "__meta_kubernetes_service_annotationpresent_include_release_openshift_io_self_managed_high_availability": "true",
                    "__meta_kubernetes_service_annotationpresent_include_release_openshift_io_single_node_developer": "true",
                    "__meta_kubernetes_service_annotationpresent_service_alpha_openshift_io_serving_cert_secret_name": "true",
                    "__meta_kubernetes_service_annotationpresent_service_alpha_openshift_io_serving_cert_signed_by": "true",
                    "__meta_kubernetes_service_annotationpresent_service_beta_openshift_io_serving_cert_signed_by": "true",
                    "__meta_kubernetes_service_label_app": "openshift-apiserver-operator",
                    "__meta_kubernetes_service_labelpresent_app": "true",
                    "__meta_kubernetes_service_name": "metrics",
                    "__metrics_path__": "/metrics",
                    "__scheme__": "https",
                    "__scrape_interval__": "30s",
                    "__scrape_timeout__": "10s",
                    "__tmp_hash": "0"
                },
                "globalUrl": "https://10.130.0.127:8443/metrics",
                "health": "up",
                "labels": {
                    "container": "openshift-apiserver-operator",
                    "endpoint": "https",
                    "instance": "10.130.0.127:8443",
                    "job": "metrics",
                    "namespace": "openshift-apiserver-operator",
                    "pod": "openshift-apiserver-operator-69bf9465f4-p6cm8",
                    "service": "metrics"
                },
                "lastError": "",
                "lastScrape": "2024-01-25T03:36:16.660767501Z",
                "lastScrapeDuration": 0.006008754,
                "scrapeInterval": "30s",
                "scrapePool": "serviceMonitor/openshift-apiserver-operator/openshift-apiserver-operator/0",
                "scrapeTimeout": "10s",
                "scrapeUrl": "https://10.130.0.127:8443/metrics"
            }
        ]
    }
}

To only list active targets.

oc exec pod/prometheus-k8s-0 --container prometheus --namespace openshift-monitoring -- curl -s 'http://localhost:9090/api/v1/targets?state=active' | python -m json.tool > prometheus-active-targets.json