{% extends "en-us/edit/cluster/ClusterGenEditPage.htm" %}
{%- block htmBodyStartClusterPage %}
{{ super() }}
{% if result is defined %}
<wa-details>
  <div slot="summary">
    <i class="fa-regular fa-people-line"></i>
    Projects in the {{ result.clusterName | e }} cluster
  </div>
  <div class="wa-stack">
    {% for project in clusterProjects %}
    <article>
      <wa-divider></wa-divider>
      <div class="wa-stack ">
        <div class="wa-cluster ">
          <i class="fa-regular fa-people-line fa-2xl "></i>
          <a href="{{ project.editPage }}">
            <strong>
              {{ project.projectName }}
            </strong> 
            project in
            <strong>
              {{ project.clusterName }}
            </strong> 
            cluster in
            <strong>
              {{ project.hubId }}
            </strong> 
            hub
          </a>
        </div>
        <div class="wa-cluster ">

          {% if project.podRestartCount > 0 %}
          <wa-tooltip for="clusterProject-{{ project.projectName }}-pod-restarts">
            <ul>
              {% for pod in project.podsRestarting %}
              <li>
                {{ pod | e }}
              </li>
              {% endfor %}
            </ul>
          </wa-tooltip>
          {% endif %}
          <wa-button id="clusterProject-{{ project.projectName }}-pod-restarts">
            pod restarts
            <wa-badge{% if project.podRestartCount > 0 %} variant="danger" attention="pulse"{% else %} variant="success"{% endif %}" pill>
              {{ project.podRestartCount }}
            </wa-badge>
          </wa-button>

          {% if project.podTerminatingCount > 0 %}
          <wa-tooltip for="tenantProject-{{ project.projectName }}-pod-restarts">
            <ul>
              {% for pod in project.podsTerminating %}
              <li>
                {{ pod | e }}
              </li>
              {% endfor %}
            </ul>
          </wa-tooltip>
          {% endif %}
          <wa-button id="tenantProject-{{ project.projectName }}-pod-terminating">
            pods terminating
            <wa-badge{% if project.podTerminatingCount > 0 %} variant="danger" attention="pulse"{% else %} variant="success"{% endif %}" pill>
              {{ project.podTerminatingCount }}
            </wa-badge>
          </wa-button>

          {% if project.fullPvcsCount > 0 %}
          <wa-tooltip for="clusterProject-{{ project.projectName }}-full-pvcs">
            <ul>
              {% for pod in project.fullPvcs %}
              <li>
                {{ pod | e }}
              </li>
              {% endfor %}
            </ul>
          </wa-tooltip>
          {% endif %}
          <wa-button id="clusterProject-{{ project.projectName }}-full-pvcs">
            full PVCs
            <wa-badge{% if project.fullPvcsCount > 0 %} variant="warning" attention="pulse"{% else %} variant="success"{% endif %}" pill>
              {{ project.fullPvcsCount }}
            </wa-badge>
          </wa-button>

          {% if project.namespaceTerminating %}
          <wa-button id="clusterProject-{{ project.projectName }}-namespace-terminating">
            terminating
            <wa-badge variant="warning" attention="pulse" pill>
              <i class="{{ FONTAWESOME_STYLE }} fa-trash-can-xmark fa-spin "></i>
            </wa-badge>
          </wa-button>
          {% endif %}

          <wa-tooltip for="clusterProject-{{ project.projectName }}-last-updated">
            <wa-format-date weekday="short" month="short" day="numeric" year="numeric" hour="numeric" minute="numeric" second="numeric" time-zone-name="short" date="{{ formatZonedDateTime(project.modified, "yyyy-MM-dd'T'HH:mm:ss.SSSX", defaultLocaleId, "UTC") }}"></wa-format-date>
          </wa-tooltip>
          <span class="wa-caption-s" id="clusterProject-{{ project.projectName }}-last-updated">
            Updated
            <wa-relative-time class="wa-caption-s" date="{{ formatZonedDateTime(project.modified, "yyyy-MM-dd'T'HH:mm:ss.SSSX", defaultLocaleId, "UTC") }}"></wa-relative-time>
          </span>
        </div>
      </div>
    </article>
    {% endfor %}
  </div>
</wa-details>
<wa-details open>
  <div slot="summary">
    Observability dashboard for cluster {{ result.clusterName }}
  </div>
  <div class="wa-stack ">
    <div class="wa-stack ">
      <div>
        <wa-button variant="brand" href="{{ SITE_BASE_URL }}/en-us/search/project?fq={{ 'hubId:' + result.hubId | urlencode() }}&fq={{ 'clusterName:' + result.clusterName | urlencode() }}&facet.field=hubId&facet.field=clusterName">
          <i slot="start" class="fa-regular fa-people-line"></i>
          View all projects in the {{ result.clusterName | e }} cluster in the {{ result.hubId }} hub
        </wa-button>
      </div>
      <div>
        <wa-button variant="brand" href="{{ SITE_BASE_URL }}/en-us/search/ai-node?fq={{ 'hubId:' + result.hubId | urlencode() }}&fq={{ 'clusterName:' + result.clusterName | urlencode() }}&facet.field=hubId&facet.field=clusterName">
          <i slot="start" class="fa-regular fa-computer"></i>
          View all nodes in the {{ result.clusterName | e }} cluster in the {{ result.hubId }} hub
        </wa-button>
      </div>
    </div>
    <div class="wa-grid ">
      <wa-card with-header>
        <header slot="header" class="wa-heading-m ">Percent of total memory usage for {{ result.clusterName }} cluster</header>
        <div id="visualization-cluster-memory-bytes" class=""></div>
      </wa-card>
      <wa-card with-header>
        <header slot="header" class="wa-heading-m ">Percent of total CPU usage for {{ result.clusterName }} cluster</header>
        <div id="visualization-cluster-cpu-bytes" class=""></div>
      </wa-card>
    </div>
    <div class="wa-grid ">
      <div class="wa-stack ">
        <div class="wa-grid ">
          <wa-card with-header>
            <header slot="header" class="wa-heading-l ">Number of GPU nodes in cluster {{ result.clusterName }}</header>
            <div id="visualization-cluster-gpu-nodes" class="wa-frame "></div>
          </wa-card>
          <wa-card with-header>
            <header slot="header" class="wa-heading-l ">Number of GPU devices in cluster {{ result.clusterName }}</header>
            <div id="visualization-cluster-gpu-devices" class="wa-frame "></div>
          </wa-card>
        </div>
        <wa-card with-header>
          <header slot="header" class="wa-heading-m ">GPU Nodes in not ready state in cluster {{ result.clusterName }}</header>
          <div id="visualization-gpu-nodes-not-ready" class="wa-grid "></div>
        </wa-card>
        <wa-card with-header>
          <header slot="header" class="wa-heading-m ">GPU Nodes without GPU product taint in cluster {{ result.clusterName }}</header>
          <div id="visualization-gpu-nodes-without-taint" class="wa-grid "></div>
        </wa-card>
      </div>
      <wa-card with-header>
        <header slot="header" class="wa-heading-l ">Number of GPU devices per node in cluster {{ result.clusterName }}</header>
        <div id="visualization-cluster-gpu-devices-by-node" class="wa-grid "></div>
      </wa-card>
    </div>
    <div class="wa-grid ">
      <wa-card with-header>
        <header slot="header" class="wa-heading-l ">GPU temperature per GPU device in cluster {{ result.clusterName }}</header>
        <div id="visualization-cluster-gpu-temperature" class="wa-grid "></div>
      </wa-card>
    </div>
    <div class="wa-grid ">
      <wa-card with-header>
        <header slot="header" class="wa-heading-l ">GPU utilization per GPU device in cluster {{ result.clusterName }}</header>
        <div id="visualization-cluster-gpu-utilization" class="wa-grid "></div>
      </wa-card>
    </div>
    <wa-card with-header>
      <header slot="header" class="wa-heading-l ">GPU utilization per project in cluster {{ result.clusterName }}</header>
      <div id="visualization-project-gpu-utilization" class="wa-grid "></div>
    </wa-card>
  </div>
</wa-details>
{% endif %}
{%- endblock htmBodyStartClusterPage %}
{%- block htmStyleClusterPage %}
#visualization-cluster-gpu-nodes, #visualization-cluster-gpu-devices {
  wa-avatar {
    --size: 3em;
  }
}
#observability-dashboard {
  .pill {
    border-radius: var(--wa-border-radius-pill);
    box-shadow: var(--wa-theme-glossy-inner-shine), var(--wa-theme-glossy-top-highlight), inset 0 .7rem 0 0 rgba(255, 255, 255, 0.1), var(--wa-theme-glossy-lower-shade), var(--wa-theme-glossy-bottom-shadow);
    &:hover {
     background-color: var(--background-color-hover, var(--background-color));
      border-color: var(--border-color-hover, var(--border-color, var(--background-color-hover)));
      color: var(--text-color-hover, var(--text-color));
    }
  }
}
{%- endblock htmStyleClusterPage %}
{%- block htmScriptInitClusterPage %}

      var facetRangeGapVal = document.querySelector("#pageSearchVal-pageFacetRangeGap-Cluster-input").value;
      var start = '{{ formatZonedDateTime(defaultRangeStart, "yyyy-MM-dd'T'HH:mm:ss.SSSX", defaultLocaleId, 'UTC') }}';
      var end = '{{ formatZonedDateTime(defaultRangeEnd, "yyyy-MM-dd'T'HH:mm:ss.SSSX", defaultLocaleId, 'UTC') }}';
      var step;
      switch (facetRangeGapVal) {
        case "+1YEAR":
          step = "1y";
          break;
        case "+1MONTH":
          step = "1M";
          break;
        case "+1DAY":
          step = "1d";
          break;
        case "+1HOUR":
          step = "1h";
          break;
        case "+1MINUTE":
          step = "1m";
          break;
        default:
          step = "7d";
      }

      var timeQuery = {
        start: start
        , end: end
        , step: step
      };

      Promise.all([
          queryMemoryBytesTotal(timeQuery)
          , queryCpuBytesTotal(timeQuery)
          , queryGpuNodesTotal()
          , queryGpuDevicesByNode()
          , queryGpuDevicesTotal()
          , queryGpuNodesNotReady()
          , queryGpuNodesWithoutTaint()
          , queryGpuTemperature()
          , queryGpuUtilization()
          , queryGpuUtilizationPerProject()
          ]);
{%- endblock htmScriptInitClusterPage %}
{%- block htmScriptsClusterPage %}
{{ super() }}
  <script>

    async function queryMemoryBytesTotal(timeQuery) {
      const urls = [];
      const hubIds = ['{{ result.hubId }}'];
      hubIds.forEach((hubId, i) => {
        urls.push('/prom-keycloak-proxy/' + encodeURIComponent(hubId) + '/api/v1/query_range?' + new URLSearchParams(Object.assign({ 
            {% if result.hubId == 'openshift-local' -%}
            query: 'cluster:memory_usage_bytes:sum / on (prometheus) cluster:capacity_memory_bytes:sum' 
            {%- else -%}
            query: 'cluster:memory_usage_bytes:sum / cluster:capacity_memory_bytes:sum{cluster="{{ result.clusterNameMetrics }}", label_node_role_kubernetes_io!="master"}' 
            {%- endif %}
            }, timeQuery)).toString());
      });
      const responses = await Promise.all(urls.map(url => fetch(url)));
      const dataPromises = responses.map(response => response.json());
      const responseJsons = await Promise.all(dataPromises);
      var end = new Date();
      var start = new Date(new Date().setHours(end.getHours() - 1));

      var panelId = 'visualization-cluster-memory-bytes';
      var traces = [];
      responseJsons.forEach((responseJson, i) => {
        responseJson.data.result.forEach((result, j) => {
          var hubId = hubIds[i];
          traces.push({
            x: result.values.map(values => new Date(values[0] * 1000))
            , y: result.values.map(values => parseFloat(values[1]))
            , mode: 'scatter'
            , name: result.metric.cluster + ' in ' + hubId
            , text: result.values.map(values => '<a href="/en-us/edit/cluster/' + result.metric.cluster + '">' + hubId + ' ' + result.metric.cluster + '</a>')
          });
        });
      });

      var layout = {
        xaxis: {
          title: 'Time in {{ defaultZoneId }}'
        }
        , yaxis: {
          title: 'Percent of total memory usage'
          , tickformat: '.0%'
        }
        , legend: {
          title: {
            text: 'Clusters'
          }
        }
      };
      Plotly.newPlot(panelId, traces, layout);
    }

    async function queryCpuBytesTotal(timeQuery) {
      const urls = [];
      const hubIds = ['{{ result.hubId }}'];
      hubIds.forEach((hubId, i) => {
        urls.push('/prom-keycloak-proxy/' + encodeURIComponent(hubId) + '/api/v1/query_range?' + new URLSearchParams(Object.assign({ 
            {% if result.hubId == 'openshift-local' -%}
            query: 'cluster:cpu_usage_cores:sum / on (prometheus) cluster:capacity_cpu_cores:sum' 
            {%- else -%}
            query: 'cluster:cpu_usage_cores:sum{cluster="{{ result.clusterNameMetrics }}"} / on(cluster) sum by (cluster) (cluster:capacity_cpu_cores:sum{label_node_role_kubernetes_io!="master"})' 
            {%- endif %}
            }, timeQuery)).toString());
      });
      const responses = await Promise.all(urls.map(url => fetch(url)));
      const dataPromises = responses.map(response => response.json());
      const responseJsons = await Promise.all(dataPromises);
      var end = new Date();
      var start = new Date(new Date().setHours(end.getHours() - 1));

      var panelId = 'visualization-cluster-cpu-bytes';
      var traces = [];
      responseJsons.forEach((responseJson, i) => {
        responseJson.data.result.forEach((result, j) => {
          var hubId = hubIds[i];
          traces.push({
            x: result.values.map(values => new Date(values[0] * 1000))
            , y: result.values.map(values => parseFloat(values[1]))
            , mode: 'scatter'
            , name: result.metric.cluster + ' in ' + hubId
            , text: result.values.map(values => '<a href="/en-us/edit/cluster/' + result.metric.cluster + '">' + hubId + ' ' + result.metric.cluster + '</a>')
          });
        });
      });

      var layout = {
        xaxis: {
          title: 'Time in {{ defaultZoneId }}'
        }
        , yaxis: {
          title: 'Percent of total CPU usage'
          , tickformat: '.0%'
        }
        , legend: {
          title: {
            text: 'Clusters'
          }
        }
      };
      Plotly.newPlot(panelId, traces, layout);
    }

    async function queryGpuNodesTotal() {
      var query = Object.assign({ query: 'sum(gpu_operator_gpu_nodes_total{cluster="{{ result.clusterNameMetrics }}"})' });
      var response = await fetch( '/prom-keycloak-proxy/' + encodeURIComponent('{{ result.hubId }}') + '/api/v1/query?' + new URLSearchParams(query).toString(), { method: 'GET' });
      if(response.ok) {
        response.json().then(function(responseJson) {

          if(responseJson.data.result.length > 0) {
            var avatar = document.createElement('wa-avatar');
            avatar.setAttribute('initials', responseJson.data.result[0].value[1])
            avatar.setAttribute('class', 'wa-accent wa-brand wa-heading-3xl ');
            avatar.setAttribute('slot', 'icon');
            document.querySelector('#visualization-cluster-gpu-nodes').append(avatar);
          }
        });
      } else {
        throw new Error(`Failed to query GPU nodes total: ${response.status}`);
      }
    }

    async function queryGpuDevicesByNode() {
      var query = Object.assign({ query: 'sum(gpu_operator_node_device_plugin_devices_total{cluster="{{ result.clusterNameMetrics }}"}) by (cluster, node)' });
      var response = await fetch( '/prom-keycloak-proxy/' + encodeURIComponent('{{ result.hubId }}') + '/api/v1/query?' + new URLSearchParams(query).toString(), { method: 'GET' });
      if(response.ok) {
        response.json().then(function(responseJson) {
          var panelId = 'visualization-cluster-gpu-devices-by-node';
          responseJson.data.result.forEach((result, i) => {
            var elemId = 'visualization-cluster-gpu-devices-by-node-' + result.metric.cluster + '-' + result.metric.node;

            var a = document.createElement('a');
            a.setAttribute('href', '/en-us/edit/ai-node/' + result.metric.cluster + '-' + result.metric.node);
            document.querySelector('#' + panelId).append(a);

            var callout = document.createElement('wa-callout');
            callout.setAttribute('id', elemId);
            callout.setAttribute('appearance', 'outlined');
            callout.setAttribute('variant', 'neutral');
            callout.setAttribute('class', 'pill');
            a.append(callout);

            var avatar = document.createElement('wa-avatar');
            avatar.setAttribute('initials', result.value[1])
            avatar.setAttribute('class', 'wa-accent wa-brand ');
            avatar.setAttribute('slot', 'icon');
            callout.append(avatar);

            var stack = document.createElement('div');
            stack.setAttribute('class', 'wa-stack wa-gap-s ')
            callout.append(stack);

            var elem0 = document.createElement('div');
            elem0.innerText = result.metric.cluster;
            stack.append(elem0);

            var elem1 = document.createElement('div');
            elem1.innerText = result.metric.node;
            stack.append(elem1);
          });
        });
      } else {
        throw new Error(`Failed to query GPU nodes total: ${response.status}`);
      }
    }

    async function queryGpuDevicesTotal() {
      var query = Object.assign({ query: 'sum(gpu_operator_node_device_plugin_devices_total{cluster="{{ result.clusterNameMetrics }}"})' });
      var response = await fetch( '/prom-keycloak-proxy/' + encodeURIComponent('{{ result.hubId }}') + '/api/v1/query?' + new URLSearchParams(query).toString(), { method: 'GET' });
      if(response.ok) {
        response.json().then(function(responseJson) {

          if(responseJson.data.result.length > 0) {
            var avatar = document.createElement('wa-avatar');
            avatar.setAttribute('initials', responseJson.data.result[0].value[1])
            avatar.setAttribute('class', 'wa-accent wa-brand wa-heading-3xl ');
            avatar.setAttribute('slot', 'icon');
            document.querySelector('#visualization-cluster-gpu-devices').append(avatar);
          }
        });
      } else {
        throw new Error(`Failed to query GPU nodes total: ${response.status}`);
      }
    }

    async function queryGpuNodesNotReady() {
      var query = Object.assign({ query: 'kube_node_status_condition{cluster="{{ result.clusterNameMetrics }}", status="unknown", condition="Ready"} * on (cluster, node) group_left(label_nvidia_com_gpu_present) (sum by (cluster, node, label_nvidia_com_gpu_present) (kube_node_labels{label_nvidia_com_gpu_present="true"})) == 1' });
      var response = await fetch( '/prom-keycloak-proxy/' + encodeURIComponent('{{ result.hubId }}') + '/api/v1/query?' + new URLSearchParams(query).toString(), { method: 'GET' });
      if(response.ok) {
        response.json().then(function(responseJson) {
          var panelId = 'visualization-gpu-nodes-not-ready';
          responseJson.data.result.forEach((result, i) => {
            var elemId = 'visualization-gpu-nodes-not-ready-' + result.metric.cluster + '-' + result.metric.node;

            var a = document.createElement('a');
            a.setAttribute('href', '/en-us/edit/ai-node/' + result.metric.cluster + '-' + result.metric.node);
            document.querySelector('#' + panelId).append(a);

            var callout = document.createElement('wa-callout');
            callout.setAttribute('id', elemId);
            callout.setAttribute('appearance', 'accent outlined');
            callout.setAttribute('variant', 'danger');
            callout.setAttribute('class', 'pill');
            a.append(callout);

            var icon = document.createElement('wa-icon');
            icon.setAttribute('slot', 'icon')
            icon.setAttribute('name', 'triangle-exclamation')
            icon.setAttribute('variant', 'solid')
            callout.append(icon);

            var stack = document.createElement('div');
            stack.setAttribute('class', 'wa-stack wa-gap-s ')
            callout.append(stack);

            var elem0 = document.createElement('div');
            elem0.innerText = result.metric.cluster;
            stack.append(elem0);

            var elem1 = document.createElement('div');
            elem1.innerText = result.metric.node;
            stack.append(elem1);
          });
        });
      } else {
        throw new Error(`Failed to query nodes not ready: ${response.status}`);
      }
    }

    async function queryGpuNodesWithoutTaint() {
      var query = Object.assign({ query: '(sum by (cluster, node, label_nvidia_com_gpu_present) (kube_node_labels{cluster="{{ result.clusterNameMetrics }}", label_nvidia_com_gpu_present="true"})) > 0 unless on(cluster, node) kube_node_spec_taint{key="nvidia.com/gpu.product"}' });
      var response = await fetch( '/prom-keycloak-proxy/' + encodeURIComponent('{{ result.hubId }}') + '/api/v1/query?' + new URLSearchParams(query).toString(), { method: 'GET' });
      if(response.ok) {
        response.json().then(function(responseJson) {
          var panelId = 'visualization-gpu-nodes-without-taint';
          responseJson.data.result.forEach((result, i) => {
            var elemId = 'visualization-gpu-nodes-without-taint-' + result.metric.cluster + '-' + result.metric.node;

            var a = document.createElement('a');
            a.setAttribute('href', '/en-us/edit/ai-node/' + result.metric.cluster + '-' + result.metric.node);
            document.querySelector('#' + panelId).append(a);

            var callout = document.createElement('wa-callout');
            callout.setAttribute('id', elemId);
            callout.setAttribute('appearance', 'accent outlined');
            callout.setAttribute('variant', 'brand');
            callout.setAttribute('class', 'pill');
            a.append(callout);

            var icon = document.createElement('wa-icon');
            icon.setAttribute('slot', 'icon')
            icon.setAttribute('name', 'triangle-exclamation')
            icon.setAttribute('variant', 'solid')
            callout.append(icon);

            var stack = document.createElement('div');
            stack.setAttribute('class', 'wa-stack wa-gap-s ')
            callout.append(stack);

            var elem0 = document.createElement('div');
            elem0.innerText = result.metric.cluster;
            stack.append(elem0);

            var elem1 = document.createElement('div');
            elem1.innerText = result.metric.node;
            stack.append(elem1);
          });
        });
      } else {
        throw new Error(`Failed to query nodes not ready: ${response.status}`);
      }
    }

    async function queryGpuTemperature() {
      var end = new Date();
      var start = new Date(new Date().setHours(end.getHours() - 24));
      var timeQuery = {
        start: start.toISOString()
        , end: end.toISOString()
        , step: '100'
      };
      var query = Object.assign(Object.assign({ query: 'sum(DCGM_FI_DEV_MEMORY_TEMP{cluster="{{ result.clusterNameMetrics }}"}) by (cluster,Hostname,modelName,UUID)' }, timeQuery));
      var response = await fetch( '/prom-keycloak-proxy/' + encodeURIComponent('{{ result.hubId }}') + '/api/v1/query_range?' + new URLSearchParams(query).toString(), { method: 'GET' });
      if(response.ok) {
        response.json().then(function(responseJson) {
          var end = new Date();
          var start = new Date(new Date().setHours(end.getHours() - 1));

          var panelId = 'visualization-cluster-gpu-temperature';
          var traces = [];
          var results = responseJson.data.result;
          results.forEach((result, i) => {
            traces.push({
              x: result.values.map(values => new Date(values[0] * 1000))
              , y: result.values.map(values => parseFloat(values[1]))
              , mode: 'lines+markers'
              , name: result.metric.cluster + ' ' + result.metric.Hostname + ' ' + result.metric.modelName
              , text: result.values.map(values => '<a href="/en-us/edit/ai-node/' + result.metric.cluster + '-' + result.metric.node + '">' + result.metric.cluster + ' ' + result.metric.Hostname + ' ' + result.metric.modelName + '</a>')
            });
          });

          var layout = {
          };
          Plotly.newPlot(panelId, traces, layout);
        });
      } else {
        throw new Error(`Failed to query nodes not ready: ${response.status}`);
      }
    }

    async function queryGpuUtilization() {
      var end = new Date();
      var start = new Date(new Date().setHours(end.getHours() - 24));
      var timeQuery = {
        start: start.toISOString()
        , end: end.toISOString()
        , step: '100'
      };
      var query = Object.assign(Object.assign({ query: 'sum(DCGM_FI_DEV_GPU_UTIL{cluster="{{ result.clusterNameMetrics }}"}) by (cluster,Hostname,modelName,UUID)' }, timeQuery));
      var response = await fetch( '/prom-keycloak-proxy/' + encodeURIComponent('{{ result.hubId }}') + '/api/v1/query_range?' + new URLSearchParams(query).toString(), { method: 'GET' });
      if(response.ok) {
        response.json().then(function(responseJson) {
          var end = new Date();
          var start = new Date(new Date().setHours(end.getHours() - 1));

          var panelId = 'visualization-cluster-gpu-utilization';
          var traces = [];
          var results = responseJson.data.result;
          results.forEach((result, i) => {
            traces.push({
              x: result.values.map(values => new Date(values[0] * 1000))
              , y: result.values.map(values => parseFloat(values[1]))
              , mode: 'lines+markers'
              , name: result.metric.cluster + ' ' + result.metric.Hostname + ' ' + result.metric.modelName
              , text: result.values.map(values => '<a href="/en-us/edit/ai-node/' + result.metric.cluster + '-' + result.metric.node + '">' + result.metric.cluster + ' ' + result.metric.Hostname + ' ' + result.metric.modelName + '</a>')
            });
          });

          var layout = {
          };
          Plotly.newPlot(panelId, traces, layout);
        });
      } else {
        throw new Error(`Failed to query nodes not ready: ${response.status}`);
      }
    }

    async function queryGpuUtilizationPerProject() {
      var end = new Date();
      var start = new Date(new Date().setHours(end.getHours() - 24));
      var timeQuery = {
        start: start.toISOString()
        , end: end.toISOString()
        , step: '100'
      };
      var query = Object.assign(Object.assign({ query: 'sum(DCGM_FI_DEV_GPU_UTIL{cluster="{{ result.clusterNameMetrics }}"}) by (cluster,exported_namespace)' }, timeQuery));
      var response = await fetch( '/prom-keycloak-proxy/' + encodeURIComponent('{{ result.hubId }}') + '/api/v1/query_range?' + new URLSearchParams(query).toString(), { method: 'GET' });
      if(response.ok) {
        response.json().then(function(responseJson) {
          var end = new Date();
          var start = new Date(new Date().setHours(end.getHours() - 1));

          var panelId = 'visualization-project-gpu-utilization';
          var traces = [];
          var results = responseJson.data.result;
          results.forEach((result, i) => {
            traces.push({
              x: result.values.map(values => new Date(values[0] * 1000))
              , y: result.values.map(values => parseFloat(values[1]))
              , mode: 'lines+markers'
              , name: result.metric.cluster + ' ' + result.metric.exported_namespace
              , text: result.values.map(values => '<a href="/en-us/edit/ai-node/' + result.metric.cluster + '-' + result.metric.exported_namespace + '">' + result.metric.cluster + ' ' + result.metric.exported_namespace + '</a>')
            });
          });

          var layout = {
          };
          Plotly.newPlot(panelId, traces, layout);
        });
      } else {
        throw new Error(`Failed to query projects not ready: ${response.status}`);
      }
    }

    async function queryGpuUtilizationPerProject() {
      var end = new Date();
      var start = new Date(new Date().setHours(end.getHours() - 24));
      var timeQuery = {
        start: start.toISOString()
        , end: end.toISOString()
        , step: '100'
      };
      var query = Object.assign(Object.assign({ query: 'sum(DCGM_FI_DEV_GPU_UTIL{cluster="{{ result.clusterNameMetrics }}"}) by (cluster,exported_namespace)' }, timeQuery));
      var response = await fetch( '/prom-keycloak-proxy/' + encodeURIComponent('{{ result.hubId }}') + '/api/v1/query_range?' + new URLSearchParams(query).toString(), { method: 'GET' });
      if(response.ok) {
        response.json().then(function(responseJson) {
          var end = new Date();
          var start = new Date(new Date().setHours(end.getHours() - 1));

          var panelId = 'visualization-project-gpu-utilization';
          var traces = [];
          var results = responseJson.data.result;
          results.forEach((result, i) => {
            traces.push({
              x: result.values.map(values => new Date(values[0] * 1000))
              , y: result.values.map(values => parseFloat(values[1]))
              , mode: 'lines+markers'
              , name: result.metric.cluster + ' ' + result.metric.exported_namespace
              , text: result.values.map(values => '<a href="/en-us/edit/ai-node/' + result.metric.cluster + '-' + result.metric.exported_namespace + '">' + result.metric.cluster + ' ' + result.metric.exported_namespace + '</a>')
            });
          });

          var layout = {
          };
          Plotly.newPlot(panelId, traces, layout);
        });
      } else {
        throw new Error(`Failed to query projects not ready: ${response.status}`);
      }
    }
  </script>
{%- endblock htmScriptsClusterPage %}