{% extends "en-us/edit/project/ProjectGenEditPage.htm" %}
{% import "en-us/Chart.htm" %}
{%- block facetChangeProjectSuccess %}
      updateDashboards();
{% endblock facetChangeProjectSuccess -%}
{%- block htmScriptInitProjectPage %}
      updateDashboards();
{%- endblock htmScriptInitProjectPage %}
{%- block htmBodyStartProjectPage %}
{{ super() }}
{% if result.description is defined %}
<p>
  {{ result.description }}
</p>
{% endif %}
{% if result is defined %}
<wa-details open class="HtmRow" id="observability-dashboard">
  <div slot="summary">
    Observability dashboard for OpenShift projects
  </div>
  <div class="wa-stack ">
    <div class="wa-grid ">
      {{ dashboardCard('visualization-project-memory-bytes', 'Total project memory usage in cluster ' ~ result.clusterName ~ ' project ' ~ result.projectName, min=0) }}
      {{ dashboardCard('visualization-quota-memory-bytes', 'Project quota memory usage in cluster ' ~ result.clusterName ~ ' project ' ~ result.projectName, min=0) }}
    </div>
    <div class="wa-grid ">
      {{ dashboardCard('visualization-project-cpu-bytes', 'Total project CPU usage in cluster ' ~ result.clusterName ~ ' project ' ~ result.projectName, min=0) }}
      {{ dashboardCard('visualization-quota-cpu-bytes', 'Project quota CPU usage in cluster ' ~ result.clusterName ~ ' project ' ~ result.projectName, min=0) }}
    </div>
  </div>
</wa-details>
<wa-details open class="HtmRow" id="network-performance-dashboard">
  <div slot="summary">
    Network Performance Metrics
  </div>
  <div class="wa-stack ">
    <div class="wa-grid ">
      {{ dashboardCard('visualization-network-receive', 'Network receive bytes in cluster ' ~ result.clusterName ~ ' project ' ~ result.projectName, min=0) }}
      {{ dashboardCard('visualization-network-transmit', 'Network transmit bytes in cluster ' ~ result.clusterName ~ ' project ' ~ result.projectName, min=0) }}
    </div>
  </div>
</wa-details>
<wa-details{% if gpuEnabled is defined and gpuEnabled %} open{% endif %} class="gpu_details " id="gpu-performance-dashboard">
  <div slot="summary">
    GPU Performance Metrics
  </div>
  <div class="wa-stack ">
    <div class="wa-grid ">
      {{ dashboardCard('visualization-project-gpu-utilization', 'Project GPU utilization in cluster ' ~ result.clusterName ~ ' project ' ~ result.projectName) }}
      {{ dashboardCard('visualization-quota-gpu-utilization', 'Project quota GPU utilization in cluster ' ~ result.clusterName ~ ' project ' ~ result.projectName) }}
    </div>
    <div class="wa-grid ">
      {{ dashboardCard('visualization-project-gpu-resource-request', 'Project GPU resource requests in cluster ' ~ result.clusterName ~ ' project ' ~ result.projectName) }}
      {{ dashboardCard('visualization-project-gpu-temperature', 'Project GPU temperature in cluster ' ~ result.clusterName ~ ' project ' ~ result.projectName) }}
    </div>
  </div>
</wa-details>
<wa-details{% if vllmEnabled is defined and vllmEnabled %} open{% endif %} class="vllm_details " id="llm-performance-dashboard">
  <div slot="summary">
    LLM Performance Metrics
  </div>
  <div class="wa-stack ">
    <div class="wa-grid ">
      {{ dashboardCard('visualization-llm-e2e-latency', 'LLM E2E Request Latency in ' ~ result.projectName) }}
      {{ dashboardCard('visualization-llm-token-throughput', 'LLM Token Throughput in ' ~ result.projectName) }}
    </div>
    <div class="wa-grid ">
      {{ dashboardCard('visualization-llm-ttft-latency', 'LLM Time To First Token in ' ~ result.projectName) }}
      {{ dashboardCard('visualization-llm-tpot-latency', 'LLM Time Per Output Token in ' ~ result.projectName) }}
    </div>
  </div>
</wa-details>
<wa-details{% if gpuEnabled is defined and gpuEnabled %} open{% endif %} class="gpu_details " id="billing-section">
  <div slot="summary">
    GPU Resource Utilization
  </div>
  <div class="wa-stack">
    <wa-card with-header>
      <header slot="header" class="wa-heading-m">GPU Hours Used</header>
      <table id="billing-gpu-hours-table" class="wa-zebra-rows wa-hover-rows" style="display: none; width: 100%;">
        <thead>
          <tr>
            <th style="text-align: left;">GPU Model</th>
            <th style="text-align: left;">Pod</th>
            <th style="text-align: left;">Node</th>
            <th style="text-align: right;">GPU Hours</th>
          </tr>
        </thead>
        <tbody id="billing-gpu-hours-tbody">
        </tbody>
        <tfoot>
          <tr>
            <th colspan="3" style="text-align: right;">Total GPU Hours:</th>
            <th id="billing-gpu-hours-total" style="text-align: right;"></th>
          </tr>
        </tfoot>
      </table>
      <div id="billing-gpu-hours-empty" style="display: none;">
        No GPU usage data found for the selected time range.
      </div>
      <wa-details summary="Query Info" style="margin-top: 1rem;">
        <table class="wa-zebra-rows">
          <tbody id="billing-range-info">
          </tbody>
        </table>
        <p style="margin-top: 0.5rem;"><strong>Query:</strong></p>
        <code id="billing-query-display" style="word-break: break-all;"></code>
      </wa-details>
    </wa-card>
  </div>
</wa-details>
<wa-details open class="HtmRow" id="billing-section">
  <div slot="summary">
    Resource Utilization
  </div>
  <div class="wa-stack">
    <wa-card with-header>
      <header slot="header" class="wa-heading-m">Storage Utilization</header>
      <div id="storage-quota-section" style="display: none; margin-bottom: 1rem;">
        <div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 0.5rem;">
          <strong>Storage Used:</strong>
          <span id="storage-used-text"></span>
        </div>
        <div style="display: flex; justify-content: space-between; align-items: center;">
          <strong>Storage Allocated:</strong>
          <span id="storage-allocated-text"></span>
        </div>
      </div>
      <table id="storage-utilization-table" class="wa-zebra-rows wa-hover-rows" style="display: none; width: 100%;">
        <thead>
          <tr>
            <th style="text-align: left;">PVC Name</th>
            <th style="text-align: center;">Status</th>
            <th style="text-align: right;">Allocated</th>
            <th style="text-align: right;">Used</th>
            <th style="text-align: right;">Usage %</th>
          </tr>
        </thead>
        <tbody id="storage-utilization-tbody">
        </tbody>
        <tfoot>
          <tr>
            <th colspan="2" style="text-align: right;">Totals:</th>
            <th id="storage-allocated-total" style="text-align: right;"></th>
            <th id="storage-used-total" style="text-align: right;"></th>
            <th id="storage-percent-total" style="text-align: right;"></th>
          </tr>
        </tfoot>
      </table>
      <div id="storage-utilization-empty" style="display: none;">
        No storage usage data found. PVCs must be mounted to running pods to show usage.
      </div>
      <wa-details summary="Query Info" style="margin-top: 1rem;">
        <p><strong>Quota Query:</strong></p>
        <code id="storage-quota-query-display" style="word-break: break-all;"></code>
        <p style="margin-top: 0.5rem;"><strong>PVC Allocated Query:</strong></p>
        <code id="storage-allocated-query-display" style="word-break: break-all;"></code>
        <p style="margin-top: 0.5rem;"><strong>Used Query:</strong></p>
        <code id="storage-used-query-display" style="word-break: break-all;"></code>
      </wa-details>
    </wa-card>
  </div>
</wa-details>
{% endif %}
{%- endblock htmBodyStartProjectPage %}

{%- block htmScriptsProjectPage %}
      <script src="https://cdn.jsdelivr.net/npm/chart.js@4.5.1/dist/chart.umd.min.js"></script>
      <script src="https://cdn.jsdelivr.net/npm/hammerjs@2.0.8/hammer.min.js"></script>
      <script src="{{ STATIC_BASE_URL }}/js/chartjs-plugin-zoom.js"></script>
{{ super() }}
  <script>
        {{ updateDashboardsFunctionStart() }}
        // Time step for LLM metrics
        var llmTimeQuery = {
          start: start
          , end: end
          , step: "10m"
          , timeZone: timeZone
        };

        Promise.all([
            {{ dashboardPromise('visualization-project-memory-bytes') }}
            , {{ dashboardPromise('visualization-quota-memory-bytes') }}
            , {{ dashboardPromise('visualization-project-cpu-bytes') }}
            , {{ dashboardPromise('visualization-quota-cpu-bytes') }}
            , {{ dashboardPromise('visualization-network-receive') }}
            , {{ dashboardPromise('visualization-network-transmit') }}
            , queryStorageUtilization('{{ result.hubId }}', '{{ result.clusterName }}', '{{ result.projectName }}')
            , {{ dashboardPromise('visualization-project-gpu-utilization') }}
            , {{ dashboardPromise('visualization-quota-gpu-utilization') }}
            , {{ dashboardPromise('visualization-project-gpu-resource-request') }}
            , {{ dashboardPromise('visualization-project-gpu-temperature') }}
            , {{ dashboardPromise('visualization-llm-e2e-latency') }}
            , {{ dashboardPromise('visualization-llm-token-throughput') }}
            , {{ dashboardPromise('visualization-llm-ttft-latency') }}
            , {{ dashboardPromise('visualization-llm-tpot-latency') }}
            , queryGpuBillingHours('{{ result.hubId }}', '{{ result.clusterName }}', '{{ result.projectName }}', timeQuery)
            ]);
        {{ updateDashboardsFunctionEnd() }}

        {{ hubClusterProjectDashboardAsyncFunction(
          divId='visualization-project-memory-bytes'
          , classSimpleName='Project'
          , legendTitle='Projects'
          , metricPrefix='namespace:container_memory_usage_bytes:sum{'
          , metricSuffix='}'
          , modelVar='projectName'
          , timeZone=timeZone
          , formatBytes=true
          , yLabelSuffix=' memory'
          , datasetLabel='${result.metric.namespace} in ${result.metric.cluster} in ${hubId}'
        ) }}
        {{ hubClusterProjectDashboardAsyncFunction(
          divId='visualization-quota-memory-bytes'
          , classSimpleName='Project'
          , legendTitle='Projects'
          , metricPrefix='namespace:container_memory_usage_bytes:sum{'
          , metricMiddle='} / on(cluster, namespace) group_left() kube_resourcequota{resource="limits.memory", type="hard", '
          , metricSuffix='}'
          , modelVar='projectName'
          , timeZone=timeZone
          , tickFormat='.0%'
          , formatPercent=true
          , yLabelSuffix='% of quota'
          , datasetLabel='${result.metric.namespace} in ${result.metric.cluster} in ${hubId}'
        ) }}
        {{ hubClusterProjectDashboardAsyncFunction(
          divId='visualization-project-cpu-bytes'
          , classSimpleName='Project'
          , legendTitle='Projects'
          , metricPrefix='node_namespace_pod_container:container_cpu_usage_seconds_total:sum{'
          , metricSuffix='}'
          , modelVar='projectName'
          , timeZone=timeZone
          , emptyClusterReplace='node_namespace_pod_container:container_cpu_usage_seconds_total:sum'
          , emptyClusterReplaceWith='namespace:container_cpu_usage:sum'
          , yLabelSuffix=' CPUs'
          , datasetLabel='${result.metric.namespace} in ${result.metric.cluster} in ${hubId}'
        ) }}
        {{ hubClusterProjectDashboardAsyncFunction(
          divId='visualization-quota-cpu-bytes'
          , classSimpleName='Project'
          , legendTitle='Projects'
          , metricPrefix='node_namespace_pod_container:container_cpu_usage_seconds_total:sum{'
          , metricMiddle='} / on(cluster, namespace) group_left() kube_resourcequota{resource="limits.cpu", type="hard", '
          , metricSuffix='}'
          , modelVar='projectName'
          , timeZone=timeZone
          , tickFormat='.0%'
          , formatPercent=true
          , yLabelSuffix='% of quota'
          , datasetLabel='${result.metric.namespace} in ${result.metric.cluster} in ${hubId}'
        ) }}
        {{ hubClusterProjectDashboardAsyncFunction(
          divId='visualization-network-receive'
          , classSimpleName='Project'
          , legendTitle='Projects'
          , metricPrefix='sum(irate(container_network_receive_bytes_total{'
          , metricSuffix='}[10m])) by (cluster, namespace)'
          , modelVar='projectName'
          , timeZone=timeZone
          , formatBytes=true
          , yLabelSuffix=' received'
          , datasetLabel='${result.metric.namespace} in ${result.metric.cluster} in ${hubId}'
        ) }}
        {{ hubClusterProjectDashboardAsyncFunction(
          divId='visualization-network-transmit'
          , classSimpleName='Project'
          , legendTitle='Projects'
          , metricPrefix='sum(irate(container_network_transmit_bytes_total{'
          , metricSuffix='}[10m])) by (cluster, namespace)'
          , modelVar='projectName'
          , timeZone=timeZone
          , formatBytes=true
          , yLabelSuffix=' transmitted'
          , datasetLabel='${result.metric.namespace} in ${result.metric.cluster} in ${hubId}'
        ) }}
        {{ hubClusterProjectDashboardAsyncFunction(
          divId='visualization-project-gpu-utilization'
          , classSimpleName='Project'
          , legendTitle='Projects'
          , metricPrefix='sum by (cluster, exported_namespace) (DCGM_FI_DEV_GPU_UTIL{'
          , metricSuffix='} / 100)'
          , modelVar='projectName'
          , timeZone=timeZone
          , tickFormat='.0%'
          , modelPrefixLabel='exported_namespace'
          , yLabelSuffix='% of GPU'
          , openDetailsSelector='.gpu_details'
          , datasetLabel='${result.metric.exported_namespace} in ${result.metric.cluster} in ${hubId}'
        ) }}
        {{ hubClusterProjectDashboardAsyncFunction(
          divId='visualization-quota-gpu-utilization'
          , classSimpleName='Project'
          , legendTitle='Projects'
          , metricPrefix='sum by (cluster, exported_namespace) (DCGM_FI_DEV_GPU_UTIL{'
          , metricMiddle='} / 100 / on(cluster, exported_namespace) group_left() label_replace(kube_resourcequota{resource="requests.nvidia.com/gpu", type="hard", '
          , metricSuffix='}, "exported_namespace", "$1", "namespace", "(.*)"))'
          , modelVar='projectName'
          , timeZone=timeZone
          , tickFormat='.0%'
          , modelPrefixLabel='exported_namespace'
          , modelMiddleLabel='namespace'
          , yLabelSuffix='% of quota'
          , datasetLabel='${result.metric.exported_namespace} in ${result.metric.cluster} in ${hubId}'
        ) }}
        {{ hubClusterProjectDashboardAsyncFunction(
          divId='visualization-project-gpu-resource-request'
          , classSimpleName='Project'
          , legendTitle='Projects'
          , metricPrefix='sum by (cluster, namespace) (kube_pod_resource_request{resource =~ ".*/gpu", '
          , metricSuffix='})'
          , modelVar='projectName'
          , timeZone=timeZone
          , modelPrefixLabel='namespace'
          , yLabelSuffix='GPU request'
          , datasetLabel='${result.metric.namespace} in ${result.metric.cluster} in ${hubId}'
        ) }}
        {{ hubClusterProjectDashboardAsyncFunction(
          divId='visualization-project-gpu-temperature'
          , classSimpleName='Project'
          , legendTitle='Projects'
          , metricPrefix='sum by (cluster, exported_namespace) (DCGM_FI_DEV_GPU_TEMP{'
          , metricSuffix='})'
          , modelVar='projectName'
          , timeZone=timeZone
          , modelPrefixLabel='exported_namespace'
          , yLabelSuffix='temperature ° C'
          , datasetLabel='${result.metric.exported_namespace} in ${result.metric.cluster} in ${hubId}'
        ) }}
        {{ hubClusterProjectDashboardAsyncFunction(
          divId='visualization-llm-e2e-latency'
          , classSimpleName='Project'
          , legendTitle='E2E Latency'
          , metricPrefix='rate(vllm:e2e_request_latency_seconds_sum{'
          , metricMiddle='}[10m]) / rate(vllm:e2e_request_latency_seconds_count{'
          , metricSuffix='}[10m])'
          , modelVar='projectName'
          , timeZone=timeZone
          , yLabelPrefix='Latency'
          , yLabelSuffix=' (ms)'
          , openDetailsSelector='.vllm_details'
          , datasetLabel='Average (${result.metric.model_name})'
          , metricLabels=['model_name', 'pod']
          , displayLabels=['model', 'pod']
        ) }}
        {{ hubClusterProjectDashboardAsyncFunction(
          divId='visualization-llm-token-throughput'
          , classSimpleName='Project'
          , legendTitle='Token Throughput'
          , metricPrefix='label_replace(rate(vllm:prompt_tokens_total{'
          , metricMiddle='}[10m]), "token_type", "Prompt Tokens/Sec", "namespace", "(.+)") or label_replace(rate(vllm:generation_tokens_total{'
          , metricSuffix='}[10m]), "token_type", "Generation Tokens/Sec", "namespace", "(.+)")'
          , modelVar='projectName'
          , timeZone=timeZone
          , yLabelPrefix='Tokens per Second'
          , datasetLabel='${result.metric.token_type} (${result.metric.model_name})'
          , metricLabels=['token_type', 'pod']
          , displayLabels=['token type', 'pod']
        ) }}
        {{ hubClusterProjectDashboardAsyncFunction(
          divId='visualization-llm-ttft-latency'
          , classSimpleName='Project'
          , legendTitle='Time To First Token'
          , metricPrefix='rate(vllm:time_to_first_token_seconds_sum{'
          , metricMiddle='}[10m]) / rate(vllm:time_to_first_token_seconds_count{'
          , metricSuffix='}[10m])'
          , modelVar='projectName'
          , timeZone=timeZone
          , yLabelPrefix='Latency'
          , yLabelSuffix=' (ms)'
          , datasetLabel='Average TTFT (${result.metric.model_name})'
          , metricLabels=['model_name', 'pod']
          , displayLabels=['model', 'pod']
        ) }}
        {{ hubClusterProjectDashboardAsyncFunction(
          divId='visualization-llm-tpot-latency'
          , classSimpleName='Project'
          , legendTitle='Time Per Output Token'
          , metricPrefix='rate(vllm:time_per_output_token_seconds_sum{'
          , metricMiddle='}[10m]) / rate(vllm:time_per_output_token_seconds_count{'
          , metricSuffix='}[10m])'
          , modelVar='projectName'
          , timeZone=timeZone
          , yLabelPrefix='Latency'
          , yLabelSuffix=' (ms)'
          , datasetLabel='Mean TPOT (${result.metric.model_name})'
          , metricLabels=['model_name', 'pod']
          , displayLabels=['model', 'pod']
        ) }}

        async function queryStorageUtilization(hubId, clusterName, projectName) {
          // Query for all PVCs with their allocated storage (shows all PVCs regardless of mount status)
          var allocatedQuery = 'kube_persistentvolumeclaim_resource_requests_storage_bytes{namespace="' + projectName + '"}';

          // Query for storage used per mounted PVC
          var usedQuery = 'kubelet_volume_stats_used_bytes{cluster="' + clusterName + '", namespace="' + projectName + '"}';

          // Query for namespace storage quota (matches any storage-related quota resource)
          var quotaQuery = 'kube_resourcequota{namespace="' + projectName + '", resource=~".*requests.storage.*"}';

          var allocatedUrl = '/prom-keycloak-proxy/' + encodeURIComponent(hubId) + '/api/v1/query?' + new URLSearchParams({
            query: allocatedQuery
          }).toString();

          var usedUrl = '/prom-keycloak-proxy/' + encodeURIComponent(hubId) + '/api/v1/query?' + new URLSearchParams({
            query: usedQuery
          }).toString();

          var quotaUrl = '/prom-keycloak-proxy/' + encodeURIComponent(hubId) + '/api/v1/query?' + new URLSearchParams({
            query: quotaQuery
          }).toString();

          // Update UI elements
          var $table = document.getElementById('storage-utilization-table');
          var $tbody = document.getElementById('storage-utilization-tbody');
          var $allocatedTotal = document.getElementById('storage-allocated-total');
          var $usedTotal = document.getElementById('storage-used-total');
          var $percentTotal = document.getElementById('storage-percent-total');
          var $empty = document.getElementById('storage-utilization-empty');
          var $allocatedQueryDisplay = document.getElementById('storage-allocated-query-display');
          var $usedQueryDisplay = document.getElementById('storage-used-query-display');
          var $quotaQueryDisplay = document.getElementById('storage-quota-query-display');
          var $quotaSection = document.getElementById('storage-quota-section');
          var $usedText = document.getElementById('storage-used-text');
          var $allocatedText = document.getElementById('storage-allocated-text');

          if (!$table || !$tbody || !$empty) {
            return;
          }

          // Reset state
          $table.style.display = 'none';
          $empty.style.display = 'none';
          if ($quotaSection) $quotaSection.style.display = 'none';

          // Display query info
          if ($quotaQueryDisplay) {
            $quotaQueryDisplay.innerText = quotaQuery;
          }
          if ($allocatedQueryDisplay) {
            $allocatedQueryDisplay.innerText = allocatedQuery;
          }
          if ($usedQueryDisplay) {
            $usedQueryDisplay.innerText = usedQuery;
          }

          try {
            var [allocatedResponse, usedResponse, quotaResponse] = await Promise.all([fetch(allocatedUrl), fetch(usedUrl), fetch(quotaUrl)]);

            if (!allocatedResponse.ok || !usedResponse.ok) {
              $empty.style.display = 'block';
              $empty.innerText = 'HTTP Error fetching storage metrics';
              return;
            }

            var [allocatedJson, usedJson, quotaJson] = await Promise.all([allocatedResponse.json(), usedResponse.json(), quotaResponse.json()]);

            if (allocatedJson.status === 'error') {
              $empty.style.display = 'block';
              $empty.innerText = 'Prometheus error: ' + (allocatedJson.error || 'Unknown error');
              return;
            }

            // Build a map of PVC -> used bytes (only for mounted PVCs) and calculate total used
            var usedMap = {};
            var totalActualUsed = 0;
            if (usedJson.data && usedJson.data.result) {
              usedJson.data.result.forEach((result) => {
                var pvc = result.metric.persistentvolumeclaim || 'unknown';
                if (result.value && result.value.length > 1) {
                  var usedBytes = parseFloat(result.value[1]) || 0;
                  usedMap[pvc] = usedBytes;
                  totalActualUsed += usedBytes;
                }
              });
            }

            // Process quota data - show both used and allocated vs quota limit
            if ($quotaSection) {
              if (quotaJson.status === 'error') {
                console.warn('Quota query error:', quotaJson.error);
              } else if (quotaJson.data && quotaJson.data.result && quotaJson.data.result.length > 0) {
                var quotaHard = 0;
                var quotaAllocated = 0;

                quotaJson.data.result.forEach((result) => {
                  var type = result.metric.type;
                  var value = 0;
                  if (result.value && result.value.length > 1) {
                    value = parseFloat(result.value[1]) || 0;
                  }
                  if (type === 'hard') {
                    quotaHard = value;
                  } else if (type === 'used') {
                    quotaAllocated = value;
                  }
                });

                if (quotaHard > 0) {
                  // Storage used (actual bytes used)
                  if ($usedText) {
                    var usedPercent = (totalActualUsed / quotaHard) * 100;
                    $usedText.innerText = formatBytes(totalActualUsed) + ' / ' + formatBytes(quotaHard) + ' (' + usedPercent.toFixed(1) + '%)';
                  }

                  // Storage allocated (PVC requests)
                  if ($allocatedText) {
                    var allocatedPercent = (quotaAllocated / quotaHard) * 100;
                    $allocatedText.innerText = formatBytes(quotaAllocated) + ' / ' + formatBytes(quotaHard) + ' (' + allocatedPercent.toFixed(1) + '%)';
                  }

                  $quotaSection.style.display = 'block';
                } else {
                  console.log('Quota query returned data but quotaHard is 0. Results:', quotaJson.data.result);
                }
              } else {
                console.log('Quota query returned no results for:', quotaQuery);
              }
            }

            if (allocatedJson.data && allocatedJson.data.result && allocatedJson.data.result.length > 0) {
              $tbody.innerHTML = '';
              var totalAllocated = 0;
              var totalUsed = 0;

              allocatedJson.data.result.forEach((result) => {
                var pvc = result.metric.persistentvolumeclaim || 'unknown';

                var allocatedBytes = 0;
                if (result.value && result.value.length > 1) {
                  allocatedBytes = parseFloat(result.value[1]) || 0;
                }

                var isMounted = usedMap.hasOwnProperty(pvc);
                var usedBytes = isMounted ? usedMap[pvc] : 0;
                var usagePercent = allocatedBytes > 0 && isMounted ? (usedBytes / allocatedBytes * 100) : 0;

                totalAllocated += allocatedBytes;
                totalUsed += usedBytes;

                var $tr = document.createElement('tr');
                $tbody.append($tr);

                var $tdPvc = document.createElement('td');
                $tdPvc.innerText = pvc;
                $tr.append($tdPvc);

                var $tdStatus = document.createElement('td');
                $tdStatus.style.textAlign = 'center';
                $tdStatus.innerText = isMounted ? 'Mounted' : 'Unmounted';
                $tr.append($tdStatus);

                var $tdAllocated = document.createElement('td');
                $tdAllocated.innerText = formatBytes(allocatedBytes);
                $tdAllocated.style.textAlign = 'right';
                $tr.append($tdAllocated);

                var $tdUsed = document.createElement('td');
                $tdUsed.innerText = isMounted ? formatBytes(usedBytes) : '-';
                $tdUsed.style.textAlign = 'right';
                $tr.append($tdUsed);

                var $tdPercent = document.createElement('td');
                $tdPercent.style.textAlign = 'right';
                if (isMounted) {
                  $tdPercent.innerText = usagePercent.toFixed(1) + '%';
                  if (usagePercent >= 90) {
                    $tdPercent.style.color = 'var(--wa-color-danger-600)';
                    $tdPercent.style.fontWeight = 'bold';
                  } else if (usagePercent >= 75) {
                    $tdPercent.style.color = 'var(--wa-color-warning-600)';
                  }
                } else {
                  $tdPercent.innerText = '-';
                }
                $tr.append($tdPercent);
              });

              var totalPercent = totalAllocated > 0 ? (totalUsed / totalAllocated * 100) : 0;
              if ($allocatedTotal) $allocatedTotal.innerText = formatBytes(totalAllocated);
              if ($usedTotal) $usedTotal.innerText = formatBytes(totalUsed);
              if ($percentTotal) $percentTotal.innerText = totalPercent.toFixed(1) + '%';

              $table.style.display = 'table';
            } else {
              $empty.style.display = 'block';
            }
          } catch (error) {
            $empty.style.display = 'block';
            $empty.innerText = 'Error loading storage data: ' + error.message;
          }
        }

        async function queryGpuBillingHours(hubId, clusterName, projectName, timeQuery) {
          // Calculate the duration for the subquery
          var startTime = new Date(timeQuery.start).getTime();
          var endTime = new Date(timeQuery.end).getTime();
          var durationSeconds = Math.floor((endTime - startTime) / 1000);
          var durationString = durationSeconds + 's';

          // Build the query to calculate GPU hours with GPU model info:
          // 1. Get pods requesting nvidia GPUs (excluding unschedulable)
          // 2. Join with kube_node_labels to get the GPU product name
          // 3. sum_over_time adds up all values at 1-minute intervals
          // 4. Divide by 60 to convert minutes to hours
          var podQuery = '(kube_pod_resource_request{resource=~"nvidia.com.*", node!="", cluster="' + clusterName + '", namespace="' + projectName + '"} unless on(pod, namespace) kube_pod_status_unschedulable{cluster="' + clusterName + '", namespace="' + projectName + '"})';
          var joinQuery = podQuery + ' * on(node) group_left(label_nvidia_com_gpu_product) (max by (node, label_nvidia_com_gpu_product) (kube_node_labels{cluster="' + clusterName + '", label_nvidia_com_gpu_product!=""}))';
          var query = 'sum_over_time((' + joinQuery + ')[' + durationString + ':1m]) / 60';

          // Use instant query API (not query_range) since sum_over_time returns a single value
          var url = '/prom-keycloak-proxy/' + encodeURIComponent(hubId) + '/api/v1/query?' + new URLSearchParams({
            query: query
            , time: timeQuery.end
          }).toString();

          // Update UI elements
          var $table = document.getElementById('billing-gpu-hours-table');
          var $tbody = document.getElementById('billing-gpu-hours-tbody');
          var $total = document.getElementById('billing-gpu-hours-total');
          var $empty = document.getElementById('billing-gpu-hours-empty');
          var $queryDisplay = document.getElementById('billing-query-display');
          var $rangeInfo = document.getElementById('billing-range-info');

          if (!$table || !$tbody || !$total || !$empty) {
            return;
          }

          // Reset state
          $table.style.display = 'none';
          $empty.style.display = 'none';

          // Display query info
          if ($queryDisplay) {
            $queryDisplay.innerText = query;
          }
          if ($rangeInfo) {
            $rangeInfo.innerHTML = '';
            var rangeItems = [
              { label: 'Start', value: moment.tz(timeQuery.start, timeQuery.timeZone).format('YYYY-MM-DD HH:mm:ss') }
              , { label: 'End', value: moment.tz(timeQuery.end, timeQuery.timeZone).format('YYYY-MM-DD HH:mm:ss') }
              , { label: 'Time Zone', value: timeQuery.timeZone }
            ];
            rangeItems.forEach((item) => {
              var $tr = document.createElement('tr');
              $rangeInfo.append($tr);
              var $th = document.createElement('th');
              $th.innerText = item.label;
              $th.style.textAlign = 'left';
              $th.style.paddingRight = '2rem';
              $tr.append($th);
              var $td = document.createElement('td');
              $td.innerText = item.value;
              $tr.append($td);
            });
          }

          try {
            var response = await fetch(url);

            if (!response.ok) {
              $empty.style.display = 'block';
              $empty.innerText = 'HTTP Error: ' + response.status + ' ' + response.statusText;
              return;
            }

            var responseJson = await response.json();

            if (responseJson.status === 'error') {
              $empty.style.display = 'block';
              $empty.innerText = 'Prometheus error: ' + (responseJson.error || 'Unknown error');
              return;
            }

            if (responseJson.data && responseJson.data.result && responseJson.data.result.length > 0) {
              $tbody.innerHTML = '';
              var totalGpuHours = 0;

              responseJson.data.result.forEach((result) => {
                var gpuModel = result.metric.label_nvidia_com_gpu_product || result.metric.resource || 'unknown';
                var pod = result.metric.pod || 'unknown';
                var node = result.metric.node || 'unknown';

                var gpuHours = 0;
                if (result.value && result.value.length > 1) {
                  gpuHours = parseFloat(result.value[1]) || 0;
                }

                totalGpuHours += gpuHours;

                var $tr = document.createElement('tr');
                $tbody.append($tr);

                var $tdGpuModel = document.createElement('td');
                $tdGpuModel.innerText = gpuModel;
                $tr.append($tdGpuModel);

                var $tdPod = document.createElement('td');
                $tdPod.innerText = pod;
                $tr.append($tdPod);

                var $tdNode = document.createElement('td');
                $tdNode.innerText = node;
                $tr.append($tdNode);

                var $tdGpuHours = document.createElement('td');
                $tdGpuHours.innerText = gpuHours.toFixed(2);
                $tdGpuHours.style.textAlign = 'right';
                $tr.append($tdGpuHours);
              });

              $total.innerText = totalGpuHours.toFixed(2);
              $table.style.display = 'table';
            } else {
              $empty.style.display = 'block';
            }
          } catch (error) {
            $empty.style.display = 'block';
            $empty.innerText = 'Error loading GPU usage data: ' + error.message;
          }
        }

  </script>
{%- endblock htmScriptsProjectPage %}