fix: handle TiKV metrics fetch errors (#6897) (#6898)

Author: ti-chi-bot

pkg/tikvapi/v1/client.go

@@ -70,12 +70,14 @@ func (c *tikvClient) GetLeaderCount() (int, error) {
 	apiURL := fmt.Sprintf("%s/%s", c.url, metricsPath)
 	transport := c.httpClient.Transport
 	mfChan := make(chan *dto.MetricFamily, metricChanSize)
+	errChan := make(chan error, 1)
 
-	var fetchErr error
 	go func() {
 		if err := prom2json.FetchMetricFamilies(apiURL, mfChan, transport); err != nil {
-			fetchErr = fmt.Errorf("fail to fetch metric families from %s, error: %w", apiURL, err)
+			errChan <- fmt.Errorf("fail to fetch metric families from %s, error: %w", apiURL, err)
+			return
 		}
+		errChan <- nil
 	}()
 
 	fms := []*prom2json.Family{}
@@ -93,7 +95,7 @@ func (c *tikvClient) GetLeaderCount() (int, error) {
 		}
 	}
 
-	if fetchErr != nil {
+	if fetchErr := <-errChan; fetchErr != nil {
 		return 0, fetchErr
 	}
 	return 0, fmt.Errorf("metric %s{type=\"%s\"} not found for %s", metricNameRegionCount, metricLabelNameLeaderCount, apiURL)

pkg/tikvapi/v1/client_test.go

@@ -45,3 +45,18 @@ tikv_raftstore_region_count{type="region"} 50
 	require.NoError(t, err)
 	assert.Equal(t, 20, count)
 }
+
+func TestTiKVClient_GetLeaderCountFetchError(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusInternalServerError)
+		_, err := w.Write([]byte("boom"))
+		assert.NoError(t, err)
+	}))
+	defer server.Close()
+
+	client := NewTiKVClient(server.URL, 5*time.Second, nil, false)
+	count, err := client.GetLeaderCount()
+	require.Error(t, err)
+	assert.Equal(t, 0, count)
+	assert.Contains(t, err.Error(), "fail to fetch metric families")
+}