added gzip encoding support for outgoing requests

.changeset/afraid-hats-remember.md

@@ -0,0 +1,5 @@
+---
+'grafana-infinity-datasource': minor
+---
+
+Added support for gzip compression for outgoing requests by default. Fixes [#1003](https://github.com/grafana/grafana-infinity-datasource/issues/1003)

pkg/infinity/client.go

@@ -2,6 +2,7 @@ package infinity
 
 import (
 	"bytes"
+	"compress/gzip"
 	"context"
 	"crypto/tls"
 	"crypto/x509"
@@ -239,7 +240,7 @@ func (client *Client) req(ctx context.Context, url string, body io.Reader, setti
 		// therefore any incoming error is considered downstream
 		return nil, res.StatusCode, duration, errorsource.DownstreamError(err, false)
 	}
-	bodyBytes, err := io.ReadAll(res.Body)
+	bodyBytes, err := getBodyBytes(res)
 	if err != nil {
 		logger.Debug("error reading response body", "url", url, "error", err.Error())
 		return nil, res.StatusCode, duration, errorsource.DownstreamError(err, false)
@@ -258,6 +259,18 @@ func (client *Client) req(ctx context.Context, url string, body io.Reader, setti
 	return string(bodyBytes), res.StatusCode, duration, err
 }
 
+func getBodyBytes(res *http.Response) ([]byte, error) {
+	if strings.EqualFold(res.Header.Get("Content-Encoding"), "gzip") {
+		reader, err := gzip.NewReader(res.Body)
+		if err != nil {
+			return nil, err
+		}
+		defer reader.Close()
+		return io.ReadAll(reader)
+	}
+	return io.ReadAll(res.Body)
+}
+
 // https://stackoverflow.com/questions/31398044/got-error-invalid-character-%C3%AF-looking-for-beginning-of-value-from-json-unmar
 func removeBOMContent(input []byte) []byte {
 	return bytes.TrimPrefix(input, []byte("\xef\xbb\xbf"))

pkg/infinity/headers.go

@@ -19,10 +19,11 @@ const (
 )
 
 const (
-	headerKeyAccept        = "Accept"
-	headerKeyContentType   = "Content-Type"
-	HeaderKeyAuthorization = "Authorization"
-	HeaderKeyIdToken       = "X-Id-Token"
+	headerKeyAccept         = "Accept"
+	headerKeyContentType    = "Content-Type"
+	headerKeyAcceptEncoding = "Accept-Encoding"
+	HeaderKeyAuthorization  = "Authorization"
+	HeaderKeyIdToken        = "X-Id-Token"
 )
 
 func ApplyAcceptHeader(query models.Query, settings models.InfinitySettings, req *http.Request, includeSect bool) *http.Request {
@@ -65,6 +66,11 @@ func ApplyContentTypeHeader(query models.Query, settings models.InfinitySettings
 	return req
 }
 
+func ApplyAcceptEncodingHeader(query models.Query, settings models.InfinitySettings, req *http.Request, includeSect bool) *http.Request {
+	req.Header.Set(headerKeyAcceptEncoding, "gzip")
+	return req
+}
+
 func ApplyHeadersFromSettings(settings models.InfinitySettings, req *http.Request, includeSect bool) *http.Request {
 	for key, value := range settings.CustomHeaders {
 		val := dummyHeader
@@ -156,4 +162,4 @@ func getQueryReqHeader(requestHeaders map[string]string, headerName string) stri
 	}
 
 	return ""
-}
+}

pkg/infinity/request.go

@@ -28,6 +28,7 @@ func GetRequest(ctx context.Context, settings models.InfinitySettings, body io.R
 	}
 	req = ApplyAcceptHeader(query, settings, req, includeSect)
 	req = ApplyContentTypeHeader(query, settings, req, includeSect)
+	req = ApplyAcceptEncodingHeader(query, settings, req, includeSect)
 	req = ApplyHeadersFromSettings(settings, req, includeSect)
 	req = ApplyHeadersFromQuery(query, settings, req, includeSect)
 	req = ApplyBasicAuth(settings, req, includeSect)

pkg/infinity/request_test.go

@@ -204,37 +204,37 @@ func TestClient_GetExecutedURL(t *testing.T) {
 		{
 			query:   models.Query{URL: "https://foo.com"},
 			url:     "https://foo.com",
-			command: "curl -X 'GET' 'https://foo.com'",
+			command: "curl -X 'GET' -H 'Accept-Encoding: gzip' 'https://foo.com'",
 		},
 		{
 			settings: models.InfinitySettings{UserName: "hello", Password: "world", BasicAuthEnabled: true},
 			query:    models.Query{URL: "https://foo.com"},
 			url:      "https://foo.com",
-			command:  "curl -X 'GET' -H 'Authorization: Basic xxxxxxxx' 'https://foo.com'",
+			command:  "curl -X 'GET' -H 'Accept-Encoding: gzip' -H 'Authorization: Basic xxxxxxxx' 'https://foo.com'",
 		},
 		{
 			settings: models.InfinitySettings{AuthenticationMethod: "bearerToken", BearerToken: "world2"},
 			query:    models.Query{URL: "https://foo.com"},
 			url:      "https://foo.com",
-			command:  "curl -X 'GET' -H 'Authorization: Bearer xxxxxxxx' 'https://foo.com'",
+			command:  "curl -X 'GET' -H 'Accept-Encoding: gzip' -H 'Authorization: Bearer xxxxxxxx' 'https://foo.com'",
 		},
 		{
 			settings: models.InfinitySettings{AuthenticationMethod: "apiKey", ApiKeyType: "header", ApiKeyKey: "hello", ApiKeyValue: "world"},
 			query:    models.Query{URL: "https://foo.com"},
 			url:      "https://foo.com",
-			command:  "curl -X 'GET' -H 'Hello: xxxxxxxx' 'https://foo.com'",
+			command:  "curl -X 'GET' -H 'Accept-Encoding: gzip' -H 'Hello: xxxxxxxx' 'https://foo.com'",
 		},
 		{
 			settings: models.InfinitySettings{ForwardOauthIdentity: true},
 			query:    models.Query{URL: "https://foo.com"},
 			url:      "https://foo.com",
-			command:  "curl -X 'GET' -H 'Authorization: xxxxxxxx' 'https://foo.com'",
+			command:  "curl -X 'GET' -H 'Accept-Encoding: gzip' -H 'Authorization: xxxxxxxx' 'https://foo.com'",
 		},
 		{
 			settings: models.InfinitySettings{CustomHeaders: map[string]string{"good": "bye"}, SecureQueryFields: map[string]string{"me": "too"}},
 			query:    models.Query{URL: "https://foo.com?something=${__qs.me}", Type: "json", URLOptions: models.URLOptions{Method: "POST", Body: "my request body with ${__qs.me} value", Headers: []models.URLOptionKeyValuePair{{Key: "hello", Value: "world"}}}},
 			url:      "https://foo.com?me=xxxxxxxx&something=xxxxxxxx",
-			command:  "curl -X 'POST' -d 'my request body with ${__qs.me} value' -H 'Accept: application/json;q=0.9,text/plain' -H 'Content-Type: application/json' -H 'Good: xxxxxxxx' -H 'Hello: xxxxxxxx' 'https://foo.com?me=xxxxxxxx&something=xxxxxxxx'",
+			command:  "curl -X 'POST' -d 'my request body with ${__qs.me} value' -H 'Accept: application/json;q=0.9,text/plain' -H 'Accept-Encoding: gzip' -H 'Content-Type: application/json' -H 'Good: xxxxxxxx' -H 'Hello: xxxxxxxx' 'https://foo.com?me=xxxxxxxx&something=xxxxxxxx'",
 		},
 	}
 	for _, tt := range tests {

pkg/testsuite/golden/csv_backend_url_default.jsonc

@@ -59,7 +59,7 @@
 //          "duration": 123,
 //          "error": ""
 //      },
-//      "executedQueryString": "###############\n## URL\n###############\n\nhttp://127.0.0.1:8080\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' -H 'Accept: text/csv; charset=utf-8' 'http://127.0.0.1:8080'"
+//      "executedQueryString": "###############\n## URL\n###############\n\nhttp://127.0.0.1:8080\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' -H 'Accept: text/csv; charset=utf-8' -H 'Accept-Encoding: gzip' 'http://127.0.0.1:8080'"
 //  }
 //  Name: response
 //  Dimensions: 2 Fields by 2 Rows
@@ -139,7 +139,7 @@
             "duration": 123,
             "error": ""
           },
-          "executedQueryString": "###############\n## URL\n###############\n\nhttp://127.0.0.1:8080\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' -H 'Accept: text/csv; charset=utf-8' 'http://127.0.0.1:8080'"
+          "executedQueryString": "###############\n## URL\n###############\n\nhttp://127.0.0.1:8080\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' -H 'Accept: text/csv; charset=utf-8' -H 'Accept-Encoding: gzip' 'http://127.0.0.1:8080'"
         },
         "fields": [
           {

pkg/testsuite/golden/csv_default_url_default.jsonc

@@ -59,7 +59,7 @@
 //          "duration": 123,
 //          "error": ""
 //      },
-//      "executedQueryString": "###############\n## URL\n###############\n\nhttp://127.0.0.1:8080\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' -H 'Accept: text/csv; charset=utf-8' 'http://127.0.0.1:8080'"
+//      "executedQueryString": "###############\n## URL\n###############\n\nhttp://127.0.0.1:8080\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' -H 'Accept: text/csv; charset=utf-8' -H 'Accept-Encoding: gzip' 'http://127.0.0.1:8080'"
 //  }
 //  Name: response
 //  Dimensions: 0 Fields by 0 Rows
@@ -133,7 +133,7 @@
             "duration": 123,
             "error": ""
           },
-          "executedQueryString": "###############\n## URL\n###############\n\nhttp://127.0.0.1:8080\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' -H 'Accept: text/csv; charset=utf-8' 'http://127.0.0.1:8080'"
+          "executedQueryString": "###############\n## URL\n###############\n\nhttp://127.0.0.1:8080\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' -H 'Accept: text/csv; charset=utf-8' -H 'Accept-Encoding: gzip' 'http://127.0.0.1:8080'"
         },
         "fields": []
       },

pkg/testsuite/golden/html_backend_url_default.jsonc

@@ -90,7 +90,7 @@
 //          "duration": 123,
 //          "error": ""
 //      },
-//      "executedQueryString": "###############\n## URL\n###############\n\nhttp://127.0.0.1:8080\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' 'http://127.0.0.1:8080'"
+//      "executedQueryString": "###############\n## URL\n###############\n\nhttp://127.0.0.1:8080\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' -H 'Accept-Encoding: gzip' 'http://127.0.0.1:8080'"
 //  }
 //  Name: response
 //  Dimensions: 5 Fields by 6 Rows
@@ -205,7 +205,7 @@
             "duration": 123,
             "error": ""
           },
-          "executedQueryString": "###############\n## URL\n###############\n\nhttp://127.0.0.1:8080\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' 'http://127.0.0.1:8080'"
+          "executedQueryString": "###############\n## URL\n###############\n\nhttp://127.0.0.1:8080\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' -H 'Accept-Encoding: gzip' 'http://127.0.0.1:8080'"
         },
         "fields": [
           {

pkg/testsuite/golden/html_default_url_default.jsonc

@@ -90,7 +90,7 @@
 //          "duration": 123,
 //          "error": ""
 //      },
-//      "executedQueryString": "###############\n## URL\n###############\n\nhttp://127.0.0.1:8080\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' 'http://127.0.0.1:8080'"
+//      "executedQueryString": "###############\n## URL\n###############\n\nhttp://127.0.0.1:8080\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' -H 'Accept-Encoding: gzip' 'http://127.0.0.1:8080'"
 //  }
 //  Name: response
 //  Dimensions: 0 Fields by 0 Rows
@@ -195,7 +195,7 @@
             "duration": 123,
             "error": ""
           },
-          "executedQueryString": "###############\n## URL\n###############\n\nhttp://127.0.0.1:8080\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' 'http://127.0.0.1:8080'"
+          "executedQueryString": "###############\n## URL\n###############\n\nhttp://127.0.0.1:8080\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' -H 'Accept-Encoding: gzip' 'http://127.0.0.1:8080'"
         },
         "fields": []
       },

pkg/testsuite/golden/json_default_url_default.jsonc

@@ -61,7 +61,7 @@
 //          "duration": 123,
 //          "error": ""
 //      },
-//      "executedQueryString": "###############\n## URL\n###############\n\nhttp://127.0.0.1:8080\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' -H 'Accept: application/json;q=0.9,text/plain' 'http://127.0.0.1:8080'"
+//      "executedQueryString": "###############\n## URL\n###############\n\nhttp://127.0.0.1:8080\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' -H 'Accept: application/json;q=0.9,text/plain' -H 'Accept-Encoding: gzip' 'http://127.0.0.1:8080'"
 //  }
 //  Name: response
 //  Dimensions: 0 Fields by 0 Rows
@@ -137,7 +137,7 @@
             "duration": 123,
             "error": ""
           },
-          "executedQueryString": "###############\n## URL\n###############\n\nhttp://127.0.0.1:8080\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' -H 'Accept: application/json;q=0.9,text/plain' 'http://127.0.0.1:8080'"
+          "executedQueryString": "###############\n## URL\n###############\n\nhttp://127.0.0.1:8080\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' -H 'Accept: application/json;q=0.9,text/plain' -H 'Accept-Encoding: gzip' 'http://127.0.0.1:8080'"
         },
         "fields": []
       },

pkg/testsuite/golden/remote/csv_query.jsonc

@@ -59,7 +59,7 @@
 //          "duration": 123,
 //          "error": ""
 //      },
-//      "executedQueryString": "###############\n## URL\n###############\n\nhttp://bar\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' -H 'Accept: text/csv; charset=utf-8' 'http://bar'"
+//      "executedQueryString": "###############\n## URL\n###############\n\nhttp://bar\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' -H 'Accept: text/csv; charset=utf-8' -H 'Accept-Encoding: gzip' 'http://bar'"
 //  }
 //  Name: q1
 //  Dimensions: 0 Fields by 0 Rows
@@ -133,7 +133,7 @@
             "duration": 123,
             "error": ""
           },
-          "executedQueryString": "###############\n## URL\n###############\n\nhttp://bar\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' -H 'Accept: text/csv; charset=utf-8' 'http://bar'"
+          "executedQueryString": "###############\n## URL\n###############\n\nhttp://bar\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' -H 'Accept: text/csv; charset=utf-8' -H 'Accept-Encoding: gzip' 'http://bar'"
         },
         "fields": []
       },

pkg/testsuite/golden/remote/groq_query.jsonc

@@ -59,7 +59,7 @@
 //          "duration": 123,
 //          "error": ""
 //      },
-//      "executedQueryString": "###############\n## URL\n###############\n\nhttp://foo\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' 'http://foo'\n###############\n## GROQ\n###############\n\n*{1,2,3}\n"
+//      "executedQueryString": "###############\n## URL\n###############\n\nhttp://foo\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' -H 'Accept-Encoding: gzip' 'http://foo'\n###############\n## GROQ\n###############\n\n*{1,2,3}\n"
 //  }
 //  Name: q1
 //  Dimensions: 0 Fields by 0 Rows
@@ -133,7 +133,7 @@
             "duration": 123,
             "error": ""
           },
-          "executedQueryString": "###############\n## URL\n###############\n\nhttp://foo\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' 'http://foo'\n###############\n## GROQ\n###############\n\n*{1,2,3}\n"
+          "executedQueryString": "###############\n## URL\n###############\n\nhttp://foo\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' -H 'Accept-Encoding: gzip' 'http://foo'\n###############\n## GROQ\n###############\n\n*{1,2,3}\n"
         },
         "fields": []
       },

pkg/testsuite/golden/remote/json_query.jsonc

@@ -63,7 +63,7 @@
 //          "duration": 123,
 //          "error": ""
 //      },
-//      "executedQueryString": "###############\n## URL\n###############\n\nhttp://foo\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' -H 'Accept: application/json;q=0.9,text/plain' 'http://foo'"
+//      "executedQueryString": "###############\n## URL\n###############\n\nhttp://foo\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' -H 'Accept: application/json;q=0.9,text/plain' -H 'Accept-Encoding: gzip' 'http://foo'"
 //  }
 //  Name: q1
 //  Dimensions: 0 Fields by 0 Rows
@@ -141,7 +141,7 @@
             "duration": 123,
             "error": ""
           },
-          "executedQueryString": "###############\n## URL\n###############\n\nhttp://foo\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' -H 'Accept: application/json;q=0.9,text/plain' 'http://foo'"
+          "executedQueryString": "###############\n## URL\n###############\n\nhttp://foo\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' -H 'Accept: application/json;q=0.9,text/plain' -H 'Accept-Encoding: gzip' 'http://foo'"
         },
         "fields": []
       },

pkg/testsuite/golden/remote/should_execute_default_query_without_error.jsonc

@@ -59,7 +59,7 @@
 //          "duration": 123,
 //          "error": ""
 //      },
-//      "executedQueryString": "###############\n## URL\n###############\n\nhttps://raw.githubusercontent.com/grafana/grafana-infinity-datasource/main/testdata/users.json\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' -H 'Accept: application/json;q=0.9,text/plain' 'https://raw.githubusercontent.com/grafana/grafana-infinity-datasource/main/testdata/users.json'"
+//      "executedQueryString": "###############\n## URL\n###############\n\nhttps://raw.githubusercontent.com/grafana/grafana-infinity-datasource/main/testdata/users.json\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' -H 'Accept: application/json;q=0.9,text/plain' -H 'Accept-Encoding: gzip' 'https://raw.githubusercontent.com/grafana/grafana-infinity-datasource/main/testdata/users.json'"
 //  }
 //  Name: response
 //  Dimensions: 0 Fields by 0 Rows
@@ -133,7 +133,7 @@
             "duration": 123,
             "error": ""
           },
-          "executedQueryString": "###############\n## URL\n###############\n\nhttps://raw.githubusercontent.com/grafana/grafana-infinity-datasource/main/testdata/users.json\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' -H 'Accept: application/json;q=0.9,text/plain' 'https://raw.githubusercontent.com/grafana/grafana-infinity-datasource/main/testdata/users.json'"
+          "executedQueryString": "###############\n## URL\n###############\n\nhttps://raw.githubusercontent.com/grafana/grafana-infinity-datasource/main/testdata/users.json\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' -H 'Accept: application/json;q=0.9,text/plain' -H 'Accept-Encoding: gzip' 'https://raw.githubusercontent.com/grafana/grafana-infinity-datasource/main/testdata/users.json'"
         },
         "fields": []
       },

pkg/testsuite/golden/remote/uql_query.jsonc

@@ -59,7 +59,7 @@
 //          "duration": 123,
 //          "error": ""
 //      },
-//      "executedQueryString": "###############\n## URL\n###############\n\nhttp://foo\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' 'http://foo'\n\n###############\n## UQL\n###############\n\nparse-json | count"
+//      "executedQueryString": "###############\n## URL\n###############\n\nhttp://foo\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' -H 'Accept-Encoding: gzip' 'http://foo'\n\n###############\n## UQL\n###############\n\nparse-json | count"
 //  }
 //  Name: q1
 //  Dimensions: 0 Fields by 0 Rows
@@ -133,7 +133,7 @@
             "duration": 123,
             "error": ""
           },
-          "executedQueryString": "###############\n## URL\n###############\n\nhttp://foo\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' 'http://foo'\n\n###############\n## UQL\n###############\n\nparse-json | count"
+          "executedQueryString": "###############\n## URL\n###############\n\nhttp://foo\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' -H 'Accept-Encoding: gzip' 'http://foo'\n\n###############\n## UQL\n###############\n\nparse-json | count"
         },
         "fields": []
       },

pkg/testsuite/golden/scenario_azure_cost_management.jsonc

@@ -268,7 +268,7 @@
 //          "duration": 123,
 //          "error": ""
 //      },
-//      "executedQueryString": "###############\n## URL\n###############\n\nhttp://127.0.0.1:8080\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' -H 'Accept: application/json;q=0.9,text/plain' 'http://127.0.0.1:8080'"
+//      "executedQueryString": "###############\n## URL\n###############\n\nhttp://127.0.0.1:8080\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' -H 'Accept: application/json;q=0.9,text/plain' -H 'Accept-Encoding: gzip' 'http://127.0.0.1:8080'"
 //  }
 //  Name: response
 //  Dimensions: 2 Fields by 23 Rows
@@ -565,7 +565,7 @@
             "duration": 123,
             "error": ""
           },
-          "executedQueryString": "###############\n## URL\n###############\n\nhttp://127.0.0.1:8080\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' -H 'Accept: application/json;q=0.9,text/plain' 'http://127.0.0.1:8080'"
+          "executedQueryString": "###############\n## URL\n###############\n\nhttp://127.0.0.1:8080\n\n###############\n## Curl Command\n###############\n\ncurl -X 'GET' -H 'Accept: application/json;q=0.9,text/plain' -H 'Accept-Encoding: gzip' 'http://127.0.0.1:8080'"
         },
         "fields": [
           {