[draft] experiments of using proto and add microbenchmarks

benchmark/microbench_test.go

@@ -0,0 +1,158 @@
+package benchmark_test
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"sort"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/jmoiron/sqlx"
+	"github.com/metal-toolbox/fleetdb/internal/config"
+	"github.com/metal-toolbox/fleetdb/internal/dbtools"
+	"github.com/metal-toolbox/fleetdb/internal/httpsrv"
+	fleetdbapi "github.com/metal-toolbox/fleetdb/pkg/api/v1"
+	"github.com/vattle/sqlboiler/boil"
+	"go.infratographer.com/x/crdbx"
+	"go.uber.org/zap"
+)
+
+// go test -benchmem -run=^$ -tags testtools -bench . github.com/metal-toolbox/fleetdb/benchmark -benchtime=30s
+/*
+BenchmarkGetComponentsJson
+p50 latency: 3.597459ms
+p90 latency: 6.332708ms
+p99 latency: 20.509166ms
+
+BenchmarkGetComponentsProto
+p50 latency: 3.574375ms
+p90 latency: 5.545833ms
+p99 latency: 17.993708ms
+*/
+
+func init() {
+	return
+
+	go func() {
+		sqldb, err := crdbx.NewDB(crdbx.Config{
+			URI: "postgresql://root@fleetdb-crdb:26257/defaultdb?sslmode=disable",
+		}, false)
+		if err != nil {
+			fmt.Errorf("failed to initialize database connection %v", err)
+		}
+
+		boil.SetDB(sqldb)
+
+		db := sqlx.NewDb(sqldb, "postgres")
+
+		logger, _ := zap.NewDevelopment()
+		hs := &httpsrv.Server{
+			Logger:        logger,
+			Listen:        "localhost:12345",
+			Debug:         config.AppConfig.Logging.Debug,
+			DB:            db,
+			OIDCEnabled:   false,
+			SecretsKeeper: nil,
+			AuthConfigs:   config.AppConfig.APIServerJWTAuth,
+		}
+
+		if err := hs.Run(); err != nil {
+			fmt.Errorf("failed starting server %v", err)
+		}
+	}()
+
+	client, err := fleetdbapi.NewClient("http://localhost:8000", nil)
+	if err != nil {
+		// b.Fatalf("failed to create client %v\n", err)
+	}
+	componentTypeSlice, _, err := client.ListServerComponentTypes(context.Background(), nil)
+	if err != nil {
+		// b.Fail()
+		fmt.Errorf("%v", err)
+	}
+
+	csFixtureCreate := fleetdbapi.ServerComponentSlice{
+		{
+			ServerUUID:        uuid.MustParse("224c776c-239a-4853-b4b8-29fe7942f8cf"),
+			Name:              "My Lucky Fin5",
+			Vendor:            "barracuda5",
+			Model:             "a lucky fin5",
+			Serial:            "right5",
+			ComponentTypeID:   componentTypeSlice.ByName("GPU").ID,
+			ComponentTypeName: componentTypeSlice.ByName("GPU").Name,
+			ComponentTypeSlug: componentTypeSlice.ByName("GPU").Slug,
+			VersionedAttributes: []fleetdbapi.VersionedAttributes{
+				{
+					Namespace: dbtools.FixtureNamespaceVersioned,
+					Data:      json.RawMessage(`{"version":"1.0"}`),
+				},
+				{
+					Namespace: dbtools.FixtureNamespaceVersioned,
+					Data:      json.RawMessage(`{"version":"2.0"}`),
+				},
+			},
+		},
+	}
+	res, err := client.CreateComponents(context.TODO(), uuid.MustParse("224c776c-239a-4853-b4b8-29fe7942f8cf"), csFixtureCreate)
+	if err != nil {
+		fmt.Printf("%v\n", err)
+	}
+	fmt.Printf("============== res = %v\n", res)
+}
+
+func printHistogram(times []time.Duration) {
+	// Set bin sizes (e.g., 10 ms intervals)
+	n := len(times)
+	sort.Slice(times, func(i, j int) bool {
+		return times[i] < times[j]
+	})
+	fmt.Printf("\np50 latency: %v\np90 latency: %v\np99 latency: %v\n", times[n/2], times[n/100*90], times[n/100*99])
+}
+
+func BenchmarkGetComponentsJson(b *testing.B) {
+	client, err := fleetdbapi.NewClient("http://localhost:8000", nil)
+	if err != nil {
+		b.Fatalf("failed to create client %v\n", err)
+	}
+
+	// resp, _, err := client.GetComponents(context.TODO(), uuid.MustParse("224c776c-239a-4853-b4b8-29fe7942f8cf"), nil)
+	// fmt.Printf("resp= %v\b", resp)
+
+	var times []time.Duration
+	for i := 0; i < b.N; i++ {
+		start := time.Now()
+		resp, _, err := client.GetComponents(context.TODO(), uuid.MustParse("224c776c-239a-4853-b4b8-29fe7942f8cf"), nil)
+		if err != nil {
+			b.Fatal()
+		}
+		_ = resp
+		duration := time.Since(start)
+		times = append(times, duration)
+	}
+	printHistogram(times)
+}
+
+func BenchmarkGetComponentsProto(b *testing.B) {
+	client, err := fleetdbapi.NewClient("http://localhost:8000", nil)
+	if err != nil {
+		b.Fatalf("failed to create client %v\n", err)
+	}
+
+	resp, _, err := client.GetComponentsProto(context.TODO(), uuid.MustParse("224c776c-239a-4853-b4b8-29fe7942f8cf"), nil)
+	fmt.Printf("resp= %v, err = %v\n", resp, err)
+
+	var times []time.Duration
+	for i := 0; i < b.N; i++ {
+		start := time.Now()
+		resp, _, err := client.GetComponentsProto(context.TODO(), uuid.MustParse("224c776c-239a-4853-b4b8-29fe7942f8cf"), nil)
+		if err != nil {
+			b.Fatal()
+		}
+		_ = resp
+		duration := time.Since(start)
+		times = append(times, duration)
+	}
+	printHistogram(times)
+}

go.mod

@@ -37,10 +37,12 @@ require (
 	github.com/hetiansu5/urlquery v1.2.7
 	github.com/metal-toolbox/bmc-common v1.0.2
 	github.com/metal-toolbox/rivets/v2 v2.0.0
+	github.com/vattle/sqlboiler v2.5.0+incompatible
 	github.com/volatiletech/sqlboiler v3.7.1+incompatible
 	go.infratographer.com/x v0.5.4
 	gocloud.dev v0.40.0
 	golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f
+	google.golang.org/protobuf v1.35.1
 	gopkg.in/go-jose/go-jose.v2 v2.6.3
 )
 
@@ -127,7 +129,6 @@ require (
 	google.golang.org/genproto/googleapis/api v0.0.0-20241104194629-dd2ea8efbc28 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 // indirect
 	google.golang.org/grpc v1.67.1 // indirect
-	google.golang.org/protobuf v1.35.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -674,6 +674,8 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS
 github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
 github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
 github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
+github.com/vattle/sqlboiler v2.5.0+incompatible h1:Wt7km7/dtfzZD8gt1u3NWeK0AuF5UEr26kVD360w2PA=
+github.com/vattle/sqlboiler v2.5.0+incompatible/go.mod h1:Q3YlQv8muqQLqYsCfq0TjX87Ec/oUbIcLuZFZIUV614=
 github.com/volatiletech/inflect v0.0.1 h1:2a6FcMQyhmPZcLa+uet3VJ8gLn/9svWhJxJYwvE8KsU=
 github.com/volatiletech/inflect v0.0.1/go.mod h1:IBti31tG6phkHitLlr5j7shC5SOo//x0AjDzaJU1PLA=
 github.com/volatiletech/null v8.0.0+incompatible h1:7wP8m5d/gZ6kW/9GnrLtMCRre2dlEnaQ9Km5OXlK4zg=

pkg/api/v1/client.go

@@ -8,6 +8,8 @@ import (
 	"net/url"
 	"os"
 	"strings"
+
+	"github.com/metal-toolbox/fleetdb/protogen/fleetservice"
 )
 
 var apiVersion = "v1"
@@ -164,6 +166,22 @@ func (c *Client) list(ctx context.Context, path string, params queryParams, resp
 	return c.do(request, &resp)
 }
 
+// list provides a reusable method for a standard list to a hollow server
+func (c *Client) listProto(ctx context.Context, path string, params queryParams, resp *fleetservice.GetComponentsResponse) error {
+	request, err := newGetRequest(ctx, c.url, path)
+	if err != nil {
+		return err
+	}
+
+	if params != nil {
+		q := request.URL.Query()
+		params.setQuery(q)
+		request.URL.RawQuery = q.Encode()
+	}
+
+	return c.doProto(request, resp)
+}
+
 // get provides a reusable method for a standard GET of a single item
 func (c *Client) get(ctx context.Context, path string, resp interface{}) error {
 	request, err := newGetRequest(ctx, c.url, path)

pkg/api/v1/requests.go

@@ -9,7 +9,9 @@ import (
 	"net/http"
 	"net/url"
 
+	"github.com/metal-toolbox/fleetdb/protogen/fleetservice"
 	"github.com/metal-toolbox/rivets/v2/version"
+	"google.golang.org/protobuf/proto"
 )
 
 func newGetRequest(ctx context.Context, uri, path string) (*http.Request, error) {
@@ -113,3 +115,43 @@ func (c *Client) do(req *http.Request, result interface{}) error {
 
 	return json.Unmarshal(data, result)
 }
+
+func (c *Client) doProto(req *http.Request, result *fleetservice.GetComponentsResponse) error {
+	req.Header.Set("Authorization", fmt.Sprintf("bearer %s", c.authToken))
+	req.Header.Set("User-Agent", userAgentString())
+
+	// dump request if c.dumper is set
+	if err := c.dumpRequest(req); err != nil {
+		return err
+	}
+
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return err
+	}
+
+	// dump response if c.dumper is set
+	if err := c.dumpResponse(resp); err != nil {
+		return err
+	}
+
+	if err := ensureValidServerResponse(resp); err != nil {
+		return err
+	}
+
+	defer resp.Body.Close()
+
+	switch resp.StatusCode {
+	case http.StatusNoContent, http.StatusResetContent:
+		// these statuses are not allowed to have body content
+		return nil
+	default:
+	}
+
+	data, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	return proto.Unmarshal(data, result)
+}

pkg/api/v1/router.go

@@ -53,6 +53,10 @@ func (r *Router) Routes(rg *gin.RouterGroup) {
 				srvAttrs.DELETE("/:namespace", amw.AuthRequired(deleteScopes("server", "server:attributes")), r.serverAttributesDelete)
 			}
 
+			srvComponentsProto := srv.Group("/components-proto")
+			{
+				srvComponentsProto.GET("", amw.AuthRequired(readScopes("server", "server:component")), r.serverComponentGetProto)
+			}
 			// /servers/:uuid/components
 			srvComponents := srv.Group("/components")
 			{

pkg/api/v1/router_server_components.go

@@ -2,6 +2,7 @@ package fleetdbapi
 
 import (
 	"database/sql"
+	"net/http"
 
 	"github.com/gin-gonic/gin"
 	"github.com/google/uuid"
@@ -11,6 +12,7 @@ import (
 	"github.com/volatiletech/sqlboiler/v4/queries/qm"
 
 	"github.com/metal-toolbox/fleetdb/internal/models"
+	"github.com/metal-toolbox/fleetdb/protogen/fleetservice"
 )
 
 var (
@@ -109,6 +111,54 @@ func (r *Router) serverComponentGet(c *gin.Context) {
 	listResponse(c, comps, pd)
 }
 
+// serverComponentGet returns a response with the list of components referenced by the server UUID.
+func (r *Router) serverComponentGetProto(c *gin.Context) {
+	srv, err := r.loadServerFromParams(c)
+	if err != nil {
+		if errors.Is(err, ErrUUIDParse) {
+			badRequestResponse(c, "", err)
+			return
+		}
+
+		dbErrorResponse(c, err)
+
+		return
+	}
+
+	// - include Attributes, VersionedAttributes and ServerComponentyType relations
+	mods := []qm.QueryMod{
+		qm.Load("Attributes"),
+		qm.Load("VersionedAttributes", qm.Where("(namespace, created_at, server_component_id) IN (select namespace, max(created_at), server_component_id from versioned_attributes group by namespace, server_component_id)")),
+		qm.Load("ServerComponentType"),
+	}
+
+	dbComps, err := srv.ServerComponents(mods...).All(c.Request.Context(), r.DB)
+	if err != nil {
+		dbErrorResponse(c, err)
+		return
+	}
+
+	var cc []*fleetservice.Components
+	for _, comp := range dbComps {
+		cc = append(cc, &fleetservice.Components{
+			Id:                    comp.ID,
+			Name:                  comp.Name.String,
+			Vendor:                comp.Vendor.String,
+			Model:                 comp.Model.String,
+			Serial:                comp.Serial.String,
+			ServerComponentTypeId: comp.ServerComponentTypeID,
+			ServerId:              comp.ServerID,
+			// CreatedAt:             comp.CreatedAt.Time,
+			// UpdatedAt:             comp.UpdatedAt.Time,
+		})
+	}
+	rr := &fleetservice.GetComponentsResponse{
+		Components: cc,
+	}
+
+	c.ProtoBuf(http.StatusOK, rr)
+}
+
 // serverComponentsCreate stores a ServerComponentSlice object into the backend store.
 func (r *Router) serverComponentsCreate(c *gin.Context) {
 	// load server based on the UUID parameter

pkg/api/v1/server_service.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/google/uuid"
+	"github.com/metal-toolbox/fleetdb/protogen/fleetservice"
 	rivets "github.com/metal-toolbox/rivets/v2/types"
 )
 
@@ -43,6 +44,7 @@ type ClientInterface interface {
 	UpdateAttributes(ctx context.Context, u uuid.UUID, ns string, data json.RawMessage) (*ServerResponse, error)
 
 	GetComponents(context.Context, uuid.UUID, *PaginationParams) ([]ServerComponent, *ServerResponse, error)
+	GetComponentsProto(context.Context, uuid.UUID, *PaginationParams) ([]ServerComponent, *ServerResponse, error)
 	ListComponents(context.Context, *ServerComponentListParams) ([]ServerComponent, *ServerResponse, error)
 	CreateComponents(context.Context, uuid.UUID, ServerComponentSlice) (*ServerResponse, error)
 	UpdateComponents(context.Context, uuid.UUID, ServerComponentSlice) (*ServerResponse, error)
@@ -197,6 +199,18 @@ func (c *Client) GetComponents(ctx context.Context, srvUUID uuid.UUID, params *P
 	return *sc, &r, nil
 }
 
+func (c *Client) GetComponentsProto(ctx context.Context, srvUUID uuid.UUID, params *PaginationParams) (ServerComponentSlice, *fleetservice.GetComponentsResponse, error) {
+	sc := &ServerComponentSlice{}
+	r := fleetservice.GetComponentsResponse{}
+
+	path := fmt.Sprintf("%s/%s/components-proto", serversEndpoint, srvUUID)
+	if err := c.listProto(ctx, path, params, &r); err != nil {
+		return nil, nil, err
+	}
+
+	return *sc, &r, nil
+}
+
 // ListComponents will get all the components matching the given parameters
 func (c *Client) ListComponents(ctx context.Context, params *ServerComponentListParams) (ServerComponentSlice, *ServerResponse, error) {
 	sc := &ServerComponentSlice{}