Introduce size reservations for projects. (#484)

cmd/metal-api/internal/datastore/machine.go

@@ -428,12 +428,12 @@ func (rs *RethinkStore) UpdateMachine(oldMachine *metal.Machine, newMachine *met
 // FindWaitingMachine returns an available, not allocated, waiting and alive machine of given size within the given partition.
 // TODO: the algorithm can be optimized / shortened by using a rethinkdb join command and then using .Sample(1)
 // but current implementation should have a slightly better readability.
-func (rs *RethinkStore) FindWaitingMachine(projectid, partitionid, sizeid string, placementTags []string) (*metal.Machine, error) {
+func (rs *RethinkStore) FindWaitingMachine(projectid, partitionid string, size metal.Size, placementTags []string) (*metal.Machine, error) {
 	q := *rs.machineTable()
 	q = q.Filter(map[string]interface{}{
 		"allocation":  nil,
 		"partitionid": partitionid,
-		"sizeid":      sizeid,
+		"sizeid":      size.ID,
 		"state": map[string]string{
 			"value": string(metal.AvailableState),
 		},
@@ -467,21 +467,25 @@ func (rs *RethinkStore) FindWaitingMachine(projectid, partitionid, sizeid string
 		available = append(available, m)
 	}
 
-	if available == nil || len(available) < 1 {
+	if len(available) == 0 {
 		return nil, errors.New("no machine available")
 	}
 
-	query := MachineSearchQuery{
-		AllocationProject: &projectid,
-		PartitionID:       &partitionid,
-	}
-
-	var projectMachines metal.Machines
-	err = rs.SearchMachines(&query, &projectMachines)
+	var partitionMachines metal.Machines
+	err = rs.SearchMachines(&MachineSearchQuery{
+		PartitionID: &partitionid,
+	}, &partitionMachines)
 	if err != nil {
 		return nil, err
 	}
 
+	ok := checkSizeReservations(available, projectid, partitionid, partitionMachines.WithSize(size.ID).ByProjectID(), size)
+	if !ok {
+		return nil, errors.New("no machine available")
+	}
+
+	projectMachines := partitionMachines.ByProjectID()[projectid]
+
 	spreadCandidates := spreadAcrossRacks(available, projectMachines, placementTags)
 	if len(spreadCandidates) == 0 {
 		return nil, errors.New("no machine available")
@@ -499,6 +503,37 @@ func (rs *RethinkStore) FindWaitingMachine(projectid, partitionid, sizeid string
 	return &newMachine, nil
 }
 
+// checkSizeReservations returns true when an allocation is possible and
+// false when size reservations prevent the allocation for the given project in the given partition
+func checkSizeReservations(available metal.Machines, projectid, partitionid string, machinesByProject map[string]metal.Machines, size metal.Size) bool {
+	if size.Reservations == nil {
+		return true
+	}
+
+	var (
+		reservations = 0
+	)
+
+	for _, r := range size.Reservations.ForPartition(partitionid) {
+		r := r
+
+		// sum up the amount of reservations
+		reservations += r.Amount
+
+		alreadyAllocated := len(machinesByProject[r.ProjectID])
+
+		if projectid == r.ProjectID && alreadyAllocated < r.Amount {
+			// allow allocation for the project when it has a reservation and there are still allocations left
+			return true
+		}
+
+		// substract already used up reservations of the project
+		reservations = max(reservations-alreadyAllocated, 0)
+	}
+
+	return reservations < len(available)
+}
+
 func spreadAcrossRacks(allMachines, projectMachines metal.Machines, tags []string) metal.Machines {
 	var (
 		allRacks = groupByRack(allMachines)

cmd/metal-api/internal/datastore/machine_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/metal-stack/metal-api/cmd/metal-api/internal/metal"
 	"github.com/metal-stack/metal-api/cmd/metal-api/internal/testdata"
+	"github.com/stretchr/testify/require"
 	"golang.org/x/exp/slices"
 )
 
@@ -718,3 +719,151 @@ func getTestMachines(numPerRack int, rackids []string, tags []string) metal.Mach
 
 	return machines
 }
+
+func Test_checkSizeReservations(t *testing.T) {
+	var (
+		available = metal.Machines{
+			{Base: metal.Base{ID: "1"}},
+			{Base: metal.Base{ID: "2"}},
+			{Base: metal.Base{ID: "3"}},
+			{Base: metal.Base{ID: "4"}},
+			{Base: metal.Base{ID: "5"}},
+		}
+
+		partitionA = "a"
+		p0         = "0"
+		p1         = "1"
+		p2         = "2"
+
+		size = metal.Size{
+			Base: metal.Base{
+				ID: "c1-xlarge-x86",
+			},
+			Reservations: metal.Reservations{
+				{
+					Amount:       1,
+					ProjectID:    p1,
+					PartitionIDs: []string{partitionA},
+				},
+				{
+					Amount:       2,
+					ProjectID:    p2,
+					PartitionIDs: []string{partitionA},
+				},
+			},
+		}
+
+		projectMachines = map[string]metal.Machines{}
+
+		allocate = func(id, project string) {
+			available = slices.DeleteFunc(available, func(m metal.Machine) bool {
+				return m.ID == id
+			})
+			projectMachines[project] = append(projectMachines[project], metal.Machine{Base: metal.Base{ID: id}})
+		}
+	)
+
+	// 5 available, 3 reserved, project 0 can allocate
+	ok := checkSizeReservations(available, p0, partitionA, projectMachines, size)
+	require.True(t, ok)
+	allocate(available[0].ID, p0)
+
+	require.Equal(t, metal.Machines{
+		{Base: metal.Base{ID: "2"}},
+		{Base: metal.Base{ID: "3"}},
+		{Base: metal.Base{ID: "4"}},
+		{Base: metal.Base{ID: "5"}},
+	}, available)
+	require.Equal(t, map[string]metal.Machines{
+		p0: {
+			{Base: metal.Base{ID: "1"}},
+		},
+	}, projectMachines)
+
+	// 4 available, 3 reserved, project 2 can allocate
+	ok = checkSizeReservations(available, p2, partitionA, projectMachines, size)
+	require.True(t, ok)
+	allocate(available[0].ID, p2)
+
+	require.Equal(t, metal.Machines{
+		{Base: metal.Base{ID: "3"}},
+		{Base: metal.Base{ID: "4"}},
+		{Base: metal.Base{ID: "5"}},
+	}, available)
+	require.Equal(t, map[string]metal.Machines{
+		p0: {
+			{Base: metal.Base{ID: "1"}},
+		},
+		p2: {
+			{Base: metal.Base{ID: "2"}},
+		},
+	}, projectMachines)
+
+	// 3 available, 3 reserved (1 used), project 0 can allocate
+	ok = checkSizeReservations(available, p0, partitionA, projectMachines, size)
+	require.True(t, ok)
+	allocate(available[0].ID, p0)
+
+	require.Equal(t, metal.Machines{
+		{Base: metal.Base{ID: "4"}},
+		{Base: metal.Base{ID: "5"}},
+	}, available)
+	require.Equal(t, map[string]metal.Machines{
+		p0: {
+			{Base: metal.Base{ID: "1"}},
+			{Base: metal.Base{ID: "3"}},
+		},
+		p2: {
+			{Base: metal.Base{ID: "2"}},
+		},
+	}, projectMachines)
+
+	// 2 available, 3 reserved (1 used), project 0 cannot allocate anymore
+	ok = checkSizeReservations(available, p0, partitionA, projectMachines, size)
+	require.False(t, ok)
+
+	// 2 available, 3 reserved (1 used), project 2 can allocate
+	ok = checkSizeReservations(available, p2, partitionA, projectMachines, size)
+	require.True(t, ok)
+	allocate(available[0].ID, p2)
+
+	require.Equal(t, metal.Machines{
+		{Base: metal.Base{ID: "5"}},
+	}, available)
+	require.Equal(t, map[string]metal.Machines{
+		p0: {
+			{Base: metal.Base{ID: "1"}},
+			{Base: metal.Base{ID: "3"}},
+		},
+		p2: {
+			{Base: metal.Base{ID: "2"}},
+			{Base: metal.Base{ID: "4"}},
+		},
+	}, projectMachines)
+
+	// 1 available, 3 reserved (2 used), project 0 and 2 cannot allocate anymore
+	ok = checkSizeReservations(available, p0, partitionA, projectMachines, size)
+	require.False(t, ok)
+	ok = checkSizeReservations(available, p2, partitionA, projectMachines, size)
+	require.False(t, ok)
+
+	// 1 available, 3 reserved (2 used), project 1 can allocate
+	ok = checkSizeReservations(available, p1, partitionA, projectMachines, size)
+	require.True(t, ok)
+	allocate(available[0].ID, p1)
+
+	require.Equal(t, metal.Machines{}, available)
+	require.Equal(t, map[string]metal.Machines{
+		p0: {
+			{Base: metal.Base{ID: "1"}},
+			{Base: metal.Base{ID: "3"}},
+		},
+		p1: {
+			{Base: metal.Base{ID: "5"}},
+		},
+		p2: {
+			{Base: metal.Base{ID: "2"}},
+			{Base: metal.Base{ID: "4"}},
+		},
+	}, projectMachines)
+}

cmd/metal-api/internal/grpc/boot-service.go

@@ -133,7 +133,7 @@ func (b *BootService) Register(ctx context.Context, req *v1.BootServiceRegisterR
 
 	size, _, err := b.ds.FromHardware(machineHardware)
 	if err != nil {
-		size = metal.UnknownSize
+		size = metal.UnknownSize()
 		b.log.Errorw("no size found for hardware, defaulting to unknown size", "hardware", machineHardware, "error", err)
 	}
 

cmd/metal-api/internal/grpc/boot-service_test.go

@@ -90,7 +90,7 @@ func TestBootService_Register(t *testing.T) {
 			neighbormac2:   testdata.Switch2.Nics[0].MacAddress,
 			numcores:       2,
 			memory:         100,
-			expectedSizeId: metal.UnknownSize.ID,
+			expectedSizeId: metal.UnknownSize().ID,
 		},
 	}
 
@@ -109,7 +109,7 @@ func TestBootService_Register(t *testing.T) {
 					Conflict: "replace",
 				})).Return(testdata.EmptyResult, nil)
 			}
-			mock.On(r.DB("mockdb").Table("size").Get(metal.UnknownSize.ID)).Return([]metal.Size{*metal.UnknownSize}, nil)
+			mock.On(r.DB("mockdb").Table("size").Get(metal.UnknownSize().ID)).Return([]metal.Size{*metal.UnknownSize()}, nil)
 			mock.On(r.DB("mockdb").Table("switch").Filter(r.MockAnything(), r.FilterOpts{})).Return([]metal.Switch{testdata.Switch1, testdata.Switch2}, nil)
 			mock.On(r.DB("mockdb").Table("event").Filter(r.MockAnything(), r.FilterOpts{})).Return([]metal.ProvisioningEventContainer{}, nil)
 			mock.On(r.DB("mockdb").Table("event").Insert(r.MockAnything(), r.InsertOpts{})).Return(testdata.EmptyResult, nil)

cmd/metal-api/internal/metal/machine.go

@@ -171,6 +171,22 @@ func (ms Machines) ByProjectID() map[string]Machines {
 	return res
 }
 
+func (ms Machines) WithSize(id string) Machines {
+	var res Machines
+
+	for _, m := range ms {
+		m := m
+
+		if m.SizeID != id {
+			continue
+		}
+
+		res = append(res, m)
+	}
+
+	return res
+}
+
 // MachineNetwork stores the Network details of the machine
 type MachineNetwork struct {
 	NetworkID           string   `rethinkdb:"networkid" json:"networkid"`