Skip to content

supports binding vm to nlb server groups by setting nlb_server_groups #173

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
xiaozhu36 merged 1 commit into from
Sep 5, 2024
Merged

supports binding vm to nlb server groups by setting nlb_server_groups #173

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 43 additions & 29 deletions src/bosh-alicloud-cpi/action/create_vm.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ package action
import (
"bosh-alicloud-cpi/alicloud"
"bosh-alicloud-cpi/registry"
"strconv"

//"github.com/cloudfoundry-incubator/bosh-alicloud-cpi-release/src/bosh-alicloud-cpi/registry"
"encoding/json"
Expand Down Expand Up @@ -38,31 +39,32 @@ type InstanceProps struct {
SystemDisk DiskInfo `json:"system_disk"`
Tags map[string]string `json:"tags"`

Region string `json:"region"`
AvailabilityZone string `json:"availability_zone"`
InstanceName string `json:"instance_name"`
InstanceType string `json:"instance_type"`
SlbServerGroupWeight json.Number `json:"slb_server_group_weight"`
SlbServerGroupPort json.Number `json:"slb_server_group_port"`
SlbServerGroup []string `json:"slb_server_group"`
Slbs []string `json:"slbs"`
SlbWeight json.Number `json:"slb_weight"`
NlbServerGroupWeight json.Number `json:"nlb_server_group_weight"`
NlbServerGroupPort json.Number `json:"nlb_server_group_port"`
NlbServerGroupIds []string `json:"nlb_server_group_ids"`
NlbServerGroups []NlbServerGroupProps `json:"nlb_server_groups"`
Password string `json:"password"`
KeyPairName string `json:"key_pair_name"`
SecurityGroupIds []string `json:"security_group_ids"`
ChargeType string `json:"charge_type"`
ChargePeriod json.Number `json:"charge_period"`
ChargePeriodUnit string `json:"charge_period_unit"`
AutoRenew string `json:"auto_renew"`
AutoRenewPeriod json.Number `json:"auto_renew_period"`
SpotStrategy alicloud.SpotStrategyType `json:"spot_strategy"`
SpotPriceLimit float64 `json:"spot_price_limit"`
RamRoleName string `json:"ram_role_name"`
StemcellId string `json:"stemcell_id"`
Region string `json:"region"`
AvailabilityZone string `json:"availability_zone"`
InstanceName string `json:"instance_name"`
InstanceType string `json:"instance_type"`
SlbServerGroupWeight json.Number `json:"slb_server_group_weight"`
SlbServerGroupPort json.Number `json:"slb_server_group_port"`
SlbServerGroup []string `json:"slb_server_group"`
Slbs []string `json:"slbs"`
SlbWeight json.Number `json:"slb_weight"`
// Deprecated: NlbServerGroupWeight, NlbServerGroupPort, NlbServerGroupIds has been deprecated
NlbServerGroupWeight json.Number `json:"nlb_server_group_weight"`
NlbServerGroupPort json.Number `json:"nlb_server_group_port"`
NlbServerGroupIds []string `json:"nlb_server_group_ids"`
NlbServerGroups []alicloud.NlbServerGroupProps `json:"nlb_server_groups"`
Password string `json:"password"`
KeyPairName string `json:"key_pair_name"`
SecurityGroupIds []string `json:"security_group_ids"`
ChargeType string `json:"charge_type"`
ChargePeriod json.Number `json:"charge_period"`
ChargePeriodUnit string `json:"charge_period_unit"`
AutoRenew string `json:"auto_renew"`
AutoRenewPeriod json.Number `json:"auto_renew_period"`
SpotStrategy alicloud.SpotStrategyType `json:"spot_strategy"`
SpotPriceLimit float64 `json:"spot_price_limit"`
RamRoleName string `json:"ram_role_name"`
StemcellId string `json:"stemcell_id"`
}

type CreateVMMethod struct {
Expand Down Expand Up @@ -429,7 +431,15 @@ func (a CreateVMMethod) updateInstance(instCid string, associatedDiskCIDs []apiv
return bosherr.WrapErrorf(err, "bind %s to slbServerGroup %s failed,weight:%d,port:%d ", instCid, slbServerGroup, slbServerGroupWeight, slbServerGroupPort)
}
}
if len(instProps.NlbServerGroupIds) > 0 {
nlbGroups := make(map[string]alicloud.NlbServerGroupProps)
if len(instProps.NlbServerGroups) > 0 {
for _, group := range instProps.NlbServerGroups {
nlbGroups[group.ServerGroupId] = group
}
if err := a.networks.BindNlbServerGroups(instProps.Region, instCid, nlbGroups); err != nil {
return bosherr.WrapErrorf(err, "bind %s to nlbServerGroup failed.", instCid)
}
} else if len(instProps.NlbServerGroupIds) > 0 {
nlbServerGroupPort, err := instProps.NlbServerGroupPort.Int64()
if err != nil {
return bosherr.WrapErrorf(err, "invalid nlb_server_group_port: '%v'. Error", instProps.NlbServerGroupPort)
Expand All @@ -439,11 +449,15 @@ func (a CreateVMMethod) updateInstance(instCid string, associatedDiskCIDs []apiv
return bosherr.WrapErrorf(err, "invalid nlb_server_group_weight: '%v'. Error", instProps.NlbServerGroupWeight)
}
for _, nlbServerGroup := range instProps.NlbServerGroupIds {
err := a.networks.BindNlbServerGroup(instProps.Region, instCid, nlbServerGroup, int(nlbServerGroupWeight), int(nlbServerGroupPort))
if err != nil {
return bosherr.WrapErrorf(err, "bind %s to nlbServerGroup %s failed, weight: %d, port: %d.", instCid, nlbServerGroup, nlbServerGroupWeight, nlbServerGroupPort)
nlbGroups[nlbServerGroup] = alicloud.NlbServerGroupProps{
Port: json.Number(strconv.FormatInt(nlbServerGroupPort, 10)),
ServerGroupId: nlbServerGroup,
Weight: json.Number(strconv.FormatInt(nlbServerGroupWeight, 10)),
}
}
if err := a.networks.BindNlbServerGroups(instProps.Region, instCid, nlbGroups); err != nil {
return bosherr.WrapErrorf(err, "bind %s to nlbServerGroup failed.", instCid)
}
}
return nil
}
Expand Down
4 changes: 4 additions & 0 deletions src/bosh-alicloud-cpi/action/delete_vm.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,10 @@ func (a DeleteVMMethod) DeleteVM(cid apiv1.VMCID) error {
return nil
}

if err := a.instances.RemoveFromNlbServerGroups(instCid); err != nil {
a.Logger.Error("RemoveFromNlbServerGroups", "remove %s from nlbServerGroup failed. Error: %v", instCid, err)
}

eniIds := a.instances.GetAttachedNetworkInterfaceIds(instCid)

err = a.instances.ChangeInstanceStatus(instCid, alicloud.Deleted, func(status alicloud.InstanceStatus) (bool, error) {
Expand Down
5 changes: 0 additions & 5 deletions src/bosh-alicloud-cpi/action/networks.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,6 @@ type NetworkProps struct {
VSwitchId string `json:"vswitch_id"`
InternetChargeType string `json:"internet_charge_type,omitempty"`
}
type NlbServerGroupProps struct {
ServerGroupId string `json:"server_group_id"`
Port []string `json:"port"`
Weight string `json:"weight"`
}

func NewNetworks(args apiv1.Networks) (Networks, error) {
r := Networks{networks: args}
Expand Down
7 changes: 4 additions & 3 deletions src/bosh-alicloud-cpi/alicloud/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,15 @@ import (
"bosh-alicloud-cpi/registry"
"encoding/json"
"fmt"
openapi "github.com/alibabacloud-go/darabonba-openapi/v2/client"
"github.com/alibabacloud-go/tea/tea"
credential "github.com/aliyun/credentials-go/credentials"
"os"
"strings"
"sync"
"time"

openapi "github.com/alibabacloud-go/darabonba-openapi/v2/client"
"github.com/alibabacloud-go/tea/tea"
credential "github.com/aliyun/credentials-go/credentials"

bosherr "github.com/cloudfoundry/bosh-utils/errors"
boshlog "github.com/cloudfoundry/bosh-utils/logger"
boshsys "github.com/cloudfoundry/bosh-utils/system"
Expand Down
3 changes: 2 additions & 1 deletion src/bosh-alicloud-cpi/alicloud/disk_manager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,11 @@
package alicloud

import (
"strings"

openapi "github.com/alibabacloud-go/darabonba-openapi/v2/client"
openapiutil "github.com/alibabacloud-go/openapi-util/service"
"github.com/alibabacloud-go/tea/tea"
"strings"

util "github.com/alibabacloud-go/tea-utils/v2/service"
"github.com/aliyun/alibaba-cloud-sdk-go/services/ecs"
Expand Down
91 changes: 89 additions & 2 deletions src/bosh-alicloud-cpi/alicloud/instance_manager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,13 @@ package alicloud
import (
"encoding/json"
"fmt"
"strconv"
"strings"
"time"

openapi "github.com/alibabacloud-go/darabonba-openapi/v2/client"
openapiutil "github.com/alibabacloud-go/openapi-util/service"
"github.com/alibabacloud-go/tea/tea"
"strings"
"time"

"github.com/aliyun/alibaba-cloud-sdk-go/services/ecs"

Expand Down Expand Up @@ -56,6 +58,7 @@ type InstanceManager interface {
// Cleanup the left network interfaces
GetAttachedNetworkInterfaceIds(cid string) []string
CleanupInstanceNetworkInterfaces(cid string, eniIds []string) error
RemoveFromNlbServerGroups(cid string) error
}

type InstanceManagerImpl struct {
Expand Down Expand Up @@ -367,6 +370,90 @@ func (a InstanceManagerImpl) CleanupInstanceNetworkInterfaces(cid string, eniIds
return nil
}

func (a InstanceManagerImpl) RemoveFromNlbServerGroups(cid string) error {
conn, err := a.config.NlbTeaClient(a.config.OpenApi.Region)
if err != nil {
return err
}
params := &openapi.Params{
Version: tea.String("2022-04-30"),
Protocol: tea.String("HTTPS"),
Method: tea.String("POST"),
AuthType: tea.String("AK"),
Style: tea.String("RPC"),
Pathname: tea.String("/"),
ReqBodyType: tea.String("formData"),
BodyType: tea.String("json"),
}
runtime := &util.RuntimeOptions{}
runtime.SetAutoretry(true)
invoker := NewInvoker()
invoker.AddCatcher(NlbBindServerCatcher_Conflict_Lock)

action := "ListServerGroupServers"
params.Action = tea.String(action)
body := map[string]interface{}{
"ServerIds.1": tea.String(cid),
"RegionId": tea.String(a.config.OpenApi.Region),
"MaxResults": tea.Int64(100),
}
request := &openapi.OpenApiRequest{
Body: body,
}

removed := make(map[string]int)
err = invoker.Run(func() error {
resp, e := conn.CallApi(params, request, runtime)
if e != nil {
a.logger.Error("InstanceManager", "%s %s failed %v. Retry...", action, cid, err)
return e
}
if resp != nil && resp["body"].(map[string]interface{})["Servers"] != nil {
for _, item := range resp["body"].(map[string]interface{})["Servers"].([]interface{}) {
server := item.(map[string]interface{})
if server["ServerGroupId"] != nil && fmt.Sprint(server["ServerGroupId"]) != "" {
if server["Port"] == nil {
removed[fmt.Sprint(server["ServerGroupId"])] = 0
} else {
port, err := strconv.Atoi(fmt.Sprint(server["Port"]))
if err != nil {
a.logger.Error("InstanceManager", "%s convert server port %s to int failed %v. Retry...", cid, server["Port"], err)
port = 0
}
removed[fmt.Sprint(server["ServerGroupId"])] = port
}
}
}
}
return nil
})

// remove servers
action = "RemoveServersFromServerGroup"
params.Action = tea.String(action)
for serverGroupId, port := range removed {
body = map[string]interface{}{
"Servers.1.ServerId": tea.String(cid),
"Servers.1.ServerType": tea.String("Ecs"),
"Servers.1.Port": tea.Int(port),
"ServerGroupId": tea.String(serverGroupId),
"RegionId": tea.String(a.config.OpenApi.Region),
"ClientToken": buildClientToken(action),
}
request = &openapi.OpenApiRequest{
Body: body,
}
err = invoker.Run(func() error {
_, e := conn.CallApi(params, request, runtime)
if e != nil {
a.logger.Error("NetworkManager", "%s %s failed %v. Retry...", action, cid, err)
}
return e
})
}
return err
}

func (a InstanceManagerImpl) GetInstanceUserData() {
//client := a.config.NewEcsClient()
//client.AddTags()
Expand Down
64 changes: 39 additions & 25 deletions src/bosh-alicloud-cpi/alicloud/network_manager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,11 @@ package alicloud
import (
"encoding/json"
"fmt"
"strings"

openapi "github.com/alibabacloud-go/darabonba-openapi/v2/client"
openapiutil "github.com/alibabacloud-go/openapi-util/service"
util "github.com/alibabacloud-go/tea-utils/v2/service"
"github.com/alibabacloud-go/tea/tea"
"strings"

"github.com/aliyun/alibaba-cloud-sdk-go/sdk/errors"
"github.com/aliyun/alibaba-cloud-sdk-go/services/ecs"
Expand All @@ -26,7 +26,7 @@ type NetworkManager interface {

BindSLB(region, instanceId, slbId string, weight int) error
BindSlbServerGroup(region, instanceId, slbId string, weight int, port int) error
BindNlbServerGroup(region, instanceId, nlbServerGroupId string, weight int, port int) error
BindNlbServerGroups(region, instanceId string, nlbServerGroups map[string]NlbServerGroupProps) error
DescribeSecurityGroupAttribute(region, groupId string) (ecs.DescribeSecurityGroupAttributeResponse, error)
JoinSecurityGroup(region, instanceId, groupId string) error
}
Expand All @@ -46,6 +46,12 @@ type SlbServerGroupBackendServerType struct {
Port int
}

type NlbServerGroupProps struct {
ServerGroupId string `json:"server_group_id"`
Port json.Number `json:"port"`
Weight json.Number `json:"weight"`
}

func NewNetworkManager(config Config, logger boshlog.Logger) NetworkManager {
return NetworkManagerImpl{
config: config,
Expand Down Expand Up @@ -174,7 +180,7 @@ func (a NetworkManagerImpl) BindSlbServerGroup(region, instanceId string, slbSer
return err
}

func (a NetworkManagerImpl) BindNlbServerGroup(region, instanceId string, nlbServerGroupId string, weight int, port int) error {
func (a NetworkManagerImpl) BindNlbServerGroups(region, instanceId string, nlbServerGroups map[string]NlbServerGroupProps) error {
conn, err := a.config.NlbTeaClient(region)
if err != nil {
return err
Expand All @@ -191,33 +197,41 @@ func (a NetworkManagerImpl) BindNlbServerGroup(region, instanceId string, nlbSer
ReqBodyType: tea.String("formData"),
BodyType: tea.String("json"),
}
queries := map[string]interface{}{
"Servers.1.Port": port,
"ServerGroupId": nlbServerGroupId,
"Servers.1.ServerId": instanceId,
"Servers.1.ServerType": "Ecs",
"ClientToken": buildClientToken("AddServersToServerGroup"),
"RegionId": tea.String(a.config.OpenApi.Region),
}
if weight != 0 {
queries["Servers.1.Weight"] = weight
}
queries["ClientToken"] = buildClientToken(action)
request := &openapi.OpenApiRequest{
Query: openapiutil.Query(queries),
}
runtime := &util.RuntimeOptions{}
runtime.SetAutoretry(true)
invoker := NewInvoker()
invoker.AddCatcher(NlbBindServerCatcher_Conflict_Lock)

err = invoker.Run(func() error {
_, e := conn.CallApi(params, request, runtime)
if e != nil {
a.logger.Error("NetworkManager", "BindNlbServerGroup %s to %s failed %v. Retry...", instanceId, nlbServerGroupId, err)
for serverGroupId, serverGroup := range nlbServerGroups {
nlbServerGroupPort, err := serverGroup.Port.Int64()
if err != nil {
return bosherr.WrapErrorf(err, "invalid nlb_server_groups.port: '%v'. Error", serverGroup.Port)
}
return e
})
nlbServerGroupWeight, err := serverGroup.Weight.Int64()
if err != nil {
return bosherr.WrapErrorf(err, "invalid nlb_server_groups.weight: '%v'. Error", serverGroup.Weight)
}
body := map[string]interface{}{
"Servers.1.ServerId": tea.String(instanceId),
"Servers.1.ServerType": tea.String("Ecs"),
"Servers.1.Port": int(nlbServerGroupPort),
"Servers.1.Weight": int(nlbServerGroupWeight),
"ServerGroupId": tea.String(serverGroupId),
"RegionId": tea.String(a.config.OpenApi.Region),
"ClientToken": buildClientToken(action),
}

request := &openapi.OpenApiRequest{
Body: body,
}
err = invoker.Run(func() error {
_, e := conn.CallApi(params, request, runtime)
if e != nil {
a.logger.Error("NetworkManager", "%s %s failed %v. Retry...", action, instanceId, err)
}
return e
})
}
return err
}

Expand Down