Merge pull request #23 from splunk/mw/te-183-pagination

TE-183 pagination

Author: michaelw

.circleci/config.yml

@@ -24,7 +24,10 @@ jobs:
           command: make lint
       - run:
           name: Wait for Splunk Container
-          command: curl -4sSk --retry 40 --retry-connrefused --retry-delay 3 -o /dev/null ${SPLUNK_ADDR}
+          command: |
+            curl -4sSk --retry 40 --retry-connrefused --retry-delay 3 -o /dev/null ${SPLUNK_ADDR}
+            sleep 5
+            curl -4sSk --retry 40 --retry-connrefused --retry-delay 3 -o /dev/null ${SPLUNK_ADDR}
       - run:
           name: Test
           command: make test TESTREPORT=test-results/go/results.xml

clients/splunk/deployment.go

@@ -11,9 +11,21 @@ func newDeploymentService(client *Client) *DeploymentService {
 	}
 }
 
-// GetSearchPeers returns information about all search peers
-func (d *DeploymentService) GetSearchPeers() ([]ServerInfoEntry, *Response, error) {
-	info := make([]ServerInfoEntry, 0)
-	resp, err := Receive(d.client.New().Get("search/distributed/peers"), &info)
+var (
+	ServerInfoEntryFilterDefault *PaginationFilter
+
+	ServerInfoEntryFilterMinimal *PaginationFilter = &PaginationFilter{
+		Filter: []string{"host", "host_fqdn", "server_roles"},
+	}
+)
+
+// SearchPeers returns information about all search peers
+func (d *DeploymentService) SearchPeers(filter *PaginationFilter) ([]ServerInfoEntry, *Response, error) {
+	var info []ServerInfoEntry
+	sling := d.client.New().Get("search/distributed/peers")
+	if filter != ServerInfoEntryFilterDefault {
+		sling = sling.QueryStruct(filter)
+	}
+	resp, err := Receive(sling, &info)
 	return info, resp, err
 }

clients/splunk/splunk.go

@@ -7,11 +7,26 @@ import (
 	"time"
 )
 
+// API https://docs.splunk.com/Documentation/Splunk/8.0.2/RESTREF/RESTprolog#Request_and_response_details
+type PaginationFilter struct {
+	Count     int      `url:"count,omitempty"` // NOTE: we omit zero value, since this is already set in outputMode
+	Filter    []string `url:"f,omitempty"`
+	Offset    int      `url:"offset,omitempty"`
+	Search    string   `url:"search,omitempty"`
+	SortDir   string   `url:"sort_dir,omitempty"`
+	SortKey   string   `url:"sort_key,omitempty"`
+	SortMode  string   `url:"sort_mode,omitempty"`
+	Summarize bool     `url:"summarize,omitempty"`
+}
+
 type outputMode struct {
-	Mode string `url:"output_mode"`
+	Mode string `url:"output_mode,omitempty"`
+	// by default, we do not want any pagination,
+	// override with PaginationFilter
+	Count int `url:"count"`
 }
 
-var jsonOutputMode = outputMode{"json"}
+var jsonOutputMode = outputMode{"json", 0}
 
 // API https://docs.splunk.com/Documentation/Splunk/latest/RESTREF/RESTprolog
 type API struct {

conn.go

@@ -151,9 +151,12 @@ func (config *splunkConfig) newConnection(ctx context.Context) (*splunk.API, err
 	if err != nil {
 		return nil, err
 	}
+
 	tr := &http.Transport{
 		TLSClientConfig: tlsConfig,
+		Proxy:           http.ProxyFromEnvironment,
 	}
+
 	// client is the underlying transport for API calls, including Login (for obtaining session token)
 	client := &http.Client{
 		Transport: tr,

path_creds_create.go

@@ -3,6 +3,7 @@ package splunk
 import (
 	"context"
 	"fmt"
+	"strings"
 
 	"github.com/hashicorp/errwrap"
 	uuid "github.com/hashicorp/go-uuid"
@@ -127,7 +128,7 @@ func (b *backend) credsReadHandlerStandalone(ctx context.Context, req *logical.R
 func findNode(nodeFQDN string, hosts []splunk.ServerInfoEntry, roleConfig *roleConfig) (bool, error) {
 	for _, host := range hosts {
 		// check if node_fqdn is in either of HostFQDN or Host. User might not always the FQDN on the cli input
-		if host.Content.HostFQDN == nodeFQDN || host.Content.Host == nodeFQDN {
+		if strings.EqualFold(host.Content.HostFQDN, nodeFQDN) || strings.EqualFold(host.Content.Host, nodeFQDN) {
 			// Return true if the requested node type is allowed
 			if strutil.StrListContains(roleConfig.AllowedServerRoles, "*") {
 				return true, nil
@@ -174,11 +175,12 @@ func (b *backend) credsReadHandlerMulti(ctx context.Context, req *logical.Reques
 		return nil, err
 	}
 
-	nodes, _, err := conn.Deployment.GetSearchPeers()
+	nodes, _, err := conn.Deployment.SearchPeers(splunk.ServerInfoEntryFilterMinimal)
 	if err != nil {
-		b.Logger().Error("Error while reading SearchPeers from cluster master", err)
+		b.Logger().Error("Error while reading SearchPeers from cluster master", "err", err)
 		return nil, errwrap.Wrapf("unable to read searchpeers from cluster master: {{err}}", err)
 	}
+
 	_, err = findNode(nodeFQDN, nodes, role)
 	if err != nil {
 		return nil, err
@@ -242,10 +244,10 @@ func (b *backend) credsReadHandler(ctx context.Context, req *logical.Request, d
 	node_fqdn, present := d.GetOk("node_fqdn")
 	// if node_fqdn is specified then the treat the request for a multi-node deployment
 	if present {
-		b.Logger().Debug(fmt.Sprintf("node_fqdn: [%s] specified for role: [%s]. using clustered mode getting temporary creds", node_fqdn.(string), name))
+		b.Logger().Debug("node_fqdn specified for role. using clustered mode for getting temporary creds", "nodeFQDN", node_fqdn.(string), "role", name)
 		return b.credsReadHandlerMulti(ctx, req, d)
 	}
-	b.Logger().Debug(fmt.Sprintf("node_fqdn not specified for role: [%s]. using standalone mode getting temporary creds", name))
+	b.Logger().Debug("node_fqdn not specified for role. using standalone mode for getting temporary creds", "role", name)
 	return b.credsReadHandlerStandalone(ctx, req, d)
 }
 

path_creds_create_test.go

@@ -0,0 +1,144 @@
+package splunk
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"path/filepath"
+	"testing"
+
+	"gotest.tools/assert"
+
+	"github.com/splunk/vault-plugin-splunk/clients/splunk"
+)
+
+func Test_findNode(t *testing.T) {
+	nodes := make([]splunk.ServerInfoEntry, 0)
+
+	gp := filepath.Join("testdata", t.Name()+".json")
+	jsonResponseSearchDistributedPeers, err := ioutil.ReadFile(gp)
+	assert.NilError(t, err)
+
+	err = json.Unmarshal(jsonResponseSearchDistributedPeers, &nodes)
+	assert.NilError(t, err)
+
+	type args struct {
+		nodeFQDN   string
+		hosts      []splunk.ServerInfoEntry
+		roleConfig *roleConfig
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    bool
+		wantErr bool
+	}{
+		{
+			name: "server entry first",
+			args: args{
+				nodeFQDN: "idm-i-074b0895939212e99.foo.example.com",
+				hosts:    nodes,
+				roleConfig: &roleConfig{
+					AllowedServerRoles: []string{"*"},
+				},
+			},
+			want:    true,
+			wantErr: false,
+		},
+		{
+			name: "server entry last",
+			args: args{
+				nodeFQDN: "sh-i-0a12fdd509c2a2954.foo.example.com",
+				hosts:    nodes,
+				roleConfig: &roleConfig{
+					AllowedServerRoles: []string{"*"},
+				},
+			},
+			want:    true,
+			wantErr: false,
+		},
+		{
+			name: "server entry case insensitive",
+			args: args{
+				nodeFQDN: "SH-I-0A12FDD509C2A2954.FOO.EXAMPLE.COM",
+				hosts:    nodes,
+				roleConfig: &roleConfig{
+					AllowedServerRoles: []string{"*"},
+				},
+			},
+			want:    true,
+			wantErr: false,
+		},
+		{
+			name: "server entry short name",
+			args: args{
+				nodeFQDN: "SH-I-0A12FDD509C2A2954",
+				hosts:    nodes,
+				roleConfig: &roleConfig{
+					AllowedServerRoles: []string{"*"},
+				},
+			},
+			want:    true,
+			wantErr: false,
+		},
+		{
+			name: "server entry not found",
+			args: args{
+				nodeFQDN: "unknown-host",
+				hosts:    nodes,
+				roleConfig: &roleConfig{
+					AllowedServerRoles: []string{"*"},
+				},
+			},
+			want:    false,
+			wantErr: true,
+		},
+		{
+			name: "role match mismatch",
+			args: args{
+				nodeFQDN: "sh-i-0a12fdd509c2a2954.foo.example.com",
+				hosts:    nodes,
+				roleConfig: &roleConfig{
+					AllowedServerRoles: []string{"unknown-role"},
+				},
+			},
+			want:    false,
+			wantErr: true,
+		},
+		{
+			name: "role match first",
+			args: args{
+				nodeFQDN: "sh-i-0a12fdd509c2a2954.foo.example.com",
+				hosts:    nodes,
+				roleConfig: &roleConfig{
+					AllowedServerRoles: []string{"cluster_search_head"},
+				},
+			},
+			want:    true,
+			wantErr: false,
+		},
+		{
+			name: "role match last",
+			args: args{
+				nodeFQDN: "sh-i-0a12fdd509c2a2954.foo.example.com",
+				hosts:    nodes,
+				roleConfig: &roleConfig{
+					AllowedServerRoles: []string{"unknown_role", "kv_store"},
+				},
+			},
+			want:    true,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := findNode(tt.args.nodeFQDN, tt.args.hosts, tt.args.roleConfig)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("findNode() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if got != tt.want {
+				t.Errorf("findNode() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

secret_creds.go

@@ -109,7 +109,7 @@ func (b *backend) secretCredsRevokeHandler(ctx context.Context, req *logical.Req
 }
 
 func (b *backend) ensureNodeConnection(ctx context.Context, config *splunkConfig, nodeFQDN string) (*splunk.API, error) {
-	b.Logger().Debug(fmt.Sprintf("connection for node_fqdn: [%s]", nodeFQDN))
+	b.Logger().Debug("node connection", "nodeFQDN", nodeFQDN)
 	if nodeFQDN == "" {
 		return b.ensureConnection(ctx, config)
 	}