Update index.Readable interface to support retreiving new doc.Document

Author: nbroyles

src/dbnode/storage/index/read_through_segment.go

@@ -262,13 +262,23 @@ func (s *readThroughSegmentReader) AllDocs() (index.IDDocIterator, error) {
 	return s.reader.AllDocs()
 }
 
+// Metadata is a pass through call, since there's no postings list to cache.
+func (s *readThroughSegmentReader) Metadata(id postings.ID) (doc.Metadata, error) {
+	return s.reader.Metadata(id)
+}
+
+// MetadataIterator is a pass through call, since there's no postings list to cache.
+func (s *readThroughSegmentReader) MetadataIterator(pl postings.List) (doc.MetadataIterator, error) {
+	return s.reader.MetadataIterator(pl)
+}
+
 // Doc is a pass through call, since there's no postings list to cache.
-func (s *readThroughSegmentReader) Doc(id postings.ID) (doc.Metadata, error) {
+func (s *readThroughSegmentReader) Doc(id postings.ID) (doc.Document, error) {
 	return s.reader.Doc(id)
 }
 
 // Docs is a pass through call, since there's no postings list to cache.
-func (s *readThroughSegmentReader) Docs(pl postings.List) (doc.MetadataIterator, error) {
+func (s *readThroughSegmentReader) Docs(pl postings.List) (doc.Iterator, error) {
 	return s.reader.Docs(pl)
 }
 

src/m3ninx/doc/types.go

@@ -39,3 +39,23 @@ type MetadataIterator interface {
 	// Close releases any internal resources used by the iterator.
 	Close() error
 }
+
+// Iterator provides an iterator over a collection of documents. It is NOT
+// safe for multiple goroutines to invoke methods on an Iterator simultaneously.
+type Iterator interface {
+	// Next returns a bool indicating if the iterator has any more documents
+	// to return.
+	Next() bool
+
+	// Current returns the current document. It is only safe to call Current immediately
+	// after a call to Next confirms there are more elements remaining. The Document
+	// returned from Current is only valid until the following call to Next(). Callers
+	// should copy the Document if they need it live longer.
+	Current() Document
+
+	// Err returns any errors encountered during iteration.
+	Err() error
+
+	// Close releases any internal resources used by the iterator.
+	Close() error
+}

src/m3ninx/index/index_mock.go

@@ -0,0 +1,79 @@
+// Copyright (c) 2021  Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package index
+
+import (
+	"github.com/m3db/m3/src/m3ninx/doc"
+	"github.com/m3db/m3/src/m3ninx/postings"
+)
+
+type documentIterator struct {
+	retriever    DocRetriever
+	postingsIter postings.Iterator
+
+	currDoc doc.Document
+	currID  postings.ID
+	closed  bool
+	err     error
+}
+
+// NewIterator returns a new Iterator
+func NewIterator(r DocRetriever, pi postings.Iterator) doc.Iterator {
+	return &documentIterator{
+		retriever:    r,
+		postingsIter: pi,
+	}
+}
+
+func (e *documentIterator) Next() bool {
+	if e.closed || e.err != nil || !e.postingsIter.Next() {
+		return false
+	}
+	id := e.postingsIter.Current()
+	e.currID = id
+
+	d, err := e.retriever.Doc(id)
+	if err != nil {
+		e.err = err
+		return false
+	}
+	e.currDoc = d
+	return true
+}
+
+func (e *documentIterator) Current() doc.Document {
+	return e.currDoc
+}
+
+func (e *documentIterator) Err() error {
+	return e.err
+}
+
+func (e *documentIterator) Close() error {
+	if e.closed {
+		return errIteratorClosed
+	}
+	e.closed = true
+	e.currDoc = doc.Document{}
+	e.currID = postings.ID(0)
+	err := e.postingsIter.Close()
+	return err
+}

src/m3ninx/index/iterator.go

@@ -0,0 +1,112 @@
+// Copyright (c) 2021  Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package index
+
+import (
+	"testing"
+
+	"github.com/m3db/m3/src/m3ninx/doc"
+	"github.com/m3db/m3/src/m3ninx/index/segment/fst/encoding"
+	"github.com/m3db/m3/src/m3ninx/postings"
+	xtest "github.com/m3db/m3/src/x/test"
+
+	"github.com/golang/mock/gomock"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDocIterator(t *testing.T) {
+	mockCtrl := xtest.NewController(t)
+	defer mockCtrl.Finish()
+
+	docs := []doc.Metadata{
+		{
+			ID: []byte("doc-id-1"),
+			Fields: []doc.Field{
+				{
+					Name:  []byte("apple"),
+					Value: []byte("red"),
+				},
+			},
+		},
+		{
+			ID: []byte("doc-id-2"),
+			Fields: []doc.Field{
+				{
+					Name:  []byte("banana"),
+					Value: []byte("yellow"),
+				},
+			},
+		},
+	}
+
+	encodedDocsWithIds := make([]docsWithIDs, 0, len(docs))
+	for i, d := range docs {
+		encodedDocsWithIds = append(encodedDocsWithIds, docsWithIDs{
+			id: postings.ID(i),
+			doc: doc.NewDocumentFromEncoded(
+				doc.Encoded{Bytes: docToBytes(d)}),
+		})
+	}
+
+	retriever := NewMockDocRetriever(mockCtrl)
+	gomock.InOrder(
+		retriever.EXPECT().Doc(encodedDocsWithIds[0].id).Return(encodedDocsWithIds[0].doc, nil),
+		retriever.EXPECT().Doc(encodedDocsWithIds[1].id).Return(encodedDocsWithIds[1].doc, nil),
+	)
+
+	postingsIter := postings.NewMockIterator(mockCtrl)
+	gomock.InOrder(
+		postingsIter.EXPECT().Next().Return(true),
+		postingsIter.EXPECT().Current().Return(encodedDocsWithIds[0].id),
+		postingsIter.EXPECT().Next().Return(true),
+		postingsIter.EXPECT().Current().Return(encodedDocsWithIds[1].id),
+		postingsIter.EXPECT().Next().Return(false),
+		postingsIter.EXPECT().Close().Return(nil),
+	)
+
+	it := NewIterator(retriever, postingsIter)
+
+	require.True(t, it.Next())
+	require.Equal(t, encodedDocsWithIds[0].doc, it.Current())
+	require.True(t, it.Next())
+	require.Equal(t, encodedDocsWithIds[1].doc, it.Current())
+	require.False(t, it.Next())
+	require.NoError(t, it.Err())
+
+	require.NoError(t, it.Close())
+}
+
+type docsWithIDs struct {
+	id  postings.ID
+	doc doc.Document
+}
+
+func docToBytes(d doc.Metadata) []byte {
+	enc := encoding.NewEncoder(1024)
+	n := enc.PutBytes(d.ID)
+	n += enc.PutUvarint(uint64(len(d.Fields)))
+	for _, f := range d.Fields { // nolint:gocritic
+		n += enc.PutBytes(f.Name)
+		n += enc.PutBytes(f.Value)
+	}
+
+	return enc.Bytes()
+}

src/m3ninx/index/iterator_test.go

@@ -30,7 +30,7 @@ import (
 var errIteratorClosed = errors.New("iterator has been closed")
 
 type idDocIterator struct {
-	retriever    DocRetriever
+	retriever    MetadataRetriever
 	postingsIter postings.Iterator
 
 	currDoc doc.Metadata
@@ -40,7 +40,7 @@ type idDocIterator struct {
 }
 
 // NewIDDocIterator returns a new NewIDDocIterator.
-func NewIDDocIterator(r DocRetriever, pi postings.Iterator) IDDocIterator {
+func NewIDDocIterator(r MetadataRetriever, pi postings.Iterator) IDDocIterator {
 	return &idDocIterator{
 		retriever:    r,
 		postingsIter: pi,
@@ -54,7 +54,7 @@ func (it *idDocIterator) Next() bool {
 	id := it.postingsIter.Current()
 	it.currID = id
 
-	d, err := it.retriever.Doc(id)
+	d, err := it.retriever.Metadata(id)
 	if err != nil {
 		it.err = err
 		return false