Skip to content

Commit 9c9dc0d

Browse files
Implement bare-bones blob listing
1 parent 95c98af commit 9c9dc0d

File tree

4 files changed

+119
-12
lines changed

4 files changed

+119
-12
lines changed

gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileSystemProvider.java

Lines changed: 47 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,10 @@
2929
import com.google.common.annotations.VisibleForTesting;
3030
import com.google.common.base.MoreObjects;
3131
import com.google.common.base.Throwables;
32+
import com.google.common.collect.AbstractIterator;
3233
import com.google.common.primitives.Ints;
3334
import com.google.gcloud.storage.Acl;
35+
import com.google.gcloud.storage.Blob;
3436
import com.google.gcloud.storage.BlobId;
3537
import com.google.gcloud.storage.BlobInfo;
3638
import com.google.gcloud.storage.CopyWriter;
@@ -46,6 +48,7 @@
4648
import java.nio.file.AccessMode;
4749
import java.nio.file.AtomicMoveNotSupportedException;
4850
import java.nio.file.CopyOption;
51+
import java.nio.file.DirectoryIteratorException;
4952
import java.nio.file.DirectoryStream;
5053
import java.nio.file.DirectoryStream.Filter;
5154
import java.nio.file.FileAlreadyExistsException;
@@ -64,11 +67,11 @@
6467
import java.util.ArrayList;
6568
import java.util.Collections;
6669
import java.util.HashMap;
70+
import java.util.Iterator;
6771
import java.util.List;
6872
import java.util.Map;
6973
import java.util.Objects;
7074
import java.util.Set;
71-
7275
import javax.annotation.Nullable;
7376
import javax.annotation.concurrent.ThreadSafe;
7477

@@ -84,6 +87,33 @@ public final class CloudStorageFileSystemProvider extends FileSystemProvider {
8487
// used only when we create a new instance of CloudStorageFileSystemProvider.
8588
private static StorageOptions defaultStorageOptions;
8689

90+
private static class LazyPathIterator extends AbstractIterator<Path> {
91+
private final Iterator<Blob> blobIterator;
92+
private final Filter<? super Path> filter;
93+
private final CloudStorageFileSystem fileSystem;
94+
95+
LazyPathIterator(CloudStorageFileSystem fileSystem, Iterator<Blob> blobIterator, Filter<? super Path> filter) {
96+
this.blobIterator = blobIterator;
97+
this.filter = filter;
98+
this.fileSystem = fileSystem;
99+
}
100+
101+
@Override
102+
protected Path computeNext() {
103+
while (blobIterator.hasNext()) {
104+
Path path = fileSystem.getPath(blobIterator.next().name());
105+
try {
106+
if (filter.accept(path)) {
107+
return path;
108+
}
109+
} catch (IOException ex) {
110+
throw new DirectoryIteratorException(ex);
111+
}
112+
}
113+
return endOfData();
114+
}
115+
}
116+
87117
/**
88118
* Sets default options that are only used by the constructor.
89119
*/
@@ -532,13 +562,23 @@ public void createDirectory(Path dir, FileAttribute<?>... attrs) {
532562
checkNotNullArray(attrs);
533563
}
534564

535-
/**
536-
* Throws {@link UnsupportedOperationException} because this feature hasn't been implemented yet.
537-
*/
538565
@Override
539-
public DirectoryStream<Path> newDirectoryStream(Path dir, Filter<? super Path> filter) {
540-
// TODO: Implement me.
541-
throw new UnsupportedOperationException();
566+
public DirectoryStream<Path> newDirectoryStream(Path dir, final Filter<? super Path> filter) {
567+
final CloudStoragePath cloudPath = checkPath(dir);
568+
checkNotNull(filter);
569+
String prefix = cloudPath.toString();
570+
final Iterator<Blob> blobIterator = storage.list(cloudPath.bucket(), Storage.BlobListOption.prefix(prefix), Storage.BlobListOption.fields()).iterateAll();
571+
return new DirectoryStream<Path>() {
572+
@Override
573+
public Iterator<Path> iterator() {
574+
return new LazyPathIterator(cloudPath.getFileSystem(), blobIterator, filter);
575+
}
576+
577+
@Override
578+
public void close() throws IOException {
579+
// Does nothing since there's nothing to close. Commenting this method to quiet codacy.
580+
}
581+
};
542582
}
543583

544584
/**

gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileSystemTest.java

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,10 @@
3434
import java.nio.file.FileSystem;
3535
import java.nio.file.FileSystems;
3636
import java.nio.file.Files;
37+
import java.nio.file.Path;
3738
import java.nio.file.Paths;
39+
import java.util.ArrayList;
40+
import java.util.List;
3841

3942
/**
4043
* Unit tests for {@link CloudStorageFileSystem}.
@@ -134,4 +137,27 @@ public void testNullness() throws IOException, NoSuchMethodException, SecurityEx
134137
tester.testAllPublicInstanceMethods(fs);
135138
}
136139
}
140+
141+
@Test
142+
public void testListFiles() throws IOException {
143+
try (FileSystem fs = CloudStorageFileSystem.forBucket("bucket")) {
144+
List<Path> goodPaths = new ArrayList<>();
145+
List<Path> paths = new ArrayList<>();
146+
goodPaths.add(fs.getPath("dir/angel"));
147+
goodPaths.add(fs.getPath("dir/alone"));
148+
paths.add(fs.getPath("dir/dir2/another_angel"));
149+
paths.add(fs.getPath("atroot"));
150+
paths.addAll(goodPaths);
151+
goodPaths.add(fs.getPath("dir/dir2/"));
152+
for (Path path : paths) {
153+
Files.write(path, ALONE.getBytes(UTF_8));
154+
}
155+
156+
List<Path> got = new ArrayList<>();
157+
for (Path path : Files.newDirectoryStream(fs.getPath("/dir/"))) {
158+
got.add(path);
159+
}
160+
assertThat(got).containsExactlyElementsIn(goodPaths);
161+
}
162+
}
137163
}

gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageReadChannelTest.java

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,12 @@
2222
import static org.mockito.Mockito.mock;
2323
import static org.mockito.Mockito.times;
2424
import static org.mockito.Mockito.verify;
25-
import static org.mockito.Mockito.verifyNoMoreInteractions;
2625
import static org.mockito.Mockito.verifyZeroInteractions;
2726
import static org.mockito.Mockito.when;
2827

2928
import com.google.gcloud.ReadChannel;
3029
import com.google.gcloud.storage.Blob;
3130
import com.google.gcloud.storage.BlobId;
32-
import com.google.gcloud.storage.BlobInfo;
3331
import com.google.gcloud.storage.Storage;
3432

3533
import org.junit.Before;

gcloud-java-storage/src/main/java/com/google/gcloud/storage/testing/FakeStorageRpc.java

Lines changed: 46 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,10 @@
2626
import java.io.InputStream;
2727
import java.math.BigInteger;
2828
import java.nio.file.FileAlreadyExistsException;
29+
import java.util.ArrayList;
2930
import java.util.Arrays;
3031
import java.util.HashMap;
32+
import java.util.List;
3133
import java.util.Map;
3234

3335
import javax.annotation.concurrent.NotThreadSafe;
@@ -91,8 +93,49 @@ public Tuple<String, Iterable<Bucket>> list(Map<Option, ?> options) throws Stora
9193
@Override
9294
public Tuple<String, Iterable<StorageObject>> list(String bucket, Map<Option, ?> options)
9395
throws StorageException {
94-
potentiallyThrow(options);
95-
return null;
96+
String preprefix = "";
97+
for (Map.Entry<Option, ?> e : options.entrySet()) {
98+
switch (e.getKey()) {
99+
case PREFIX:
100+
preprefix = (String) e.getValue();
101+
if (preprefix.startsWith("/")) {
102+
preprefix = preprefix.substring(1);
103+
}
104+
break;
105+
case FIELDS:
106+
// ignore and return all the fields
107+
break;
108+
default:
109+
throw new UnsupportedOperationException("Unknown option: " + e.getKey());
110+
}
111+
}
112+
final String prefix = preprefix;
113+
114+
List<StorageObject> values = new ArrayList<>();
115+
Map<String, StorageObject> folders = new HashMap<>();
116+
for (StorageObject so : stuff.values()) {
117+
if (!so.getName().startsWith(prefix)) {
118+
continue;
119+
}
120+
int nextSlash = so.getName().indexOf("/", prefix.length());
121+
if (nextSlash >= 0) {
122+
String folderName = so.getName().substring(0, nextSlash + 1);
123+
if (folders.containsKey(folderName)) {
124+
continue;
125+
}
126+
StorageObject fakeFolder = new StorageObject();
127+
fakeFolder.setName(folderName);
128+
fakeFolder.setBucket(so.getBucket());
129+
fakeFolder.setGeneration(so.getGeneration());
130+
folders.put(folderName, fakeFolder);
131+
continue;
132+
}
133+
values.add(so);
134+
}
135+
values.addAll(folders.values());
136+
// null cursor to indicate there is no more data (empty string would cause us to be called again).
137+
// The type cast seems to be necessary to help Java's typesystem remember that collections are iterable.
138+
return Tuple.of(null, (Iterable<StorageObject>) values);
96139
}
97140

98141
/**
@@ -111,7 +154,7 @@ public Bucket get(Bucket bucket, Map<Option, ?> options) throws StorageException
111154
public StorageObject get(StorageObject object, Map<Option, ?> options) throws StorageException {
112155
// we allow the "ID" option because we need to, but then we give a whole answer anyways
113156
// because the caller won't mind the extra fields.
114-
if (throwIfOption && !options.isEmpty() && options.size()>1
157+
if (throwIfOption && !options.isEmpty() && options.size() > 1
115158
&& options.keySet().toArray()[0] != Storage.BlobGetOption.fields(Storage.BlobField.ID)) {
116159
throw new UnsupportedOperationException();
117160
}

0 commit comments

Comments
 (0)