mongodb · xuguang-li · Dec 23, 2020 · Nov 10, 2021 · Nov 10, 2021
diff --git a/build.gradle b/build.gradle
@@ -7,7 +7,7 @@ apply plugin: 'com.github.ben-manes.versions'
 
 ext.configDir = new File(rootDir, 'config')
 ext.hadoopBinaries = "${rootDir}/hadoop-binaries".toString()
-ext.javaDriverVersion = '3.2.1'
+ext.javaDriverVersion = '3.12.5'
 ext.hiveVersion = System.getenv("HIVE_VERSION") ?: '1.2.1'
 ext.pigVersion = System.getenv("PIG_VERSION") ?: '0.15.0'
 ext.hadoopVersion = System.getenv("HADOOP_VERSION") ?: '2.7.2'
@@ -58,7 +58,7 @@ if(project.hasProperty("clusterVersion")) {
 }
 
 allprojects {
-    version = '2.0.2'
+    version = '2.0.3'
     group = 'org.mongodb.mongo-hadoop'
 }
 
@@ -184,7 +184,7 @@ configure(subprojects) {
         options.links 'http://hadoop.apache.org/docs/r2.7.2/api'
         options.links 'http://api.mongodb.org/java/3.2/'
         if (JavaVersion.current().isJava8Compatible()) {
-          options.addStringOption('Xdoclint:none', '-quiet')
+            options.addStringOption('Xdoclint:none', '-quiet')
         }
     }
 
@@ -196,9 +196,9 @@ configure(subprojects) {
 //        tests testJar
 //    }
 
-    test {
-        dependsOn 'jar', 'testsJar', ':startCluster', ':downloadEnronEmails'
-    }
+//    test {
+    //       dependsOn 'jar', 'testsJar', ':startCluster', ':downloadEnronEmails'
+    //   }
 
     modifyPom {
         project {
@@ -505,4 +505,4 @@ task cleanLogs(type: Delete) {
 
 task cleanHadoop(type: Delete, dependsOn: cleanLogs) {
     delete hadoopHome, hiveHome, pigHome
-}
+}
diff --git a/core/src/main/java/com/mongodb/hadoop/input/MongoInputSplit.java b/core/src/main/java/com/mongodb/hadoop/input/MongoInputSplit.java
@@ -210,18 +210,18 @@ public void setSkip(final Integer skip) {
     @Override
     public void write(final DataOutput out) throws IOException {
         BSONObject spec = BasicDBObjectBuilder.start()
-          .add("inputURI", getInputURI().toString())
-          .add("authURI", getAuthURI() != null ? getAuthURI().toString() : null)
-          .add("keyField", getKeyField())
-          .add("fields", getFields())
-          .add("query", getQuery())
-          .add("sort", getSort())
-          .add("min", getMin())
-          .add("max", getMax())
-          .add("notimeout", getNoTimeout())
-          .add("limit", limit)
-          .add("skip", skip)
-          .get();
+                .add("inputURI", getInputURI().toString())
+                .add("authURI", getAuthURI() != null ? getAuthURI().toString() : null)
+                .add("keyField", getKeyField())
+                .add("fields", getFields())
+                .add("query", getQuery())
+                .add("sort", getSort())
+                .add("min", getMin())
+                .add("max", getMax())
+                .add("notimeout", getNoTimeout())
+                .add("limit", limit)
+                .add("skip", skip)
+                .get();
         byte[] buf = _bsonEncoder.encode(spec);
         out.write(buf);
     }
@@ -278,7 +278,7 @@ public DBCursor getCursor() {
                 coll = MongoConfigUtil.getCollection(this.inputURI);
             }
 
-            this.cursor = coll.find(this.query, this.fields).sort(this.sort);
+            this.cursor = coll.find(this.query, this.fields).hint(new BasicDBObject("_id", 1)).sort(this.sort);
             if (this.notimeout) {
                 this.cursor.setOptions(Bytes.QUERYOPTION_NOTIMEOUT);
             }
@@ -301,21 +301,21 @@ public DBCursor getCursor() {
     @Override
     public String toString() {
         String result =
-          "MongoInputSplit{inputURI hosts=" + this.inputURI.getHosts()
-            + ", inputURI namespace=" + this.inputURI.getDatabase() + "."
-            + this.inputURI.getCollection();
+                "MongoInputSplit{inputURI hosts=" + this.inputURI.getHosts()
+                        + ", inputURI namespace=" + this.inputURI.getDatabase() + "."
+                        + this.inputURI.getCollection();
         if (authURI != null) {
             result += "authURI hosts=" + authURI.getHosts()
-              + ", authURI database=" + authURI.getDatabase();
+                    + ", authURI database=" + authURI.getDatabase();
         }
         return result
-          + ", min=" + this.min + ", max=" + this.max
-          + ", query=" + this.query
-          + ", sort=" + this.sort
-          + ", fields=" + this.fields
-          + ", limit=" + this.limit
-          + ", skip=" + this.skip
-          + ", notimeout=" + this.notimeout + '}';
+                + ", min=" + this.min + ", max=" + this.max
+                + ", query=" + this.query
+                + ", sort=" + this.sort
+                + ", fields=" + this.fields
+                + ", limit=" + this.limit
+                + ", skip=" + this.skip
+                + ", notimeout=" + this.notimeout + '}';
     }
 
     @Override
@@ -364,14 +364,14 @@ public boolean equals(final Object o) {
             return false;
         }
         if (limit == null && that.getLimit() != null
-          || !limit.equals(that.getLimit())) {
+                || !limit.equals(that.getLimit())) {
             return false;
         }
         if (skip == null && that.getSkip() != null
-          || !skip.equals(that.getSkip())) {
+                || !skip.equals(that.getSkip())) {
             return false;
         }
         return true;
     }
 
-}
+}
diff --git a/gradle/functions.gradle b/gradle/functions.gradle
@@ -3,7 +3,7 @@ import org.apache.tools.ant.filters.ReplaceTokens
 def downloadFile(url) {
     def tmpDir = new File(System.properties['java.io.tmpdir'])
     def file = new File(tmpDir, new File(new URL(url).getPath()).getName())
-    
+
     def count = 0;
     while (!file.exists()) {
         try {
@@ -50,6 +50,7 @@ task installPig() << {
 }
 
 task downloadEnronEmails() << {
+    //extract(dataHome, dataHome, downloadFile('https://mongodb-enron-email.s3-website-us-east-1.amazonaws.com/mongodb-enron-email/enron_mongo.tar.bz2'))
     extract(dataHome, dataHome, downloadFile('https://s3.amazonaws.com/mongodb-enron-email/enron_mongo.tar.bz2'))
 }
 
@@ -75,7 +76,7 @@ task copyFiles(dependsOn: [installHadoop, installHive, installPig]) << {
     safeCopy("streaming/build/libs/mongo-hadoop-streaming-${project(':core').version}.jar", hadoopLib, "mongo-hadoop-streaming.jar")
     safeCopy("hive/build/libs/mongo-hadoop-hive-${project(':core').version}.jar", hiveHome + '/lib', "mongo-hadoop-hive.jar")
     safeCopy(findJar(":core", "mongo-java-driver"), hadoopLib, "mongo-java-driver.jar")
-    
+
     println "Updating cluster configuration"
     copy {
         from 'clusterConfigs'
@@ -109,4 +110,4 @@ def safeCopy(fromPath, toPath, newName) {
     if (!copied) {
         throw new GradleException("Failed to copy a file: " + fromPath, new FileNotFoundException(fromPath))
     }
-}
+}
diff --git a/gradle/hadoop.gradle b/gradle/hadoop.gradle
@@ -14,10 +14,10 @@ def execute(command, args = [], outStream = null, errStream = null, background =
     env << System.getProperties()
 
     def executor = new org.zeroturnaround.exec.ProcessExecutor().command([command.toString()] + args)
-                                                                .readOutput(true)
-                                                                .environment(env)
-                                                                .redirectOutput(outStream)
-                                                                .redirectError(errStream != null ? errStream : outStream);
+            .readOutput(true)
+            .environment(env)
+            .redirectOutput(outStream)
+            .redirectError(errStream != null ? errStream : outStream);
     if (!background) {
         try {
             def result = executor.execute();
@@ -39,7 +39,7 @@ def stopService(signal, service, name) {
             if (Os.isFamily(Os.FAMILY_WINDOWS)) {
                 if (signal == 'TERM') {
                     execute("taskkill", "/PID", process.split()[0])
-                 } else{
+                } else{
                     execute("taskkill", "/F", "/PID", process.split()[0])
                 }
             }else{
@@ -139,26 +139,26 @@ task historicalYield(dependsOn: startCluster) << {
 
     exec() {
         commandLine mongoimport, "-d", "mongo_hadoop", "-c", "yield_historical.in", "--drop",
-                    "examples/treasury_yield/src/main/resources/yield_historical_in.json"
+                "examples/treasury_yield/src/main/resources/yield_historical_in.json"
     }
 
     hadoop("examples/treasury_yield/build/libs/treasury_yield-${project(':core').version}.jar",
-           "com.mongodb.hadoop.examples.treasury.TreasuryYieldXMLConfig", [
+            "com.mongodb.hadoop.examples.treasury.TreasuryYieldXMLConfig", [
             "mongo.input.uri=mongodb://localhost:27017/mongo_hadoop.yield_historical.in",
             "mongo.output.uri=mongodb://localhost:27017/mongo_hadoop.yield_historical.out"
     ])
 }
 
 task sensorData(dependsOn: 'startCluster') << {
     hadoop("examples/sensors/build/libs/sensors-${project(':core').version}.jar",
-           "com.mongodb.hadoop.examples.sensors.Devices", [])
+            "com.mongodb.hadoop.examples.sensors.Devices", [])
 
     hadoop("examples/sensors/build/libs/sensors-${project(':core').version}.jar",
-           "com.mongodb.hadoop.examples.sensors.Logs", ["io.sort.mb=100"])
+            "com.mongodb.hadoop.examples.sensors.Logs", ["io.sort.mb=100"])
 }
 
-
-task enronEmails(dependsOn: [downloadEnronEmails, startCluster]) << {
+//downloadEnronEmails
+task enronEmails(dependsOn: [startCluster]) << {
     // Create BSON file input directory.
     exec() {
         commandLine "${hadoopHome}/bin/hdfs", "dfs", "-mkdir", "-p", "/messages"
@@ -170,8 +170,8 @@ task enronEmails(dependsOn: [downloadEnronEmails, startCluster]) << {
              "-c", "org.apache.hadoop.io.compress.BZip2Codec",
              "-o", "hdfs://localhost:8020/messages"])
     // MR job
-    hadoop("examples/enron/build/libs/enron-${project(':core').version}.jar",
-           "com.mongodb.hadoop.examples.enron.EnronMail", [])
+    //hadoop("examples/enron/build/libs/enron-${project(':core').version}.jar",
+    //       "com.mongodb.hadoop.examples.enron.EnronMail", [])
 }
 
 task shakespeare(dependsOn: [downloadShakespeare, startCluster]) << {
@@ -214,4 +214,4 @@ def hadoop(jar, className, args, commandArgs = []) {
         environment << hadoopEnv
         commandLine line
     }
-}
+}
diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar
diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
@@ -1,6 +1,5 @@
-#Mon Mar 09 18:25:42 PDT 2015
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.8.1-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-2.2.1-all.zip
diff --git a/gradlew b/gradlew
@@ -161,4 +161,4 @@ function splitJvmOpts() {
 eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
 JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
 
-exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"
+exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain