Merge branch 'develop' of https://github.com/marklogic/java-client-api.git into develop

Author: georgeajit

src/main/java/com/marklogic/client/datamovement/HostAvailabilityListener.java

@@ -182,7 +182,7 @@ public void processFailure(WriteBatch batch, Throwable throwable) {
         logger.warn("Retrying failed batch: {}, results so far: {}, uris: {}",
           batch.getJobBatchNumber(), batch.getJobWritesSoFar(),
           Stream.of(batch.getItems()).map(event->event.getTargetUri()).collect(Collectors.toList()));
-        batch.getBatcher().retry(batch);
+        batch.getBatcher().retryWithFailureListeners(batch);
       } catch (RuntimeException e) {
         logger.error("Exception during retry", e);
         processFailure(batch, e);
@@ -202,7 +202,7 @@ public void processFailure(QueryBatchException queryBatch) {
         logger.warn("Retrying failed batch: {}, results so far: {}, forest: {}, forestBatch: {}, forest results so far: {}",
           queryBatch.getJobBatchNumber(), queryBatch.getJobResultsSoFar(), queryBatch.getForest().getForestName(),
           queryBatch.getForestBatchNumber(), queryBatch.getForestResultsSoFar());
-        queryBatch.getBatcher().retry(queryBatch);
+        queryBatch.getBatcher().retryWithFailureListeners(queryBatch);
       } catch (RuntimeException e) {
         logger.error("Exception during retry", e);
         processFailure(new QueryBatchException(queryBatch, e));

src/main/java/com/marklogic/client/datamovement/QueryBatcher.java

@@ -310,4 +310,18 @@ public interface QueryBatcher extends Batcher {
    * @param queryBatchListener  the QueryBatchListener which needs to be applied
    */
   void retryListener(QueryBatch batch, QueryBatchListener queryBatchListener);
+
+  /**
+   * Retry in the same thread to query a batch that failed. If it fails again,
+   * all the failure listeners associated with the batcher using onQueryFailure
+   * method would be processed.
+   * 
+   * Note : Use this method with caution as there is a possibility of infinite
+   * loops. If a batch fails and one of the failure listeners calls this method
+   * to retry with failure listeners and if the batch again fails, this would go
+   * on as an infinite loop until the batch succeeds.
+   * 
+   * @param queryEvent the information about the batch that failed
+   */
+  void retryWithFailureListeners(QueryEvent queryEvent);
 }

src/main/java/com/marklogic/client/datamovement/WriteBatcher.java

@@ -321,4 +321,18 @@ WriteBatcher addAs(String uri, DocumentMetadataWriteHandle metadataHandle,
    * @throws IllegalStateException if this job has not yet been started
    */
   JobTicket getJobTicket();
+
+  /**
+   * Retry in the same thread to send a batch that failed. If it fails again,
+   * all the failure listeners associated with the batcher using onBatchFailure
+   * method would be processed.
+   * 
+   * Note : Use this method with caution as there is a possibility of infinite
+   * loops. If a batch fails and one of the failure listeners calls this method
+   * to retry with failure listeners and if the batch again fails, this would go
+   * on as an infinite loop until the batch succeeds.
+   * 
+   * @param queryEvent the information about the batch that failed
+   */
+  public void retryWithFailureListeners(WriteBatch writeBatch);
 }

src/main/java/com/marklogic/client/datamovement/impl/DataMovementManagerImpl.java

@@ -51,7 +51,7 @@
 public class DataMovementManagerImpl implements DataMovementManager {
   private static Logger logger = LoggerFactory.getLogger(DataMovementManager.class);
   private DataMovementServices service = new DataMovementServices();
-  private ConcurrentHashMap<String, JobTicket> activeJobs = new ConcurrentHashMap<>();
+  private static ConcurrentHashMap<String, JobTicket> activeJobs = new ConcurrentHashMap<>();
   private ForestConfiguration forestConfig;
   private DatabaseClient primaryClient;
   // clientMap key is the hostname_database

src/main/java/com/marklogic/client/datamovement/impl/QueryBatcherImpl.java

@@ -124,11 +124,19 @@ public QueryBatcherImpl onQueryFailure(QueryFailureListener listener) {
    */
   @Override
   public void retry(QueryEvent queryEvent) {
+    retry(queryEvent, false);
+  }
+
+  @Override
+  public void retryWithFailureListeners(QueryEvent queryEvent) {
+    retry(queryEvent, true);
+  }
+
+  private void retry(QueryEvent queryEvent, boolean callFailListeners) {
     if ( isStopped() == true ) {
       logger.warn("Job is now stopped, aborting the retry");
       return;
     }
-    boolean callFailListeners = false;
     Forest retryForest = null;
     for ( Forest forest : getForestConfig().listForests() ) {
       if ( forest.equals(queryEvent.getForest()) ) {
@@ -151,7 +159,6 @@ public void retry(QueryEvent queryEvent) {
       queryEvent.getForestBatchNumber(), start, queryEvent.getJobBatchNumber(), callFailListeners);
     runnable.run();
   }
-
   /*
    * Accepts a QueryBatch which was successfully retrieved from the server and a 
    * QueryBatchListener which was failed to apply and retry that listener on the batch. 

src/main/java/com/marklogic/client/datamovement/impl/WriteBatcherImpl.java

@@ -461,26 +461,38 @@ public WriteBatcher onBatchFailure(WriteFailureListener listener) {
     return this;
   }
 
+  @Override
+  public void retryWithFailureListeners(WriteBatch batch) {
+    retry(batch, true);
+  }
+
   @Override
   public void retry(WriteBatch batch) {
+    retry(batch, false);
+  }
+
+  private void retry(WriteBatch batch, boolean callFailListeners) {
     if ( isStopped() == true ) {
       logger.warn("Job is now stopped, aborting the retry");
       return;
     }
     if ( batch == null ) throw new IllegalArgumentException("batch must not be null");
     boolean forceNewTransaction = true;
     BatchWriteSet writeSet = newBatchWriteSet(forceNewTransaction, batch.getJobBatchNumber());
-    writeSet.onFailure(throwable -> {
-      if ( throwable instanceof RuntimeException ) throw (RuntimeException) throwable;
-      else throw new DataMovementException("Failed to retry batch", throwable);
-    });
-    for ( WriteEvent doc : batch.getItems() ) {
+    if ( !callFailListeners ) {
+      writeSet.onFailure(throwable -> {
+        if ( throwable instanceof RuntimeException )
+          throw (RuntimeException) throwable;
+        else
+          throw new DataMovementException("Failed to retry batch", throwable);
+      });
+    }
+    for (WriteEvent doc : batch.getItems()) {
       writeSet.getWriteSet().add(doc.getTargetUri(), doc.getMetadata(), doc.getContent());
     }
     BatchWriter runnable = new BatchWriter(writeSet);
     runnable.run();
   }
-
   @Override
   public WriteBatchListener[]        getBatchSuccessListeners() {
     return successListeners.toArray(new WriteBatchListener[successListeners.size()]);

src/main/java/com/marklogic/client/example/cookbook/README.md

@@ -0,0 +1,53 @@
+# Using Cookbook Examples
+
+The most important use of cookbook examples is reading the source code.  You
+can do this on [github](https://github.com/marklogic/java-client-api) or on
+your machine once you've cloned the code from github.
+
+To run the examples, first edit the
+[Example.properties](../../../../../../resources/Example.properties) file in the
+distribution to specify the connection parameters for your server. Most
+Cookbook examples have a main method, so they can be run from the command-line
+like so:
+
+    java -cp $CLASSPATH com.marklogic.client.example.cookbook.DocumentWrite
+
+This, of course, requires that you have all necessary dependencies in the env
+variable $CLASSPATH.  You can get the classpath for your machine using the
+[maven dependency plugin](http://maven.apache.org/plugins/maven-dependency-plugin/usage.html)
+command:
+
+    mvn dependency:build-classpath
+
+# Testing Cookbook Examples
+
+Most cookbook examples pass their unit test if they run without error.  First
+edit the [Example.properties](../../../../../../resources/Example.properties) file
+in the distribution to specify the connection parameters for your server. Then
+run `mvn test` while specifying the unit test you want to run, for example:
+
+    mvn test -Dtest=DocumentWriteTest
+
+# Creating a Cookbook Example
+
+We encourage community-contributed cookbook examples!  Make sure you follow
+the guidelines in [CONTRIBUTING.md](../../../../../../../../CONTRIBUTING.md)
+when you submit a pull request.  Each cookbook example should be runnable from
+the command-line, so it should have a static `main` method.  The approach in
+the code should come as close as possible to production code (code one would
+reasonably expect to use in a production application), while remaining as
+simple as possible to facilitate grokking for newbies to the Java Client API.
+It should have helpful comments throughout, including javadocs since it will
+show up in the published javadocs.  It should be added to
+[AllCookbookExamples.java](https://github.com/marklogic/java-client-api/blob/develop/src/main/java/com/marklogic/client/example/cookbook/AllCookbookExamples.java)
+in order of recommended examples for developers to review.
+
+It should have a unit test added to
+[this package](https://github.com/marklogic/java-client-api/tree/develop/src/test/java/com/marklogic/client/test/example/cookbook).
+The unit test can test whatever is needed, however most cookbook unit tests
+just run the class and consider it success if no errors are thrown.  Some
+cookbook examples, such as SSLClientCreator and KerberosClientCreator cannot be
+included in unit tests because the unit tests require a server configured with
+digest authentication and those tests require a different authentication
+scheme.  Any cookbook examples not included in unit tests run the risk of
+breaking without anyone noticing--hence we have unit tests whenever possible.

src/main/java/com/marklogic/client/example/cookbook/datamovement/BulkExportToJdbc.java

@@ -53,10 +53,28 @@
 
 import java.text.SimpleDateFormat;
 
+/** BulkExportToJdbc shows how simple it is to use Data Movement SDK to move
+ * massive data sets from a source MarkLogic Server to a JDBC target server.
+ * In this example all employees are exported using a query matching all docs
+ * in directory /employees/.  Using the
+ * [Shortcut Method](https://docs.marklogic.com/javadoc/client/overview-summary.html#ShortcutMethods)
+ * `getContentAs` and the Employee POJO class (pre-registered with the handle
+ * registry by DatabaseClientSingleton), we can easily serialize each document
+ * to an Employee object.  From there it's straightforward to use Spring's
+ * JdbcTemplate to write the employees, their salaries, and their titles via
+ * JDBC.  Of course, Spring's JdbcTemplate is not required--you could choose
+ * your favorite JDBC libraries to use with Data Movement SDK.  And of course
+ * you don't need to deserialize to pojos--you could use any of the Java Client
+ * API handles to deserialize the matching documents.
+ */
 public class BulkExportToJdbc {
   private static Logger logger = LoggerFactory.getLogger(BulkExportToJdbc.class);
+  // this is the date format required by our relational database tables
   public static final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
 
+  // we're using a small thread count and batch size because the example
+  // dataset is small, but with a larger dataset you'd use more threads and
+  // larger batches
   private static int threadCount = 3;
   private static int batchSize   = 3;
 
@@ -68,25 +86,49 @@ public static void main(String[] args) throws IOException, SQLException {
   }
 
   public void run() throws IOException, SQLException {
+    // connect to JDBC and initialize JdbcTemplate
     JdbcTemplate jdbcTemplate = new JdbcTemplate(getDataSource());
-    final boolean isMySQLDB;
-    SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
+    // query for all employees in directory /employees/
     StructuredQueryDefinition query = new StructuredQueryBuilder().directory(true, "/employees/");
+    // run the query on each forest in the cluster and asynchronously paginate
+    // through matches, sending them to the onUrisReady listener ExportListener
     QueryBatcher qb = moveMgr.newQueryBatcher(query)
       .withBatchSize(batchSize)
       .withThreadCount(threadCount)
+
+      // use withConsistentSnapshot so the set of matches doesn't change while this job
+      // runs even though updates are still occurring concurrently in MarkLogic Server.
+      // Requires a [merge timestamp](https://docs.marklogic.com/guide/app-dev/point_in_time#id_32468)
+      // to be set on MarkLogic Server.
+      .withConsistentSnapshot()
+
       .onUrisReady(
+        // Since ExportListener meets our needs we'll use it instead of a
+        // custom listener
         new ExportListener()
+
+          // since the ExportListener uses a separate request from the QueryBatcher
+          // we must also use withConsistentSnapshot on the ExportListener
           .withConsistentSnapshot()
+
+          // this is our custom onDocumentReady listener
           .onDocumentReady(record -> {
+
+            // Employee class is registered by DatabaseClientSingleton with the
+            // handle registry so we can use the getContentAs shortcut method
             Employee employee = record.getContentAs(Employee.class);
+
+            // using jdbcTemplate (which simplifies using jdbc) we can easily
+            // write the employee to the target relational database server
             jdbcTemplate.update(
               "INSERT INTO employees_export (emp_no, hire_date, first_name, last_name, gender, birth_date) " +
               "VALUES (?, ?, ?, ?, ?, ?) ",
               employee.getEmployeeId(), dateFormat.format(employee.getHireDate().getTime()), employee.getFirstName(),
               employee.getLastName(), employee.getGender() == Gender.MALE ? "M" : "F",
               dateFormat.format(employee.getBirthDate().getTime()));
             if ( employee.getSalaries() != null ) {
+              // each employee could have many salaries, and we need to write
+              // each of those to its own row
               for ( Salary salary : employee.getSalaries() ) {
                 jdbcTemplate.update(
                   "INSERT INTO salaries_export (emp_no, salary, from_date, to_date) " +
@@ -96,6 +138,8 @@ public void run() throws IOException, SQLException {
               }
             }
             if ( employee.getTitles() != null ) {
+              // each employee could have many titles, and we need to write
+              // each of those to its own row
               for ( Title title : employee.getTitles() ) {
                 jdbcTemplate.update(
                   "INSERT INTO titles_export (emp_no, title, from_date, to_date) " +
@@ -105,23 +149,40 @@ public void run() throws IOException, SQLException {
               }
             }
           })
+
+          // in a production application we could have more elaborate error
+          // handling here
           .onBatchFailure((failedBatch,exception) -> exception.printStackTrace())
       )
+
+      // another onUrisReady listener, this one custom, and just for logging
       .onUrisReady(batch ->
         logger.debug("Batch exported {}, so far {}",
           batch.getJobBatchNumber(), batch.getJobResultsSoFar())
       )
+
+      // in a production application we could have more elaborate error
+      // handling here
       .onQueryFailure(exception -> exception.printStackTrace());
+
+    // now that the job is configured, kick it off
     JobTicket ticket = moveMgr.startJob(qb);
+
+    // wait for the job to fully complete all pagination and all listeners
     qb.awaitCompletion();
+
+    // free up resources by stopping the job
     moveMgr.stopJob(qb);
+
+    // double check that we didn't have any failed batches
     JobReport report = moveMgr.getJobReport(ticket);
     if ( report.getFailureBatchesCount() > 0 ) {
       throw new IllegalStateException("Encountered " +
         report.getFailureBatchesCount() + " failed batches");
     }
   }
 
+  // get the jdbcUrl property from Example.properties with our jdbc connection info
   private DataSource getDataSource() throws IOException {
     ExampleProperties properties = Util.loadProperties();
     return new DriverManagerDataSource(properties.jdbcUrl, properties.jdbcUser, properties.jdbcPassword);