fluo-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From GitBox <...@apache.org>
Subject [GitHub] keith-turner commented on a change in pull request #1001: [WIP] - Issue 978
Date Thu, 01 Jan 1970 00:00:00 GMT
keith-turner commented on a change in pull request #1001: [WIP] - Issue 978
URL: https://github.com/apache/fluo/pull/1001#discussion_r162719140
 
 

 ##########
 File path: modules/core/src/main/java/org/apache/fluo/core/impl/TransactionImpl.java
 ##########
 @@ -872,447 +846,694 @@ public int getSize() {
     return size;
   }
 
-  private <V> void addCallback(CompletableFuture<V> cfuture, CommitData cd,
-      OnSuccessInterface<V> onSuccessInterface) {
-    cfuture.handleAsync((result, exception) -> {
-      if (exception != null) {
-        cd.commitObserver.failed(exception);
+  // TODO exception handling!!!! How?????
+  abstract class CommitStep {
+    private CommitStep nextStep;
+
+    // the boolean indicates if the operation was successful.
+    abstract CompletableFuture<Boolean> getMainOp(CommitData cd);
+
+    // create and run this op in the event that the main op was a failure
+    abstract CompletableFuture<Void> getFailureOp(CommitData cd);
+
+    // set the next step to run if this step is successful
+    CommitStep andThen(CommitStep next) {
+      this.nextStep = next;
+      return next;
+    }
+
+
+    CompletableFuture<Void> compose(CommitData cd) {
+      try {
+        return getMainOp(cd).thenComposeAsync(successful -> {
+          if (successful) {
+            if (nextStep != null) {
+              return nextStep.compose(cd);
+            } else {
+              return CompletableFuture.completedFuture(null);
+            }
+          } else {
+            return getFailureOp(cd);
+          }
+        }, env.getSharedResources().getAsyncCommitExecutor());
+      } catch (Exception e) {
+        cd.commitObserver.failed(e);
         return null;
-      } else {
+      }
+    }
+
+  }
+
+  abstract class ConditionalStep extends CommitStep {
+
+    CommitData cd;
+
+    public abstract Collection<ConditionalMutation> createMutations(CommitData cd);
+
+    public abstract Iterator<Result> handleUnknown(CommitData cd, Iterator<Result>
results)
+        throws Exception;
+
+    public abstract boolean processResults(CommitData cd, Iterator<Result> results)
+        throws Exception;
+
+    public AsyncConditionalWriter getACW(CommitData cd) {
+      return cd.acw;
+    }
+
+    @Override
+    CompletableFuture<Boolean> getMainOp(CommitData cd) {
+      // TODO not sure threading is correct
+      Executor ace = env.getSharedResources().getAsyncCommitExecutor();
+      return getACW(cd).apply(createMutations(cd)).thenCompose(results -> {
+        // ugh icky that this is an iterator, forces copy to inspect.. could refactor async
CW to
+        // return collection
+        ArrayList<Result> resultsList = new ArrayList<>();
+        Iterators.addAll(resultsList, results);
+        boolean containsUknown = false;
+        for (Result result : resultsList) {
+          try {
+            containsUknown |= result.getStatus() == Status.UNKNOWN;
+          } catch (Exception e) {
+            cd.commitObserver.failed(e);
+          }
+        }
+        if (containsUknown) {
+          // process unknown in sync executor
+          Executor se = env.getSharedResources().getSyncCommitExecutor();
+          return CompletableFuture.supplyAsync(() -> {
+            try {
+              return handleUnknown(cd, resultsList.iterator());
+            } catch (Exception e) {
+              cd.commitObserver.failed(e);
+              // TODO - what should we return? How do we stop processResults from running?
+              return null;
+            }
+          }, se);
+        } else {
+          return CompletableFuture.completedFuture(resultsList.iterator());
+        }
+      }).thenApplyAsync(results -> {
         try {
-          onSuccessInterface.onSuccess(result);
-          return null;
+          return processResults(cd, results);
         } catch (Exception e) {
           cd.commitObserver.failed(e);
-          return null;
+          return false;
         }
-      }
-    }, env.getSharedResources().getAsyncCommitExecutor());
+      }, ace);
+    }
+
+
   }
 
-  @Override
-  public synchronized void commitAsync(AsyncCommitObserver commitCallback) {
+  class LockPrimaryStep extends ConditionalStep {
 
-    checkIfOpen();
-    status = TxStatus.COMMIT_STARTED;
-    commitAttempted = true;
+    @Override
+    public Collection<ConditionalMutation> createMutations(CommitData cd) {
+      return Collections
+          .singleton(prewrite(cd.prow, cd.pcol, cd.pval, cd.prow, cd.pcol, isTriggerRow(cd.prow)));
+    }
 
-    try {
-      CommitData cd = createCommitData();
-      beginCommitAsync(cd, commitCallback, null);
-    } catch (Exception e) {
-      e.printStackTrace();
-      commitCallback.failed(e);
+    @Override
+    public Iterator<Result> handleUnknown(CommitData cd, Iterator<Result> results)
+        throws Exception {
+
+      Result result = Iterators.getOnlyElement(results);
+      Status mutationStatus = result.getStatus();
+      // TODO convert this code to async
+      while (mutationStatus == Status.UNKNOWN) {
+        TxInfo txInfo = TxInfo.getTransactionInfo(env, cd.prow, cd.pcol, startTs);
+
+        switch (txInfo.status) {
+          case LOCKED:
+            return Collections
+                .singleton(
+                    new Result(Status.ACCEPTED, result.getMutation(), result.getTabletServer()))
+                .iterator();
+          case ROLLED_BACK:
+            return Collections
+                .singleton(
+                    new Result(Status.REJECTED, result.getMutation(), result.getTabletServer()))
+                .iterator();
+          case UNKNOWN:
+            // TODO async
+            Result newResult = cd.cw.write(result.getMutation());
+            mutationStatus = newResult.getStatus();
+            if (mutationStatus != Status.UNKNOWN) {
+              return Collections.singleton(newResult).iterator();
+            }
+            // TODO handle case were data other tx has lock
+            break;
+          case COMMITTED:
+          default:
+            throw new IllegalStateException(
+                "unexpected tx state " + txInfo.status + " " + cd.prow + " " + cd.pcol);
+
+        }
+      }
+
+      // TODO
+      throw new IllegalStateException();
     }
-  }
 
-  private void beginCommitAsync(CommitData cd, AsyncCommitObserver commitCallback,
-      RowColumn primary) {
+    @Override
+    public boolean processResults(CommitData cd, Iterator<Result> results) throws Exception
{
+      Result result = Iterators.getOnlyElement(results);
+      return result.getStatus() == Status.ACCEPTED;
+    }
 
-    if (updates.size() == 0) {
-      // TODO do async
-      deleteWeakRow();
-      commitCallback.committed();
-      return;
+    @Override
+    CompletableFuture<Void> getFailureOp(CommitData cd) {
+      // TODO can this be simplified by pushing some code to the superclass?
+      return CompletableFuture.supplyAsync(() -> {
+        final ConditionalMutation pcm = Iterables.getOnlyElement(createMutations(cd));
+
+        cd.addPrimaryToRejected();
+        getStats().setRejected(cd.getRejected());
+        // TODO do async
+        try {
+          checkForOrphanedLocks(cd);
+        } catch (Exception e) {
+          cd.commitObserver.failed(e);
+          return null;
+        }
+        if (checkForAckCollision(pcm)) {
+          cd.commitObserver.alreadyAcknowledged();
+        } else {
+          cd.commitObserver.commitFailed(cd.getShortCollisionMessage());
+        }
+
+        return null;
+      }, env.getSharedResources().getSyncCommitExecutor());
     }
 
-    for (Map<Column, Bytes> cols : updates.values()) {
-      stats.incrementEntriesSet(cols.size());
+  }
+
+  class LockOtherStep extends ConditionalStep {
+
+    @Override
+    public AsyncConditionalWriter getACW(CommitData cd) {
+      return cd.bacw;
     }
 
-    Bytes primRow = null;
-    Column primCol = null;
 
-    if (primary != null) {
-      primRow = primary.getRow();
-      primCol = primary.getColumn();
-      if (notification != null && !primary.equals(notification.getRowColumn())) {
-        throw new IllegalArgumentException("Primary must be notification");
-      }
-    } else if (notification != null) {
-      primRow = notification.getRow();
-      primCol = notification.getColumn();
-    } else {
+    @Override
+    public Collection<ConditionalMutation> createMutations(CommitData cd) {
 
-      outer: for (Entry<Bytes, Map<Column, Bytes>> entry : updates.entrySet())
{
-        for (Entry<Column, Bytes> entry2 : entry.getValue().entrySet()) {
-          if (!isReadLock(entry2.getValue())) {
-            primRow = entry.getKey();
-            primCol = entry2.getKey();
-            break outer;
+      ArrayList<ConditionalMutation> mutations = new ArrayList<>();
+
+      for (Entry<Bytes, Map<Column, Bytes>> rowUpdates : updates.entrySet())
{
+        ConditionalFlutation cm = null;
+
+        for (Entry<Column, Bytes> colUpdates : rowUpdates.getValue().entrySet()) {
+          if (cm == null) {
+            cm = prewrite(rowUpdates.getKey(), colUpdates.getKey(), colUpdates.getValue(),
cd.prow,
+                cd.pcol, false);
+          } else {
+            prewrite(cm, colUpdates.getKey(), colUpdates.getValue(), cd.prow, cd.pcol, false);
           }
         }
-      }
 
-      if (primRow == null) {
-        // there are only read locks, so nothing to write
-        deleteWeakRow();
-        commitCallback.committed();
-        return;
+        mutations.add(cm);
       }
-    }
 
-    // get a primary column
-    cd.prow = primRow;
-    Map<Column, Bytes> colSet = updates.get(cd.prow);
-    cd.pcol = primCol;
-    cd.pval = colSet.remove(primCol);
-    if (colSet.size() == 0) {
-      updates.remove(cd.prow);
-    }
+      cd.acceptedRows = new HashSet<>();
 
-    cd.commitObserver = commitCallback;
+      return mutations;
+    }
 
-    // try to lock primary column
-    final ConditionalMutation pcm =
-        prewrite(cd.prow, cd.pcol, cd.pval, cd.prow, cd.pcol, isTriggerRow(cd.prow));
+    @Override
+    public Iterator<Result> handleUnknown(CommitData cd, Iterator<Result> results)
{
+      // TODO this step does not currently handle unknown
+      return results;
+    }
 
-    CompletableFuture<Iterator<Result>> cfuture = cd.acw.apply(Collections.singletonList(pcm));
-    addCallback(cfuture, cd, result -> postLockPrimary(cd, pcm, Iterators.getOnlyElement(result)));
-  }
+    @Override
+    public boolean processResults(CommitData cd, Iterator<Result> results) throws Exception
{
+
+      while (results.hasNext()) {
+        Result result = results.next();
+        // TODO handle unknown?
+        Bytes row = Bytes.of(result.getMutation().getRow());
+        if (result.getStatus() == Status.ACCEPTED) {
+          cd.acceptedRows.add(row);
+        } else {
+          cd.addToRejected(row, updates.get(row).keySet());
+        }
+      }
 
-  private void postLockPrimary(final CommitData cd, final ConditionalMutation pcm, Result
result)
-      throws Exception {
-    final Status mutationStatus = result.getStatus();
+      return cd.getRejected().size() == 0;
+    }
 
-    if (mutationStatus == Status.ACCEPTED) {
-      lockOtherColumns(cd);
-    } else {
-      env.getSharedResources().getSyncCommitExecutor().execute(new SynchronousCommitTask(cd)
{
-        @Override
-        protected void runCommitStep(CommitData cd) throws Exception {
-          synchronousPostLockPrimary(cd, pcm, mutationStatus);
+    @Override
+    CompletableFuture<Void> getFailureOp(CommitData cd) {
+      return CompletableFuture.supplyAsync(() -> {
+        getStats().setRejected(cd.getRejected());
+        try {
+          // Does this need to be async?
+          checkForOrphanedLocks(cd);
+        } catch (Exception e) {
+          cd.commitObserver.failed(e);
+        }
+        return null;
+      }, env.getSharedResources().getSyncCommitExecutor()).thenCompose(v -> {
+        try {
+          return rollbackLocks(cd);
+        } catch (Exception e) {
+          cd.commitObserver.failed(e);
+          return null;
         }
       });
     }
   }
 
-  private void synchronousPostLockPrimary(CommitData cd, ConditionalMutation pcm,
-      Status mutationStatus) throws AccumuloException, AccumuloSecurityException, Exception
{
-    // TODO convert this code to async
-    while (mutationStatus == Status.UNKNOWN) {
-      TxInfo txInfo = TxInfo.getTransactionInfo(env, cd.prow, cd.pcol, startTs);
-
-      switch (txInfo.status) {
-        case LOCKED:
-          mutationStatus = Status.ACCEPTED;
-          break;
-        case ROLLED_BACK:
-          mutationStatus = Status.REJECTED;
-          break;
-        case UNKNOWN:
-          // TODO async
-          mutationStatus = cd.cw.write(pcm).getStatus();
-          // TODO handle case were data other tx has lock
-          break;
-        case COMMITTED:
-        default:
-          throw new IllegalStateException(
-              "unexpected tx state " + txInfo.status + " " + cd.prow + " " + cd.pcol);
+  abstract class BatchWriterStep extends CommitStep {
+    public abstract Collection<Mutation> createMutations(CommitData cd);
 
-      }
+    @Override
+    CompletableFuture<Boolean> getMainOp(CommitData cd) {
+      return env.getSharedResources().getBatchWriter()
+          .writeMutationsAsyncFuture(createMutations(cd)).thenApply(v -> true);
     }
 
-    if (mutationStatus != Status.ACCEPTED) {
-      cd.addPrimaryToRejected();
-      getStats().setRejected(cd.getRejected());
-      // TODO do async
-      checkForOrphanedLocks(cd);
-      if (checkForAckCollision(pcm)) {
-        cd.commitObserver.alreadyAcknowledged();
-      } else {
-        cd.commitObserver.commitFailed(cd.getShortCollisionMessage());
-      }
-      return;
+    @Override
+    CompletableFuture<Void> getFailureOp(CommitData cd) {
+      throw new IllegalStateException("Failure not expected");
     }
+  }
+
+
+
+  private CompletableFuture<Void> rollbackLocks(CommitData cd) throws Exception {
+    CommitStep firstStep = new RollbackOtherLocks();
+    firstStep.andThen(new RollbackPrimaryLock());
+
+    return firstStep.compose(cd)
+        .thenAccept(v -> cd.commitObserver.commitFailed(cd.getShortCollisionMessage()));
 
 Review comment:
   Since `v` is not used/needed, could use `thenRun` here  like `.thenRun(() -> cd`

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services

Mime
View raw message