fluo-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From GitBox <...@apache.org>
Subject [GitHub] keith-turner commented on a change in pull request #1001: [WIP] - Issue 978
Date Thu, 01 Jan 1970 00:00:00 GMT
keith-turner commented on a change in pull request #1001: [WIP] - Issue 978
URL: https://github.com/apache/fluo/pull/1001#discussion_r163423955
 
 

 ##########
 File path: modules/core/src/main/java/org/apache/fluo/core/impl/TransactionImpl.java
 ##########
 @@ -872,262 +847,356 @@ public int getSize() {
     return size;
   }
 
-  private <V> void addCallback(CompletableFuture<V> cfuture, CommitData cd,
-      OnSuccessInterface<V> onSuccessInterface) {
-    cfuture.handleAsync((result, exception) -> {
-      if (exception != null) {
-        cd.commitObserver.failed(exception);
-        return null;
-      } else {
-        try {
-          onSuccessInterface.onSuccess(result);
-          return null;
-        } catch (Exception e) {
-          cd.commitObserver.failed(e);
-          return null;
-        }
-      }
-    }, env.getSharedResources().getAsyncCommitExecutor());
-  }
+  // TODO exception handling!!!! How?????
+  abstract class CommitStep {
+    private CommitStep nextStep;
 
-  @Override
-  public synchronized void commitAsync(AsyncCommitObserver commitCallback) {
+    // the boolean indicates if the operation was successful.
+    abstract CompletableFuture<Boolean> getMainOp(CommitData cd);
 
-    checkIfOpen();
-    status = TxStatus.COMMIT_STARTED;
-    commitAttempted = true;
+    // create and run this op in the event that the main op was a failure
+    abstract CompletableFuture<Void> getFailureOp(CommitData cd);
 
-    try {
-      CommitData cd = createCommitData();
-      beginCommitAsync(cd, commitCallback, null);
-    } catch (Exception e) {
-      e.printStackTrace();
-      commitCallback.failed(e);
+    // set the next step to run if this step is successful
+    CommitStep andThen(CommitStep next) {
+      this.nextStep = next;
+      return next;
     }
-  }
 
-  private void beginCommitAsync(CommitData cd, AsyncCommitObserver commitCallback,
-      RowColumn primary) {
 
-    if (updates.size() == 0) {
-      // TODO do async
-      deleteWeakRow();
-      commitCallback.committed();
-      return;
+    CompletableFuture<Void> compose(CommitData cd) {
+      return getMainOp(cd).thenComposeAsync(successful -> {
+        if (successful) {
+          if (nextStep != null) {
+            return nextStep.compose(cd);
+          } else {
+            return CompletableFuture.completedFuture(null);
+          }
+        } else {
+          return getFailureOp(cd);
+        }
+      }, env.getSharedResources().getAsyncCommitExecutor());
     }
 
-    for (Map<Column, Bytes> cols : updates.values()) {
-      stats.incrementEntriesSet(cols.size());
-    }
+  }
 
-    Bytes primRow = null;
-    Column primCol = null;
+  abstract class ConditionalStep extends CommitStep {
 
-    if (primary != null) {
-      primRow = primary.getRow();
-      primCol = primary.getColumn();
-      if (notification != null && !primary.equals(notification.getRowColumn())) {
-        throw new IllegalArgumentException("Primary must be notification");
-      }
-    } else if (notification != null) {
-      primRow = notification.getRow();
-      primCol = notification.getColumn();
-    } else {
+    CommitData cd;
 
-      outer: for (Entry<Bytes, Map<Column, Bytes>> entry : updates.entrySet())
{
-        for (Entry<Column, Bytes> entry2 : entry.getValue().entrySet()) {
-          if (!isReadLock(entry2.getValue())) {
-            primRow = entry.getKey();
-            primCol = entry2.getKey();
-            break outer;
-          }
-        }
-      }
+    public abstract Collection<ConditionalMutation> createMutations(CommitData cd);
 
-      if (primRow == null) {
-        // there are only read locks, so nothing to write
-        deleteWeakRow();
-        commitCallback.committed();
-        return;
-      }
-    }
+    public abstract Iterator<Result> handleUnknown(CommitData cd, Iterator<Result>
results)
+        throws Exception;
 
-    // get a primary column
-    cd.prow = primRow;
-    Map<Column, Bytes> colSet = updates.get(cd.prow);
-    cd.pcol = primCol;
-    cd.pval = colSet.remove(primCol);
-    if (colSet.size() == 0) {
-      updates.remove(cd.prow);
+    public abstract boolean processResults(CommitData cd, Iterator<Result> results)
+        throws Exception;
+
+    public AsyncConditionalWriter getACW(CommitData cd) {
+      return cd.acw;
     }
 
-    cd.commitObserver = commitCallback;
+    @Override
+    CompletableFuture<Boolean> getMainOp(CommitData cd) {
+      // TODO not sure threading is correct
+      Executor ace = env.getSharedResources().getAsyncCommitExecutor();
+      return getACW(cd).apply(createMutations(cd)).thenCompose(results -> {
+        // ugh icky that this is an iterator, forces copy to inspect.. could refactor async
CW to
+        // return collection
+        ArrayList<Result> resultsList = new ArrayList<>();
+        Iterators.addAll(resultsList, results);
+        boolean containsUknown = false;
+        for (Result result : resultsList) {
+          try {
+            containsUknown |= result.getStatus() == Status.UNKNOWN;
+          } catch (Exception e) {
+            throw new CompletionException(e);
+          }
+        }
+        if (containsUknown) {
+          // process unknown in sync executor
+          Executor se = env.getSharedResources().getSyncCommitExecutor();
+          return CompletableFuture.supplyAsync(() -> {
+            try {
+              return handleUnknown(cd, resultsList.iterator());
+            } catch (Exception e) {
+              throw new CompletionException(e);
+            }
+          }, se);
+        } else {
+          return CompletableFuture.completedFuture(resultsList.iterator());
+        }
+      }).thenApplyAsync(results -> {
+        try {
+          return processResults(cd, results);
+        } catch (Exception e) {
+          throw new CompletionException(e);
+        }
+      }, ace);
+    }
 
-    // try to lock primary column
-    final ConditionalMutation pcm =
-        prewrite(cd.prow, cd.pcol, cd.pval, cd.prow, cd.pcol, isTriggerRow(cd.prow));
 
-    CompletableFuture<Iterator<Result>> cfuture = cd.acw.apply(Collections.singletonList(pcm));
-    addCallback(cfuture, cd, result -> postLockPrimary(cd, pcm, Iterators.getOnlyElement(result)));
   }
 
-  private void postLockPrimary(final CommitData cd, final ConditionalMutation pcm, Result
result)
-      throws Exception {
-    final Status mutationStatus = result.getStatus();
+  class LockPrimaryStep extends ConditionalStep {
 
-    if (mutationStatus == Status.ACCEPTED) {
-      lockOtherColumns(cd);
-    } else {
-      env.getSharedResources().getSyncCommitExecutor().execute(new SynchronousCommitTask(cd)
{
-        @Override
-        protected void runCommitStep(CommitData cd) throws Exception {
-          synchronousPostLockPrimary(cd, pcm, mutationStatus);
-        }
-      });
+    @Override
+    public Collection<ConditionalMutation> createMutations(CommitData cd) {
+      return Collections
+          .singleton(prewrite(cd.prow, cd.pcol, cd.pval, cd.prow, cd.pcol, isTriggerRow(cd.prow)));
     }
-  }
 
-  private void synchronousPostLockPrimary(CommitData cd, ConditionalMutation pcm,
-      Status mutationStatus) throws AccumuloException, AccumuloSecurityException, Exception
{
-    // TODO convert this code to async
-    while (mutationStatus == Status.UNKNOWN) {
-      TxInfo txInfo = TxInfo.getTransactionInfo(env, cd.prow, cd.pcol, startTs);
-
-      switch (txInfo.status) {
-        case LOCKED:
-          mutationStatus = Status.ACCEPTED;
-          break;
-        case ROLLED_BACK:
-          mutationStatus = Status.REJECTED;
-          break;
-        case UNKNOWN:
-          // TODO async
-          mutationStatus = cd.cw.write(pcm).getStatus();
-          // TODO handle case were data other tx has lock
-          break;
-        case COMMITTED:
-        default:
-          throw new IllegalStateException(
-              "unexpected tx state " + txInfo.status + " " + cd.prow + " " + cd.pcol);
+    @Override
+    public Iterator<Result> handleUnknown(CommitData cd, Iterator<Result> results)
+        throws Exception {
+
+      Result result = Iterators.getOnlyElement(results);
+      Status mutationStatus = result.getStatus();
+      // TODO convert this code to async
+      while (mutationStatus == Status.UNKNOWN) {
+        TxInfo txInfo = TxInfo.getTransactionInfo(env, cd.prow, cd.pcol, startTs);
+
+        switch (txInfo.status) {
+          case LOCKED:
+            return Collections
+                .singleton(
+                    new Result(Status.ACCEPTED, result.getMutation(), result.getTabletServer()))
+                .iterator();
+          case ROLLED_BACK:
+            return Collections
+                .singleton(
+                    new Result(Status.REJECTED, result.getMutation(), result.getTabletServer()))
+                .iterator();
+          case UNKNOWN:
+            // TODO async
+            Result newResult = cd.cw.write(result.getMutation());
+            mutationStatus = newResult.getStatus();
+            if (mutationStatus != Status.UNKNOWN) {
+              return Collections.singleton(newResult).iterator();
+            }
+            // TODO handle case were data other tx has lock
+            break;
+          case COMMITTED:
+          default:
+            throw new IllegalStateException(
+                "unexpected tx state " + txInfo.status + " " + cd.prow + " " + cd.pcol);
 
+        }
       }
+
+      // TODO
+      throw new IllegalStateException();
     }
 
-    if (mutationStatus != Status.ACCEPTED) {
-      cd.addPrimaryToRejected();
-      getStats().setRejected(cd.getRejected());
-      // TODO do async
-      checkForOrphanedLocks(cd);
-      if (checkForAckCollision(pcm)) {
-        cd.commitObserver.alreadyAcknowledged();
-      } else {
-        cd.commitObserver.commitFailed(cd.getShortCollisionMessage());
-      }
-      return;
+    @Override
+    public boolean processResults(CommitData cd, Iterator<Result> results) throws Exception
{
+      Result result = Iterators.getOnlyElement(results);
+      return result.getStatus() == Status.ACCEPTED;
+    }
+
+    @Override
+    CompletableFuture<Void> getFailureOp(CommitData cd) {
+      // TODO can this be simplified by pushing some code to the superclass?
+      return CompletableFuture.supplyAsync(() -> {
+        final ConditionalMutation pcm = Iterables.getOnlyElement(createMutations(cd));
+
+        cd.addPrimaryToRejected();
+        getStats().setRejected(cd.getRejected());
+        // TODO do async
+        try {
+          checkForOrphanedLocks(cd);
+        } catch (Exception e) {
+          throw new CompletionException(e);
+        }
+        if (checkForAckCollision(pcm)) {
+          cd.commitObserver.alreadyAcknowledged();
+        } else {
+          cd.commitObserver.commitFailed(cd.getShortCollisionMessage());
+        }
+
+        return null;
+      }, env.getSharedResources().getSyncCommitExecutor());
     }
 
-    lockOtherColumns(cd);
   }
 
-  private void lockOtherColumns(CommitData cd) {
-    ArrayList<ConditionalMutation> mutations = new ArrayList<>();
+  class LockOtherStep extends ConditionalStep {
 
-    for (Entry<Bytes, Map<Column, Bytes>> rowUpdates : updates.entrySet()) {
-      ConditionalFlutation cm = null;
+    @Override
+    public AsyncConditionalWriter getACW(CommitData cd) {
+      return cd.bacw;
+    }
 
-      for (Entry<Column, Bytes> colUpdates : rowUpdates.getValue().entrySet()) {
-        if (cm == null) {
-          cm = prewrite(rowUpdates.getKey(), colUpdates.getKey(), colUpdates.getValue(),
cd.prow,
-              cd.pcol, false);
-        } else {
-          prewrite(cm, colUpdates.getKey(), colUpdates.getValue(), cd.prow, cd.pcol, false);
+
+    @Override
+    public Collection<ConditionalMutation> createMutations(CommitData cd) {
+
+      ArrayList<ConditionalMutation> mutations = new ArrayList<>();
+
+      for (Entry<Bytes, Map<Column, Bytes>> rowUpdates : updates.entrySet())
{
+        ConditionalFlutation cm = null;
+
+        for (Entry<Column, Bytes> colUpdates : rowUpdates.getValue().entrySet()) {
+          if (cm == null) {
+            cm = prewrite(rowUpdates.getKey(), colUpdates.getKey(), colUpdates.getValue(),
cd.prow,
+                cd.pcol, false);
+          } else {
+            prewrite(cm, colUpdates.getKey(), colUpdates.getValue(), cd.prow, cd.pcol, false);
+          }
         }
+
+        mutations.add(cm);
       }
 
-      mutations.add(cm);
-    }
+      cd.acceptedRows = new HashSet<>();
 
-    cd.acceptedRows = new HashSet<>();
+      return mutations;
+    }
 
-    CompletableFuture<Iterator<Result>> cfuture = cd.bacw.apply(mutations);
-    addCallback(cfuture, cd, results -> postLockOther(cd, results));
-  }
+    @Override
+    public Iterator<Result> handleUnknown(CommitData cd, Iterator<Result> results)
{
+      // TODO this step does not currently handle unknown
+      return results;
+    }
 
-  private void postLockOther(final CommitData cd, Iterator<Result> results) throws
Exception {
-    while (results.hasNext()) {
-      Result result = results.next();
-      // TODO handle unknown?
-      Bytes row = Bytes.of(result.getMutation().getRow());
-      if (result.getStatus() == Status.ACCEPTED) {
-        cd.acceptedRows.add(row);
-      } else {
-        cd.addToRejected(row, updates.get(row).keySet());
+    @Override
+    public boolean processResults(CommitData cd, Iterator<Result> results) throws Exception
{
+
+      while (results.hasNext()) {
+        Result result = results.next();
+        // TODO handle unknown?
+        Bytes row = Bytes.of(result.getMutation().getRow());
+        if (result.getStatus() == Status.ACCEPTED) {
+          cd.acceptedRows.add(row);
+        } else {
+          cd.addToRejected(row, updates.get(row).keySet());
+        }
       }
+
+      return cd.getRejected().size() == 0;
     }
 
-    if (cd.getRejected().size() > 0) {
-      getStats().setRejected(cd.getRejected());
-      env.getSharedResources().getSyncCommitExecutor().execute(new SynchronousCommitTask(cd)
{
-        @Override
-        protected void runCommitStep(CommitData cd) throws Exception {
+    @Override
+    CompletableFuture<Void> getFailureOp(CommitData cd) {
+      return CompletableFuture.supplyAsync(() -> {
+        getStats().setRejected(cd.getRejected());
+        try {
+          // Does this need to be async?
           checkForOrphanedLocks(cd);
-          rollbackOtherLocks(cd);
+        } catch (Exception e) {
+          throw new CompletionException(e);
+        }
+        return null;
+      }, env.getSharedResources().getSyncCommitExecutor()).thenCompose(v -> {
+        try {
+          return rollbackLocks(cd);
+        } catch (Exception e) {
+          throw new CompletionException(e);
         }
       });
-    } else if (stopAfterPreCommit) {
-      cd.commitObserver.committed();
-    } else {
-      CompletableFuture<Stamp> cfuture = env.getSharedResources().getOracleClient().getStampAsync();
-      addCallback(cfuture, cd, stamp -> beginSecondCommitPhase(cd, stamp));
     }
   }
 
-  private void rollbackOtherLocks(CommitData cd) throws Exception {
-    // roll back locks
+  abstract class BatchWriterStep extends CommitStep {
+    public abstract Collection<Mutation> createMutations(CommitData cd);
 
-    // TODO let rollback be done lazily? this makes GC more difficult
+    @Override
+    CompletableFuture<Boolean> getMainOp(CommitData cd) {
+      return env.getSharedResources().getBatchWriter()
+          .writeMutationsAsyncFuture(createMutations(cd)).thenApply(v -> true);
+    }
 
-    Flutation m;
+    @Override
+    CompletableFuture<Void> getFailureOp(CommitData cd) {
+      throw new IllegalStateException("Failure not expected");
+    }
+  }
 
-    ArrayList<Mutation> mutations = new ArrayList<>(cd.acceptedRows.size());
-    for (Bytes row : cd.acceptedRows) {
-      m = new Flutation(env, row);
-      for (Entry<Column, Bytes> entry : updates.get(row).entrySet()) {
-        if (isReadLock(entry.getValue())) {
-          m.put(entry.getKey(), ColumnConstants.RLOCK_PREFIX | ReadLockUtil.encodeTs(startTs,
true),
-              DelReadLockValue.encodeRollback());
-        } else {
-          m.put(entry.getKey(), ColumnConstants.DEL_LOCK_PREFIX | startTs,
-              DelLockValue.encodeRollback(false, true));
+
+
+  private CompletableFuture<Void> rollbackLocks(CommitData cd) throws Exception {
+    CommitStep firstStep = new RollbackOtherLocks();
+    firstStep.andThen(new RollbackPrimaryLock());
+
+    return firstStep.compose(cd)
+        .thenRun(() -> cd.commitObserver.commitFailed(cd.getShortCollisionMessage()));
+
+  }
+
+
+  class RollbackOtherLocks extends BatchWriterStep {
+
+    @Override
+    public Collection<Mutation> createMutations(CommitData cd) {
+      // roll back locks
+
+      // TODO let rollback be done lazily? this makes GC more difficult
+
+      Flutation m;
+
+      ArrayList<Mutation> mutations = new ArrayList<>(cd.acceptedRows.size());
+      for (Bytes row : cd.acceptedRows) {
+        m = new Flutation(env, row);
+        for (Entry<Column, Bytes> entry : updates.get(row).entrySet()) {
+          if (isReadLock(entry.getValue())) {
+            m.put(entry.getKey(),
+                ColumnConstants.RLOCK_PREFIX | ReadLockUtil.encodeTs(startTs, true),
+                DelReadLockValue.encodeRollback());
+          } else {
+            m.put(entry.getKey(), ColumnConstants.DEL_LOCK_PREFIX | startTs,
+                DelLockValue.encodeRollback(false, true));
+          }
         }
+        mutations.add(m);
       }
-      mutations.add(m);
+
+      return mutations;
     }
+  }
+
+  class RollbackPrimaryLock extends BatchWriterStep {
 
-    CompletableFuture<Void> cfuture =
-        env.getSharedResources().getBatchWriter().writeMutationsAsyncFuture(mutations);
-    addCallback(cfuture, cd, result -> rollbackPrimaryLock(cd));
+    @Override
+    public Collection<Mutation> createMutations(CommitData cd) {
+      // mark transaction as complete for garbage collection purposes
+      Flutation m = new Flutation(env, cd.prow);
+
+      m.put(cd.pcol, ColumnConstants.DEL_LOCK_PREFIX | startTs,
+          DelLockValue.encodeRollback(startTs, true, true));
+      m.put(cd.pcol, ColumnConstants.TX_DONE_PREFIX | startTs, EMPTY);
+
+      return Collections.singletonList(m);
+    }
   }
 
-  private void rollbackPrimaryLock(CommitData cd) throws Exception {
+  @VisibleForTesting
+  public boolean commitPrimaryColumn(CommitData cd, Stamp commitStamp) {
 
-    // mark transaction as complete for garbage collection purposes
-    Flutation m = new Flutation(env, cd.prow);
+    SyncCommitObserver sco = new SyncCommitObserver();
+    cd.commitObserver = sco;
+    try {
+      CommitStep firstStep = new GetCommitStampStepTest(commitStamp);
 
-    m.put(cd.pcol, ColumnConstants.DEL_LOCK_PREFIX | startTs,
-        DelLockValue.encodeRollback(startTs, true, true));
-    m.put(cd.pcol, ColumnConstants.TX_DONE_PREFIX | startTs, EMPTY);
+      firstStep.andThen(new WriteNotificationsStep()).andThen(new CommitPrimaryStep());
 
-    CompletableFuture<Void> cfuture =
-        env.getSharedResources().getBatchWriter().writeMutationsAsyncFuture(m);
-    addCallback(cfuture, cd,
-        result -> cd.commitObserver.commitFailed(cd.getShortCollisionMessage()));
+      firstStep.compose(cd).thenRun(() -> cd.commitObserver.committed())
+          .exceptionally(throwable -> {
+            cd.commitObserver.failed(throwable);
+            return null;
+          });
+      sco.waitForCommit();
+    } catch (CommitException e) {
+      return false;
+    } catch (Exception e) {
+      throw new FluoException(e);
+    }
+    return true;
   }
 
-  private void beginSecondCommitPhase(CommitData cd, Stamp commitStamp) throws Exception
{
-    if (startTs < commitStamp.getGcTimestamp()) {
-      rollbackOtherLocks(cd);
-    } else {
+  class GetCommitStampStep extends CommitStep {
+
+    @Override
+    CompletableFuture<Boolean> getMainOp(CommitData cd) {
       // Notification are written here for the following reasons :
 
 Review comment:
   I think it would make sense to move this entire comment block to `beginCommitAsync()`.
   In the process changing `Notification are written here` to `Notification are written between
GetCommitStampStep and CommitPrimaryStep for the following reasons`

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services

Mime
View raw message