beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "ASF GitHub Bot (JIRA)" <j...@apache.org>
Subject [jira] [Work logged] (BEAM-4176) Java: Portable batch runner passes all ValidatesRunner tests that non-portable runner passes
Date Thu, 20 Sep 2018 16:58:00 GMT

     [ https://issues.apache.org/jira/browse/BEAM-4176?focusedWorklogId=146063&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-146063
]

ASF GitHub Bot logged work on BEAM-4176:
----------------------------------------

                Author: ASF GitHub Bot
            Created on: 20/Sep/18 16:57
            Start Date: 20/Sep/18 16:57
    Worklog Time Spent: 10m 
      Work Description: tweise closed pull request #6225: [BEAM-4176] move some artifact-staging
logs from info to debug
URL: https://github.com/apache/beam/pull/6225
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ArtifactServiceStager.java
b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ArtifactServiceStager.java
index cc7b2feadc7..d5dbca0c23f 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ArtifactServiceStager.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ArtifactServiceStager.java
@@ -28,6 +28,7 @@
 import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
 import java.security.MessageDigest;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -57,12 +58,16 @@
 import org.apache.beam.vendor.grpc.v1.io.grpc.Channel;
 import org.apache.beam.vendor.grpc.v1.io.grpc.stub.StreamObserver;
 import org.apache.beam.vendor.protobuf.v3.com.google.protobuf.ByteString;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /** A client to stage files on an {@link ArtifactStagingServiceGrpc ArtifactService}. */
 public class ArtifactServiceStager {
   // 2 MB per file-request
   private static final int DEFAULT_BUFFER_SIZE = 2 * 1024 * 1024;
 
+  private static final Logger LOG = LoggerFactory.getLogger(ArtifactServiceStager.class);
+
   public static ArtifactServiceStager overChannel(Channel channel) {
     return overChannel(channel, DEFAULT_BUFFER_SIZE);
   }
@@ -94,9 +99,10 @@ private ArtifactServiceStager(Channel channel, int bufferSize) {
    *
    * @return The artifact staging token returned by the service
    */
-  public String stage(String stagingSessionToken, Iterable<StagedFile> files)
+  public String stage(String stagingSessionToken, Collection<StagedFile> files)
       throws IOException, InterruptedException {
     final Map<StagedFile, CompletionStage<ArtifactMetadata>> futures = new HashMap<>();
+    LOG.info("Staging {} files (token: {})", files.size(), stagingSessionToken);
     for (StagedFile file : files) {
       futures.put(
           file,
@@ -114,6 +120,8 @@ private String stageManifest(
     try {
       StagingResult stagingResult = MoreFutures.get(stagingFuture);
       if (stagingResult.isSuccess()) {
+        LOG.info(
+            "Staged {} files (token: {})", stagingResult.getMetadata().size(), stagingSessionToken);
         Manifest manifest =
             Manifest.newBuilder().addAllArtifact(stagingResult.getMetadata()).build();
         CommitManifestResponse response =
diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/artifact/BeamFileSystemArtifactRetrievalService.java
b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/artifact/BeamFileSystemArtifactRetrievalService.java
index 613ec4d4dd7..b4482d7e4fc 100644
--- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/artifact/BeamFileSystemArtifactRetrievalService.java
+++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/artifact/BeamFileSystemArtifactRetrievalService.java
@@ -100,7 +100,7 @@ public void getManifest(
   public void getArtifact(
       ArtifactApi.GetArtifactRequest request,
       StreamObserver<ArtifactApi.ArtifactChunk> responseObserver) {
-    LOG.info("GetArtifact {}", request);
+    LOG.debug("GetArtifact {}", request);
     String name = request.getName();
     try {
       ArtifactApi.ProxyManifest proxyManifest = MANIFEST_CACHE.get(request.getRetrievalToken());
@@ -131,7 +131,7 @@ public void getArtifact(
 
       ResourceId artifactResourceId =
           FileSystems.matchNewResource(location.getUri(), false /* is directory */);
-      LOG.info("Artifact {} located in {}", name, artifactResourceId);
+      LOG.debug("Artifact {} located in {}", name, artifactResourceId);
       Hasher hasher = Hashing.md5().newHasher();
       byte[] data = new byte[ARTIFACT_CHUNK_SIZE_BYTES];
       try (InputStream stream = Channels.newInputStream(FileSystems.open(artifactResourceId)))
{
diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/artifact/BeamFileSystemArtifactStagingService.java
b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/artifact/BeamFileSystemArtifactStagingService.java
index 5769045d0de..7a2d6345966 100644
--- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/artifact/BeamFileSystemArtifactStagingService.java
+++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/artifact/BeamFileSystemArtifactStagingService.java
@@ -214,7 +214,7 @@ public void onNext(PutArtifactRequest putArtifactRequest) {
         checkNotNull(putArtifactRequest);
         checkNotNull(putArtifactRequest.getMetadata());
         metadata = putArtifactRequest.getMetadata();
-        LOG.info("stored metadata: {}", metadata);
+        LOG.debug("stored metadata: {}", metadata);
         // Check the base path exists or create the base path
         try {
           ResourceId artifactsDirId =
@@ -224,7 +224,7 @@ public void onNext(PutArtifactRequest putArtifactRequest) {
           artifactId =
               artifactsDirId.resolve(
                   encodedFileName(metadata.getMetadata()), StandardResolveOptions.RESOLVE_FILE);
-          LOG.info(
+          LOG.debug(
               "Going to stage artifact {} to {}.", metadata.getMetadata().getName(), artifactId);
           artifactWritableByteChannel = FileSystems.create(artifactId, MimeTypes.BINARY);
           hasher = Hashing.md5().newHasher();
@@ -281,7 +281,7 @@ public void onError(Throwable throwable) {
     @Override
     public void onCompleted() {
       // Close the stream.
-      LOG.info("Staging artifact completed for " + artifactId);
+      LOG.debug("Staging artifact completed for " + artifactId);
       if (artifactWritableByteChannel != null) {
         try {
           artifactWritableByteChannel.close();


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Issue Time Tracking
-------------------

    Worklog Id:     (was: 146063)
    Time Spent: 23h 10m  (was: 23h)

> Java: Portable batch runner passes all ValidatesRunner tests that non-portable runner
passes
> --------------------------------------------------------------------------------------------
>
>                 Key: BEAM-4176
>                 URL: https://issues.apache.org/jira/browse/BEAM-4176
>             Project: Beam
>          Issue Type: Bug
>          Components: runner-flink
>            Reporter: Ben Sidhom
>            Priority: Major
>         Attachments: Screen Shot 2018-08-14 at 4.18.31 PM.png, Screen Shot 2018-09-03
at 11.07.38 AM.png
>
>          Time Spent: 23h 10m
>  Remaining Estimate: 0h
>
> We need this as a sanity check that runner execution is correct.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

Mime
View raw message