usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From toddn...@apache.org
Subject [11/50] [abbrv] incubator-usergrid git commit: Move to JClouds 1.7.1 for AWS S3 access to work around problems in OpenJDK 1.7 and Oracle JDK 8.
Date Mon, 01 Jun 2015 21:48:20 GMT
Move to JClouds 1.7.1 for AWS S3 access to work around problems in OpenJDK 1.7 and Oracle JDK
8.


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e707628b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e707628b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e707628b

Branch: refs/heads/USERGRID-628
Commit: e707628bf3a0aba3975717e47fab7951e58ae264
Parents: 5f4a66b
Author: Dave Johnson <dmjohnson@apigee.com>
Authored: Fri May 15 12:16:51 2015 -0400
Committer: Dave Johnson <dmjohnson@apigee.com>
Committed: Fri May 15 12:16:51 2015 -0400

----------------------------------------------------------------------
 stack/pom.xml                                   |  2 +-
 .../rest/applications/ServiceResource.java      |  2 +-
 .../applications/assets/AssetResourceIT.java    | 50 ++++++++++++----
 .../services/assets/data/S3BinaryStore.java     | 63 ++++++++++++--------
 4 files changed, 79 insertions(+), 38 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e707628b/stack/pom.xml
----------------------------------------------------------------------
diff --git a/stack/pom.xml b/stack/pom.xml
index 4f50779..5c3d4fe 100644
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@ -95,7 +95,7 @@
     <hector-version>1.1-4</hector-version>
     <hector-test-version>1.1-4</hector-test-version>
     <jackson-version>1.9.9</jackson-version>
-    <jclouds.version>1.6.2-incubating</jclouds.version>
+    <jclouds.version>1.7.1</jclouds.version>
     <jersey-version>1.18</jersey-version>
     <junit-version>4.11</junit-version>
     <log4j-version>1.2.16</log4j-version>

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e707628b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
index d5ef311..3eee96b 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
@@ -595,7 +595,7 @@ public class ServiceResource extends AbstractContextResource {
         ServiceResults serviceResults = executeServiceRequest( ui, response, ServiceAction.GET,
null );
         Entity entity = serviceResults.getEntity();
 
-        LOG.info( "In AssetsResource.findAsset with id: {}, range: {}, modifiedSince: {}",
+        LOG.info( "In ServiceResource.executeStreamGet with id: {}, range: {}, modifiedSince:
{}",
                 new Object[] { entityId, rangeHeader, modifiedSince } );
 
         Map<String, Object> fileMetadata = AssetUtils.getFileMetadata( entity );

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e707628b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AssetResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AssetResourceIT.java
b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AssetResourceIT.java
index b41659d..666f95e 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AssetResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AssetResourceIT.java
@@ -139,10 +139,14 @@ public class AssetResourceIT extends AbstractRestIT {
         UserRepo.INSTANCE.load( resource(), access_token );
 
         byte[] data = IOUtils.toByteArray( this.getClass().getResourceAsStream( "/file-bigger-than-5M"
) );
-        FormDataMultiPart form = new FormDataMultiPart().field( "file", data, MediaType.MULTIPART_FORM_DATA_TYPE
);
 
-        JsonNode node = resource().path( "/test-organization/test-app/foos" ).queryParam(
"access_token", access_token )
-                .accept( MediaType.APPLICATION_JSON ).type( MediaType.MULTIPART_FORM_DATA
)
+        FormDataMultiPart form = new FormDataMultiPart()
+                .field( "file", data, MediaType.MULTIPART_FORM_DATA_TYPE );
+
+        JsonNode node = resource().path( "/test-organization/test-app/foos" )
+                .queryParam( "access_token", access_token )
+                .accept( MediaType.APPLICATION_JSON )
+                .type( MediaType.MULTIPART_FORM_DATA )
                 .post( JsonNode.class, form );
 
         JsonNode idNode = node.get( "entities" ).get( 0 ).get( "uuid" );
@@ -151,25 +155,47 @@ public class AssetResourceIT extends AbstractRestIT {
         logNode( node );
 
         // get entity
-        node = resource().path( "/test-organization/test-app/foos/" + uuid ).queryParam(
"access_token", access_token )
-                .accept( MediaType.APPLICATION_JSON_TYPE ).get( JsonNode.class );
+        node = resource().path( "/test-organization/test-app/foos/" + uuid )
+                .queryParam( "access_token", access_token )
+                .accept( MediaType.APPLICATION_JSON_TYPE )
+                .get( JsonNode.class );
+
         logNode( node );
         assertEquals( "application/octet-stream", node.findValue( AssetUtils.CONTENT_TYPE
).getTextValue() );
         assertEquals( 5324800, node.findValue( AssetUtils.CONTENT_LENGTH ).getIntValue()
);
         idNode = node.get( "entities" ).get( 0 ).get( "uuid" );
         assertEquals( uuid, idNode.getTextValue() );
 
-        // get data
-        InputStream is =
-                resource().path( "/test-organization/test-app/foos/" + uuid ).queryParam(
"access_token", access_token )
-                        .accept( MediaType.APPLICATION_OCTET_STREAM_TYPE ).get( InputStream.class
);
+        int retries = 0;
+        boolean done = false;
+        byte[] foundData = new byte[0];
+
+        // retry until upload complete
+        while ( !done && retries < 30 ) {
+
+            // get data
+            try {
+                InputStream is = resource().path( "/test-organization/test-app/foos/" + uuid
)
+                        .queryParam( "access_token", access_token )
+                        .accept( MediaType.APPLICATION_OCTET_STREAM_TYPE )
+                        .get( InputStream.class );
+
+                foundData = IOUtils.toByteArray( is );
+                done = true;
+
+            } catch ( Exception intentiallyIgnored ) {}
+
+            Thread.sleep(1000);
+            retries++;
+        }
 
-        byte[] foundData = IOUtils.toByteArray( is );
         assertEquals( 5324800, foundData.length );
 
         // delete
-        node = resource().path( "/test-organization/test-app/foos/" + uuid ).queryParam(
"access_token", access_token )
-                .accept( MediaType.APPLICATION_JSON_TYPE ).delete( JsonNode.class );
+        node = resource().path( "/test-organization/test-app/foos/" + uuid )
+                .queryParam( "access_token", access_token )
+                .accept( MediaType.APPLICATION_JSON_TYPE )
+                .delete( JsonNode.class );
     }
 
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e707628b/stack/services/src/main/java/org/apache/usergrid/services/assets/data/S3BinaryStore.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/assets/data/S3BinaryStore.java
b/stack/services/src/main/java/org/apache/usergrid/services/assets/data/S3BinaryStore.java
index e1748d3..29b5e47 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/assets/data/S3BinaryStore.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/assets/data/S3BinaryStore.java
@@ -104,7 +104,7 @@ public class S3BinaryStore implements BinaryStore {
     @Override
     public void write( final UUID appId, final Entity entity, InputStream inputStream ) throws
IOException {
 
-        String uploadFileName = AssetUtils.buildAssetKey( appId, entity );
+        final String uploadFileName = AssetUtils.buildAssetKey( appId, entity );
         ByteArrayOutputStream baos = new ByteArrayOutputStream();
         long written = IOUtils.copyLarge( inputStream, baos, 0, FIVE_MB );
         byte[] data = baos.toByteArray();
@@ -112,13 +112,13 @@ public class S3BinaryStore implements BinaryStore {
         final Map<String, Object> fileMetadata = AssetUtils.getFileMetadata( entity
);
         fileMetadata.put( AssetUtils.LAST_MODIFIED, System.currentTimeMillis() );
 
-        String mimeType = AssetMimeHandler.get().getMimeType( entity, data );
+        final String mimeType = AssetMimeHandler.get().getMimeType( entity, data );
 
         if ( written < FIVE_MB ) { // total smaller than 5mb
 
             BlobStore blobStore = getContext().getBlobStore();
-            BlobBuilder.PayloadBlobBuilder bb =
-                    blobStore.blobBuilder( uploadFileName ).payload( data ).calculateMD5().contentType(
mimeType );
+            BlobBuilder.PayloadBlobBuilder bb = blobStore.blobBuilder( uploadFileName )
+                    .payload( data ).calculateMD5().contentType( mimeType );
 
             fileMetadata.put( AssetUtils.CONTENT_LENGTH, written );
             if ( fileMetadata.get( AssetUtils.CONTENT_DISPOSITION ) != null ) {
@@ -134,10 +134,11 @@ public class S3BinaryStore implements BinaryStore {
         }
         else { // bigger than 5mb... dump 5 mb tmp files and upload from them
 
-            // todo: yes, AsyncBlobStore is deprecated, but there appears to be no replacement
yet
-            final AsyncBlobStore blobStore = getContext().getAsyncBlobStore();
+            // create temp file and copy entire file to that temp file
 
-            File tempFile = File.createTempFile( entity.getUuid().toString(), "tmp" );
+            LOG.debug( "Writing temp file for S3 upload" );
+
+            final File tempFile = File.createTempFile( entity.getUuid().toString(), "tmp"
);
             tempFile.deleteOnExit();
             OutputStream os = null;
             try {
@@ -149,38 +150,52 @@ public class S3BinaryStore implements BinaryStore {
                 IOUtils.closeQuietly( os );
             }
 
-            BlobBuilder.PayloadBlobBuilder bb =
-                    blobStore.blobBuilder( uploadFileName ).payload( tempFile ).calculateMD5().contentType(
mimeType );
-
             fileMetadata.put( AssetUtils.CONTENT_LENGTH, written );
-            if ( fileMetadata.get( AssetUtils.CONTENT_DISPOSITION ) != null ) {
-                bb.contentDisposition( fileMetadata.get( AssetUtils.CONTENT_DISPOSITION ).toString()
);
-            }
-            final Blob blob = bb.build();
 
-            final File finalTempFile = tempFile;
-            final ListenableFuture<String> future =
-                    blobStore.putBlob( bucketName, blob, PutOptions.Builder.multipart() );
+            // JClouds no longer supports async blob store, so we have to do this fun stuff
 
-            Runnable listener = new Runnable() {
+            LOG.debug( "Starting upload thread" );
+
+            Thread uploadThread = new Thread( new Runnable() {
                 @Override
                 public void run() {
                     try {
-                        String eTag = future.get();
+                        LOG.debug( "S3 upload thread started" );
+
+                        BlobStore blobStore = getContext().getBlobStore();
+
+                        BlobBuilder.PayloadBlobBuilder bb =  blobStore.blobBuilder( uploadFileName
)
+                                .payload( tempFile ).calculateMD5().contentType( mimeType
);
+
+                        if ( fileMetadata.get( AssetUtils.CONTENT_DISPOSITION ) != null )
{
+                            bb.contentDisposition( fileMetadata.get( AssetUtils.CONTENT_DISPOSITION
).toString() );
+                        }
+                        final Blob blob = bb.build();
+
+                        String md5sum = Hex.encodeHexString( blob.getMetadata().getContentMetadata().getContentMD5()
);
+                        fileMetadata.put( AssetUtils.CHECKSUM, md5sum );
+
+                        LOG.debug( "S3 upload starting" );
+
+                        String eTag = blobStore.putBlob( bucketName, blob );
                         fileMetadata.put( AssetUtils.E_TAG, eTag );
+
+                        LOG.debug( "S3 upload complete eTag=" + eTag);
+
                         EntityManager em = emf.getEntityManager( appId );
                         em.update( entity );
-                        finalTempFile.delete();
+                        tempFile.delete();
                     }
                     catch ( Exception e ) {
                         LOG.error( "error uploading", e );
                     }
-                    if ( finalTempFile != null && finalTempFile.exists() ) {
-                        finalTempFile.delete();
+                    if ( tempFile != null && tempFile.exists() ) {
+                        tempFile.delete();
                     }
                 }
-            };
-            future.addListener( listener, executor );
+            });
+
+            uploadThread.start();
         }
     }
 


Mime
View raw message