usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mru...@apache.org
Subject [2/2] usergrid git commit: Add a new Asset binary provider for Google Cloud Storage. Refactor the BinaryStore implementation to follow a Factory pattern.
Date Wed, 26 Apr 2017 22:44:32 GMT
Add a new Asset binary provider for Google Cloud Storage. Refactor the BinaryStore implementation to follow a Factory pattern.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/bafd4627
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/bafd4627
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/bafd4627

Branch: refs/heads/master
Commit: bafd462744803f28175b13d0c205e3b029a6ac54
Parents: 949b465
Author: Michael Russo <russomichael@google.com>
Authored: Wed Apr 26 15:43:40 2017 -0700
Committer: Michael Russo <russomichael@google.com>
Committed: Wed Apr 26 15:43:40 2017 -0700

----------------------------------------------------------------------
 .../main/resources/usergrid-default.properties  |  22 +-
 .../test/resources/usergrid-test-context.xml    |   6 +-
 .../rest/applications/ServiceResource.java      |  62 +--
 .../applications/assets/AssetsResource.java     |  38 +-
 .../main/resources/usergrid-rest-context.xml    |  15 +-
 .../applications/assets/AssetResourceIT.java    |   2 +-
 .../applications/assets/AwsAssetResourceIT.java |   4 +-
 .../assets/GoogleAssetResourceIT.java           | 419 +++++++++++++++++++
 .../applications/assets/aws/NoAWSCredsRule.java | 124 ------
 .../assets/rules/NoAWSCredsRule.java            | 136 ++++++
 .../assets/rules/NoGoogleCredsRule.java         | 118 ++++++
 stack/services/pom.xml                          |   6 +
 .../services/assets/BinaryStoreFactory.java     |  83 ++++
 .../services/assets/data/AWSBinaryStore.java    | 320 ++++++++++++++
 .../assets/data/AwsSdkS3BinaryStore.java        | 317 --------------
 .../services/assets/data/GoogleBinaryStore.java | 223 ++++++++++
 .../assets/data/LocalFileBinaryStore.java       |  17 +-
 .../services/assets/data/S3BinaryStore.java     | 346 ---------------
 .../test/resources/usergrid-test-context.xml    |  12 +-
 19 files changed, 1381 insertions(+), 889 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/config/src/main/resources/usergrid-default.properties
----------------------------------------------------------------------
diff --git a/stack/config/src/main/resources/usergrid-default.properties b/stack/config/src/main/resources/usergrid-default.properties
index 9c75ef0..0d1a193 100644
--- a/stack/config/src/main/resources/usergrid-default.properties
+++ b/stack/config/src/main/resources/usergrid-default.properties
@@ -565,13 +565,29 @@ usergrid.binary.bucketname=usergrid-binaries
 #
 usergrid.temp.files=/tmp/usergrid
 
-# Set the implementation of binary uploading to be used
-# Aws for aws s3 uploading, local
+# Set the implementation of binary uploading to be used. Valid values:
+#
+# local - uses the local file system and "usergrid.temp.files" specifies the location
+# aws - uses AWS Simple Storage Service (S3)
+# google - uses Google Cloud Platform Storage service (GCS)
+#
+#
+# If Google is specified, the following ENVIRONMENT variable must be set on the system running Usergrid:
+#
+# GOOGLE_APPLICATION_CREDENTIALS=/full/path/to/credentialfile.json
+#
+# See: https://developers.google.com/identity/protocols/application-default-credentials#howtheywork
+#
+#
+# If AWS is specified, the following JVM Arguments must be set:
+#
+# AWS_ACCESS_KEY_ID=awsKeyId
+# AWS_SECRET_KEY=awsKeySecret
+#
 usergrid.binary.uploader=local
 
 
 
-
 ###############################  Usergrid Admin  ##############################
 #
 # Usergrid has a sysadmin user which has access to the complete Usergrid system

http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/query-validator/src/test/resources/usergrid-test-context.xml
----------------------------------------------------------------------
diff --git a/stack/query-validator/src/test/resources/usergrid-test-context.xml b/stack/query-validator/src/test/resources/usergrid-test-context.xml
index 7affe6d..19af73d 100644
--- a/stack/query-validator/src/test/resources/usergrid-test-context.xml
+++ b/stack/query-validator/src/test/resources/usergrid-test-context.xml
@@ -44,7 +44,11 @@
     </bean>
 
 
-    <bean id="binaryStore" class="org.apache.usergrid.services.assets.data.LocalFileBinaryStore"/>
+    <bean id="binaryStoreFactory" class="org.apache.usergrid.services.assets.BinaryStoreFactory">
+        <constructor-arg name="properties" ref="properties"/>
+        <constructor-arg name="entityManagerFactory" ref="entityManagerFactory"/>
+        <constructor-arg name="reposLocation" value="${usergrid.temp.files}"/>
+    </bean>
 
     <bean id="setup" class="org.apache.usergrid.corepersistence.CpSetup">
         <constructor-arg ref="entityManagerFactory"/>

http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
index 3835b75..9373f5e 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
@@ -20,6 +20,7 @@ package org.apache.usergrid.rest.applications;
 import com.amazonaws.AmazonServiceException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.jaxrs.json.annotation.JSONP;
+import com.google.cloud.storage.StorageException;
 import org.apache.commons.lang.StringUtils;
 import org.apache.usergrid.management.OrganizationConfig;
 import org.apache.usergrid.management.OrganizationConfigProps;
@@ -34,10 +35,8 @@ import org.apache.usergrid.rest.applications.assets.AssetsResource;
 import org.apache.usergrid.rest.security.annotations.CheckPermissionsForPath;
 import org.apache.usergrid.security.oauth.AccessInfo;
 import org.apache.usergrid.services.*;
-import org.apache.usergrid.services.assets.data.AssetUtils;
-import org.apache.usergrid.services.assets.data.AwsSdkS3BinaryStore;
-import org.apache.usergrid.services.assets.data.BinaryStore;
-import org.apache.usergrid.services.assets.data.LocalFileBinaryStore;
+import org.apache.usergrid.services.assets.BinaryStoreFactory;
+import org.apache.usergrid.services.assets.data.*;
 import org.apache.usergrid.services.exceptions.AwsPropertiesNotFoundException;
 import org.apache.usergrid.utils.JsonUtils;
 import org.glassfish.jersey.media.multipart.BodyPart;
@@ -46,6 +45,7 @@ import org.glassfish.jersey.media.multipart.FormDataBodyPart;
 import org.glassfish.jersey.media.multipart.FormDataMultiPart;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.BeanInfoFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.annotation.Scope;
 import org.springframework.stereotype.Component;
@@ -70,15 +70,10 @@ public class ServiceResource extends AbstractContextResource {
     protected static final Logger logger = LoggerFactory.getLogger( ServiceResource.class );
     private static final String FILE_FIELD_NAME = "file";
 
-
-   // @Autowired
     private BinaryStore binaryStore;
 
     @Autowired
-    private LocalFileBinaryStore localFileBinaryStore;
-
-    @Autowired
-    private AwsSdkS3BinaryStore awsSdkS3BinaryStore;
+    private BinaryStoreFactory binaryStoreFactory;
 
     protected ServiceManager services;
 
@@ -89,17 +84,6 @@ public class ServiceResource extends AbstractContextResource {
     }
 
 
-    public void setBinaryStore(String binaryStoreType){
-
-        //TODO:GREY change this to be a property held elsewhere
-        if(binaryStoreType.equals("local")){
-            this.binaryStore = localFileBinaryStore;
-        }
-        else{
-            this.binaryStore = awsSdkS3BinaryStore;
-        }
-    }
-
 
     @Override
     public void setParent( AbstractContextResource parent ) {
@@ -749,14 +733,8 @@ public class ServiceResource extends AbstractContextResource {
     @Produces({MediaType.APPLICATION_JSON, "application/javascript"})
     private ApiResponse executeMultiPart( UriInfo ui, String callback, FormDataMultiPart multiPart,
                                               ServiceAction serviceAction ) throws Exception {
-
-        //needed for testing
-        if(properties.getProperty( PROPERTIES_USERGRID_BINARY_UPLOADER ).equals( "local" )){
-            this.binaryStore = localFileBinaryStore;
-        }
-        else{
-            this.binaryStore = awsSdkS3BinaryStore;
-        }
+        // needed for testing
+        this.binaryStore = binaryStoreFactory.getBinaryStore( properties.getProperty(PROPERTIES_USERGRID_BINARY_UPLOADER) );
 
         // collect form data values
         List<BodyPart> bodyParts = multiPart.getBodyParts();
@@ -831,12 +809,8 @@ public class ServiceResource extends AbstractContextResource {
     public Response uploadDataStream( @Context UriInfo ui, InputStream uploadedInputStream ) throws Exception {
 
         //needed for testing
-        if(properties.getProperty( PROPERTIES_USERGRID_BINARY_UPLOADER ).equals( "local" )){
-            this.binaryStore = localFileBinaryStore;
-        }
-        else{
-            this.binaryStore = awsSdkS3BinaryStore;
-        }
+        this.binaryStore = binaryStoreFactory.getBinaryStore( properties.getProperty(PROPERTIES_USERGRID_BINARY_UPLOADER) );
+
 
         ApiResponse response = createApiResponse();
         response.setAction( "get" );
@@ -871,13 +845,8 @@ public class ServiceResource extends AbstractContextResource {
             logger.trace( "ServiceResource.executeStreamGet" );
         }
 
-        //Needed for testing
-        if(properties.getProperty( PROPERTIES_USERGRID_BINARY_UPLOADER ).equals( "local" )){
-            this.binaryStore = localFileBinaryStore;
-        }
-        else{
-            this.binaryStore = awsSdkS3BinaryStore;
-        }
+        // needed for testing
+        this.binaryStore = binaryStoreFactory.getBinaryStore( properties.getProperty(PROPERTIES_USERGRID_BINARY_UPLOADER) );
 
         ApiResponse response = createApiResponse();
         response.setAction( "get" );
@@ -945,7 +914,6 @@ public class ServiceResource extends AbstractContextResource {
                 return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
             }
             catch(AmazonServiceException ase){
-
                 if( ase.getStatusCode() > 499 ){
                     logger.error(ase.getMessage());
                 }else if(logger.isDebugEnabled()){
@@ -953,6 +921,14 @@ public class ServiceResource extends AbstractContextResource {
                 }
                 return Response.status(ase.getStatusCode()).build();
             }
+            catch (StorageException se){
+                if( se.getCode() > 499 ){
+                    logger.error(se.getMessage());
+                }else if(logger.isDebugEnabled()){
+                    logger.debug(se.getMessage());
+                }
+                return Response.status(se.getCode()).build();
+            }
             catch(RuntimeException re){
                 logger.error(re.getMessage());
                 return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();

http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/rest/src/main/java/org/apache/usergrid/rest/applications/assets/AssetsResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/assets/AssetsResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/assets/AssetsResource.java
index b894205..f6fab41 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/assets/AssetsResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/assets/AssetsResource.java
@@ -26,10 +26,8 @@ import org.apache.usergrid.rest.ApiResponse;
 import org.apache.usergrid.rest.applications.ServiceResource;
 import org.apache.usergrid.rest.security.annotations.CheckPermissionsForPath;
 import org.apache.usergrid.rest.security.annotations.RequireApplicationAccess;
-import org.apache.usergrid.services.assets.data.AssetUtils;
-import org.apache.usergrid.services.assets.data.AwsSdkS3BinaryStore;
-import org.apache.usergrid.services.assets.data.BinaryStore;
-import org.apache.usergrid.services.assets.data.LocalFileBinaryStore;
+import org.apache.usergrid.services.assets.BinaryStoreFactory;
+import org.apache.usergrid.services.assets.data.*;
 import org.apache.usergrid.utils.StringUtils;
 import org.glassfish.jersey.media.multipart.FormDataParam;
 import org.slf4j.Logger;
@@ -56,15 +54,10 @@ public class AssetsResource extends ServiceResource {
 
     private static final Logger logger = LoggerFactory.getLogger( AssetsResource.class );
 
-    //@Autowired
     private BinaryStore binaryStore;
 
     @Autowired
-    private LocalFileBinaryStore localFileBinaryStore;
-
-    @Autowired
-    private AwsSdkS3BinaryStore awsSdkS3BinaryStore;
-
+    private BinaryStoreFactory binaryStoreFactory;
 
 
     @Override
@@ -123,12 +116,8 @@ public class AssetsResource extends ServiceResource {
                                 // @FormDataParam("file") FormDataContentDisposition fileDetail,
                                 @PathParam("entityId") PathSegment entityId ) throws Exception {
 
-        if(properties.getProperty( PROPERTIES_USERGRID_BINARY_UPLOADER ).equals( "local" )){
-            this.binaryStore = localFileBinaryStore;
-        }
-        else{
-            this.binaryStore = awsSdkS3BinaryStore;
-        }
+        // needed for testing
+        this.binaryStore = binaryStoreFactory.getBinaryStore( properties.getProperty(PROPERTIES_USERGRID_BINARY_UPLOADER) );
 
         if (uploadedInputStream != null ) {
     		UUID assetId = UUID.fromString( entityId.getPath() );
@@ -162,12 +151,8 @@ public class AssetsResource extends ServiceResource {
     public Response uploadDataStream( @PathParam("entityId") PathSegment entityId, InputStream uploadedInputStream )
             throws Exception {
 
-        if(properties.getProperty( PROPERTIES_USERGRID_BINARY_UPLOADER ).equals( "local" )){
-            this.binaryStore = localFileBinaryStore;
-        }
-        else{
-            this.binaryStore = awsSdkS3BinaryStore;
-        }
+        // needed for testing
+        this.binaryStore = binaryStoreFactory.getBinaryStore( properties.getProperty(PROPERTIES_USERGRID_BINARY_UPLOADER) );
 
         UUID assetId = UUID.fromString( entityId.getPath() );
         if (logger.isTraceEnabled()) {
@@ -191,12 +176,9 @@ public class AssetsResource extends ServiceResource {
     public Response findAsset( @Context UriInfo ui, @QueryParam("callback") @DefaultValue("callback") String callback,
                                @PathParam("entityId") PathSegment entityId, @HeaderParam("range") String range,
                                @HeaderParam("if-modified-since") String modifiedSince ) throws Exception {
-        if(properties.getProperty( PROPERTIES_USERGRID_BINARY_UPLOADER ).equals( "local" )){
-            this.binaryStore = localFileBinaryStore;
-        }
-        else{
-            this.binaryStore = awsSdkS3BinaryStore;
-        }
+
+        // needed for testing
+        this.binaryStore = binaryStoreFactory.getBinaryStore( properties.getProperty(PROPERTIES_USERGRID_BINARY_UPLOADER) );
 
         UUID assetId = UUID.fromString( entityId.getPath() );
         if (logger.isTraceEnabled()) {

http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/rest/src/main/resources/usergrid-rest-context.xml
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/resources/usergrid-rest-context.xml b/stack/rest/src/main/resources/usergrid-rest-context.xml
index 5e925bf..2aaa93a 100644
--- a/stack/rest/src/main/resources/usergrid-rest-context.xml
+++ b/stack/rest/src/main/resources/usergrid-rest-context.xml
@@ -63,17 +63,10 @@
 		<property name="queueCapacity" value="25" />
 	</bean>
 
-    <bean id="serviceResource"
-           class="org.apache.usergrid.rest.applications.ServiceResource" scope="prototype">
-        <property name="binaryStore" value="${usergrid.binary.uploader}"/>
+    <bean id="binaryStoreFactory" class="org.apache.usergrid.services.assets.BinaryStoreFactory">
+        <constructor-arg name="properties" ref="properties"/>
+        <constructor-arg name="entityManagerFactory" ref="entityManagerFactory"/>
+        <constructor-arg name="reposLocation" value="${usergrid.temp.files}"/>
     </bean>
 
-    <bean id="localFileBinaryStore" class="org.apache.usergrid.services.assets.data.LocalFileBinaryStore">
-        <property name="reposLocation" value="${usergrid.temp.files}"/>
-    </bean>
-
-    <bean id="awsSdkS3BinaryStore" class="org.apache.usergrid.services.assets.data.AwsSdkS3BinaryStore">
-    </bean>
-
-
 </beans>

http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AssetResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AssetResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AssetResourceIT.java
index 616d929..cc9c326 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AssetResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AssetResourceIT.java
@@ -281,7 +281,7 @@ public class AssetResourceIT extends AbstractRestIT {
 
 
     @Test
-    public void largeFileInS3() throws Exception {
+    public void largeFile() throws Exception {
 
         this.waitForQueueDrainAndRefreshIndex();
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AwsAssetResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AwsAssetResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AwsAssetResourceIT.java
index 4a9bfaa..196c662 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AwsAssetResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AwsAssetResourceIT.java
@@ -21,7 +21,7 @@ import com.amazonaws.SDKGlobalConfiguration;
 
 import net.jcip.annotations.NotThreadSafe;
 import org.apache.commons.io.IOUtils;
-import org.apache.usergrid.rest.applications.assets.aws.NoAWSCredsRule;
+import org.apache.usergrid.rest.applications.assets.rules.NoAWSCredsRule;
 import org.apache.usergrid.rest.test.resource.AbstractRestIT;
 import org.apache.usergrid.rest.test.resource.model.ApiResponse;
 import org.apache.usergrid.rest.test.resource.model.Entity;
@@ -58,7 +58,7 @@ public class AwsAssetResourceIT extends AbstractRestIT {
      * Mark tests as ignored if no AWS creds are present
      */
     @Rule
-    public NoAWSCredsRule awsCredsRule = new NoAWSCredsRule();
+    public NoAWSCredsRule credsRule = new NoAWSCredsRule();
 
     @Before
     public void setup(){

http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/GoogleAssetResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/GoogleAssetResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/GoogleAssetResourceIT.java
new file mode 100644
index 0000000..811420e
--- /dev/null
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/GoogleAssetResourceIT.java
@@ -0,0 +1,419 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.rest.applications.assets;
+
+
+import net.jcip.annotations.NotThreadSafe;
+import org.apache.commons.io.IOUtils;
+import org.apache.usergrid.rest.applications.assets.rules.NoAWSCredsRule;
+import org.apache.usergrid.rest.applications.assets.rules.NoGoogleCredsRule;
+import org.apache.usergrid.rest.test.resource.AbstractRestIT;
+import org.apache.usergrid.rest.test.resource.model.ApiResponse;
+import org.apache.usergrid.rest.test.resource.model.Entity;
+import org.apache.usergrid.services.assets.data.AssetUtils;
+import org.apache.usergrid.services.exceptions.AwsPropertiesNotFoundException;
+import org.glassfish.jersey.media.multipart.FormDataMultiPart;
+import org.junit.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.ForbiddenException;
+import javax.ws.rs.InternalServerErrorException;
+import javax.ws.rs.NotFoundException;
+import javax.ws.rs.core.MediaType;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Map;
+import java.util.UUID;
+
+import static org.apache.usergrid.management.AccountCreationProps.PROPERTIES_USERGRID_BINARY_UPLOADER;
+import static org.apache.usergrid.utils.MapUtils.hashMap;
+import static org.junit.Assert.*;
+
+
+@NotThreadSafe
+public class GoogleAssetResourceIT extends AbstractRestIT {
+
+    private Map<String, Object> originalProperties;
+    private static final Logger logger = LoggerFactory.getLogger(GoogleAssetResourceIT.class);
+
+    /**
+     * Mark tests as ignored if no credentials are present
+     */
+    @Rule
+    public NoGoogleCredsRule credsRule = new NoGoogleCredsRule();
+
+    @Before
+    public void setup() {
+        originalProperties = getRemoteTestProperties();
+        setTestProperty(PROPERTIES_USERGRID_BINARY_UPLOADER, "google");
+
+    }
+
+    @After
+    public void teardown() {
+        setTestProperties(originalProperties);
+    }
+
+
+    @Test
+    public void ensureMissingFileReturns404() {
+        Map<String, String> payload = hashMap("name", "assettest");
+        ApiResponse postResponse = pathResource(getOrgAppPath("missingFile")).post(payload);
+        UUID assetId = postResponse.getEntities().get(0).getUuid();
+        assertNotNull(assetId);
+
+        try {
+            pathResource(getOrgAppPath("missingFile/assettest")).getAssetAsStream(true);
+            fail("Should fail as there isn't an asset to retrieve.");
+        } catch (NotFoundException nfe) {
+        } catch (Exception e) {
+            logger.error("Unexpected exception", e);
+            fail("Shouldn't return any other kind of exception");
+        }
+
+    }
+
+    @Test
+    public void errorCheckingInvalidProperties() throws Exception {
+        Map<String, Object> errorTestProperties;
+        errorTestProperties = getRemoteTestProperties();
+        setTestProperty("usergrid.binary.bucketname", "xxx");
+
+        try {
+
+            Map<String, String> payload = hashMap("name", "assetname");
+            ApiResponse postResponse = pathResource(getOrgAppPath("foos")).post(payload);
+            UUID assetId = postResponse.getEntities().get(0).getUuid();
+            assertNotNull(assetId);
+
+            // post a binary asset to that entity
+
+            byte[] data = IOUtils.toByteArray(getClass().getResourceAsStream("/cassandra_eye.jpg"));
+            ApiResponse putResponse =
+                pathResource(getOrgAppPath("foos/" + assetId)).put(data, MediaType.APPLICATION_OCTET_STREAM_TYPE);
+
+
+        } catch (AwsPropertiesNotFoundException e) {
+            fail("Shouldn't interrupt runtime if access key isnt found.");
+        } catch (InternalServerErrorException uie) {
+            assertEquals(500, uie.getResponse().getStatus());
+        } finally {
+            setTestProperties(errorTestProperties);
+        }
+    }
+
+    @Test
+    public void errorCheckingInvalidPropertiesMultipartUpload() throws Exception {
+        Map<String, Object> errorTestProperties;
+        errorTestProperties = getRemoteTestProperties();
+        //test that we fail gracefully if we have missing properties
+        setTestProperty("usergrid.binary.bucketname", "xxx");
+
+        try {
+
+            byte[] data = IOUtils.toByteArray(this.getClass().getResourceAsStream("/file-bigger-than-5M"));
+            FormDataMultiPart form = new FormDataMultiPart().field("file", data, MediaType.MULTIPART_FORM_DATA_TYPE);
+            ApiResponse postResponse = pathResource(getOrgAppPath("foos")).post(form);
+            UUID assetId = postResponse.getEntities().get(0).getUuid();
+            logger.info("Waiting for upload to finish...");
+            Thread.sleep(5000);
+
+            // check that entire file was uploaded
+
+            ApiResponse getResponse = pathResource(getOrgAppPath("foos/" + assetId)).get(ApiResponse.class);
+            logger.info("Upload complete!");
+            InputStream is = pathResource(getOrgAppPath("foos/" + assetId)).getAssetAsStream();
+            byte[] foundData = IOUtils.toByteArray(is);
+            assertEquals(data.length, foundData.length);
+
+            // delete file
+
+            pathResource(getOrgAppPath("foos/" + assetId)).delete();
+
+
+        } catch (AwsPropertiesNotFoundException e) {
+            fail("Shouldn't interrupt runtime if access key isnt found.");
+        } catch (ForbiddenException fe) {
+            assertEquals(403, fe.getResponse().getStatus());
+        } finally {
+            setTestProperties(errorTestProperties);
+        }
+    }
+
+    @Test
+    public void octetStreamOnDynamicEntity() throws Exception {
+
+        this.waitForQueueDrainAndRefreshIndex();
+
+        //  post an asset entity
+
+        Map<String, String> payload = hashMap("name", "assetname");
+        ApiResponse postResponse = pathResource(getOrgAppPath("foos")).post(payload);
+        UUID assetId = postResponse.getEntities().get(0).getUuid();
+        assertNotNull(assetId);
+
+        // post a binary asset to that entity
+
+        byte[] data = IOUtils.toByteArray(getClass().getResourceAsStream("/cassandra_eye.jpg"));
+        ApiResponse putResponse = pathResource(getOrgAppPath("foos/" + assetId))
+            .put(data, MediaType.APPLICATION_OCTET_STREAM_TYPE);
+
+        // check that the asset entity has asset metadata
+
+        ApiResponse getResponse = pathResource(getOrgAppPath("foos/" + assetId)).get(ApiResponse.class);
+        Entity entity = getResponse.getEntities().get(0);
+        Map<String, Object> fileMetadata = (Map<String, Object>) entity.get("file-metadata");
+        Assert.assertEquals("image/jpeg", fileMetadata.get("content-type"));
+        Assert.assertEquals(7979, fileMetadata.get("content-length"));
+        assertEquals(assetId, entity.getUuid());
+
+        // get binary asset by UUID
+
+        InputStream is = pathResource(getOrgAppPath("foos/" + assetId)).getAssetAsStream();
+        byte[] foundData = IOUtils.toByteArray(is);
+        assertEquals(7979, foundData.length);
+
+        // get binary asset by name
+
+        is = pathResource(getOrgAppPath("foos/assetname")).getAssetAsStream();
+        foundData = IOUtils.toByteArray(is);
+        assertEquals(7979, foundData.length);
+    }
+
+
+    @Test
+    public void multipartPostFormOnDynamicEntity() throws Exception {
+
+        this.waitForQueueDrainAndRefreshIndex();
+
+        // post data larger than 5M
+
+        byte[] data = IOUtils.toByteArray(this.getClass().getResourceAsStream("/file-bigger-than-5M"));
+        FormDataMultiPart form = new FormDataMultiPart().field("file", data, MediaType.MULTIPART_FORM_DATA_TYPE);
+        ApiResponse putResponse = pathResource(getOrgAppPath("foos")).post(form);
+        this.waitForQueueDrainAndRefreshIndex();
+
+        UUID assetId = putResponse.getEntities().get(0).getUuid();
+        assertNotNull(assetId);
+
+        // retry until upload complete and we can get the data
+
+        int retries = 0;
+        boolean done = false;
+        byte[] foundData = new byte[0];
+        while (!done && retries < 30) {
+
+            try {
+                InputStream is = pathResource(getOrgAppPath("foos/" + assetId)).getAssetAsStream();
+                foundData = IOUtils.toByteArray(is);
+                done = true;
+
+            } catch (Exception intentiallyIgnored) {
+            }
+
+            Thread.sleep(1000);
+            retries++;
+        }
+
+        //  did we get expected number of bytes of data?
+
+        assertEquals(5324800, foundData.length);
+
+        pathResource(getOrgAppPath("foos/" + assetId)).delete();
+    }
+
+
+    @Test
+    public void multipartPutFormOnDynamicEntity() throws Exception {
+
+        this.waitForQueueDrainAndRefreshIndex();
+
+        // post an entity
+
+        Map<String, String> payload = hashMap("foo", "bar");
+        ApiResponse postResponse = pathResource(getOrgAppPath("foos")).post(payload);
+        UUID assetId = postResponse.getEntities().get(0).getUuid();
+        assertNotNull(assetId);
+
+        // post asset to that entity
+
+        byte[] data = IOUtils.toByteArray(this.getClass().getResourceAsStream("/cassandra_eye.jpg"));
+        FormDataMultiPart form = new FormDataMultiPart()
+            .field("foo", "bar2")
+            .field("file", data, MediaType.MULTIPART_FORM_DATA_TYPE);
+        ApiResponse putResponse = pathResource(getOrgAppPath("foos/" + assetId)).put(form);
+        this.waitForQueueDrainAndRefreshIndex();
+
+        // get entity and check asset metadata
+
+        ApiResponse getResponse = pathResource(getOrgAppPath("foos/" + assetId)).get(ApiResponse.class);
+        Entity entity = getResponse.getEntities().get(0);
+        Map<String, Object> fileMetadata = (Map<String, Object>) entity.get("file-metadata");
+        long lastModified = Long.parseLong(fileMetadata.get(AssetUtils.LAST_MODIFIED).toString());
+
+        assertEquals(assetId, entity.getUuid());
+        assertEquals("bar2", entity.get("foo"));
+        assertEquals("image/jpeg", fileMetadata.get(AssetUtils.CONTENT_TYPE));
+        assertEquals(7979, fileMetadata.get(AssetUtils.CONTENT_LENGTH));
+
+        // get asset and check size
+
+        InputStream is = pathResource(getOrgAppPath("foos/" + assetId)).getAssetAsStream();
+        byte[] foundData = IOUtils.toByteArray(is);
+        assertEquals(7979, foundData.length);
+
+        // upload new asset to entity, then check that it was updated
+
+        ApiResponse putResponse2 = pathResource(getOrgAppPath("foos/" + assetId)).put(form);
+        entity = putResponse2.getEntities().get(0);
+        fileMetadata = (Map<String, Object>) entity.get("file-metadata");
+        long justModified = Long.parseLong(fileMetadata.get(AssetUtils.LAST_MODIFIED).toString());
+        assertNotEquals(lastModified, justModified);
+    }
+
+
+    @Test
+    public void largeFile() throws Exception {
+
+        this.waitForQueueDrainAndRefreshIndex();
+
+        // upload file larger than 5MB
+
+        byte[] data = IOUtils.toByteArray(this.getClass().getResourceAsStream("/file-bigger-than-5M"));
+        FormDataMultiPart form = new FormDataMultiPart().field("file", data, MediaType.MULTIPART_FORM_DATA_TYPE);
+        ApiResponse postResponse = pathResource(getOrgAppPath("foos")).post(form);
+        UUID assetId = postResponse.getEntities().get(0).getUuid();
+        logger.info("Waiting for upload to finish...");
+        Thread.sleep(5000);
+
+        // check that entire file was uploaded
+
+        ApiResponse getResponse = pathResource(getOrgAppPath("foos/" + assetId)).get(ApiResponse.class);
+        logger.info("Upload complete!");
+        InputStream is = pathResource(getOrgAppPath("foos/" + assetId)).getAssetAsStream();
+        byte[] foundData = IOUtils.toByteArray(is);
+        assertEquals(data.length, foundData.length);
+
+        // delete file
+
+        pathResource(getOrgAppPath("foos/" + assetId)).delete();
+    }
+
+    @Test
+    public void fileTooLargeShouldResultInError() throws Exception {
+
+        this.waitForQueueDrainAndRefreshIndex();
+
+        // set max file size down to 5mb
+        setTestProperty("usergrid.binary.max-size-mb", "5");
+
+        try {
+
+            // upload a file larger than 6mb
+
+            byte[] data = IOUtils.toByteArray(this.getClass().getResourceAsStream("/ship-larger-than-6mb.gif"));
+            FormDataMultiPart form = new FormDataMultiPart().field("file", data, MediaType.MULTIPART_FORM_DATA_TYPE);
+            ApiResponse postResponse = pathResource(getOrgAppPath("bars")).post(form);
+            UUID assetId = postResponse.getEntities().get(0).getUuid();
+
+            String errorMessage = null;
+            logger.info("Waiting for upload to finish...");
+            Thread.sleep(1000);
+
+            // attempt to get asset entity, it should contain error
+
+            waitForQueueDrainAndRefreshIndex();
+            ApiResponse getResponse = pathResource(getOrgAppPath("bars/" + assetId)).get(ApiResponse.class);
+            Map<String, Object> fileMetadata = (Map<String, Object>) getResponse.getEntities().get(0).get("file-metadata");
+            assertNotNull(fileMetadata);
+            assertNotNull(fileMetadata.get("error"));
+            assertTrue(fileMetadata.get("error").toString().startsWith("Asset size "));
+
+        } finally {
+
+            // set max upload size back to default 25mb
+            setTestProperties(originalProperties);
+        }
+    }
+
+    /**
+     * Deleting a connection to an asset should not delete the asset or the asset's data
+     */
+    @Test
+    public void deleteConnectionToAsset() throws IOException {
+
+        this.waitForQueueDrainAndRefreshIndex();
+
+        // create the entity that will be the asset, an image
+
+        Map<String, String> payload = hashMap("name", "cassandra_eye.jpg");
+        ApiResponse postReponse = pathResource(getOrgAppPath("foos")).post(payload);
+        final UUID uuid = postReponse.getEntities().get(0).getUuid();
+
+        // post image data to the asset entity
+
+        byte[] data = IOUtils.toByteArray(this.getClass().getResourceAsStream("/cassandra_eye.jpg"));
+        pathResource(getOrgAppPath("foos/" + uuid)).put(data, MediaType.APPLICATION_OCTET_STREAM_TYPE);
+
+        // create an imagegallery entity
+
+        Map<String, String> imageGalleryPayload = hashMap("name", "my image gallery");
+
+        ApiResponse postResponse2 = pathResource(getOrgAppPath("imagegalleries")).post(imageGalleryPayload);
+        UUID imageGalleryId = postResponse2.getEntities().get(0).getUuid();
+
+        // connect imagegallery to asset
+
+        ApiResponse connectResponse = pathResource(
+            getOrgAppPath("imagegalleries/" + imageGalleryId + "/contains/" + uuid)).post(ApiResponse.class);
+        this.waitForQueueDrainAndRefreshIndex();
+
+        // verify connection from imagegallery to asset
+
+        ApiResponse containsResponse = pathResource(
+            getOrgAppPath("imagegalleries/" + imageGalleryId + "/contains/")).get(ApiResponse.class);
+        assertEquals(uuid, containsResponse.getEntities().get(0).getUuid());
+
+        // delete the connection
+
+        pathResource(getOrgAppPath("imagegalleries/" + imageGalleryId + "/contains/" + uuid)).delete();
+        this.waitForQueueDrainAndRefreshIndex();
+
+        // verify that connection is gone
+
+        ApiResponse listResponse = pathResource(
+            getOrgAppPath("imagegalleries/" + imageGalleryId + "/contains/")).get(ApiResponse.class);
+        assertEquals(0, listResponse.getEntityCount());
+
+        // asset should still be there
+
+        ApiResponse getResponse2 = pathResource(getOrgAppPath("foos/" + uuid)).get(ApiResponse.class);
+        Entity entity = getResponse2.getEntities().get(0);
+        Map<String, Object> fileMetadata = (Map<String, Object>) entity.get("file-metadata");
+
+        Assert.assertEquals("image/jpeg", fileMetadata.get(AssetUtils.CONTENT_TYPE));
+        Assert.assertEquals(7979, fileMetadata.get(AssetUtils.CONTENT_LENGTH));
+        assertEquals(uuid, entity.getUuid());
+
+        // asset data should still be there
+
+        InputStream assetIs = pathResource(getOrgAppPath("foos/" + uuid)).getAssetAsStream();
+        byte[] foundData = IOUtils.toByteArray(assetIs);
+        assertEquals(7979, foundData.length);
+    }
+}

http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/aws/NoAWSCredsRule.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/aws/NoAWSCredsRule.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/aws/NoAWSCredsRule.java
deleted file mode 100644
index 40a6b45..0000000
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/aws/NoAWSCredsRule.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.rest.applications.assets.aws;
-
-
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.SDKGlobalConfiguration;
-import org.apache.usergrid.rest.test.resource.AbstractRestIT;
-import org.apache.usergrid.services.exceptions.AwsPropertiesNotFoundException;
-import org.junit.Assume;
-import org.junit.internal.runners.model.MultipleFailureException;
-import org.junit.rules.TestRule;
-import org.junit.runner.Description;
-import org.junit.runners.model.Statement;
-import org.springframework.beans.factory.annotation.Autowired;
-
-import java.util.Map;
-import java.util.Properties;
-
-
-/**
- * Created in an attempt to mark no aws cred tests as ignored.  Blocked by this issue
- * https://github.com/junit-team/junit/issues/116
- *
- * Until then, simply marks as passed, which is a bit dangerous
- */
-public class NoAWSCredsRule  extends AbstractRestIT implements TestRule {
-
-    @Autowired
-    private Properties properties;
-
-
-    public Statement apply( final Statement base, final Description description ) {
-        return new Statement() {
-            @Override
-            public void evaluate() throws Throwable {
-                String accessId;
-                String secretKey;
-                String bucketName;
-                try {
-                     Map<String,Object> properties = getRemoteTestProperties();
-                    //TODO: GREY change this so that it checks for the properties, then if it doesn't have them, mark the tests as ignored.
-                    accessId = (String)System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
-                    secretKey = (String)System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );
-                    bucketName =(String) properties.get( "usergrid.binary.bucketname" );
-
-                    if(accessId==null||secretKey==null||bucketName==null){
-                        throw new AwsPropertiesNotFoundException( "Access Keys" );
-                    }
-                    base.evaluate();
-
-                }
-                catch ( Throwable t ) {
-
-                    if ( !isMissingCredsException( t ) ) {
-                        throw t;
-                    }
-
-                    //do this so our test gets marked as ignored.  Not pretty, but it works
-                    Assume.assumeTrue( false );
-
-
-                }
-            }
-        };
-    }
-
-
-    private boolean isMissingCredsException( final Throwable t ) {
-
-        if ( t instanceof AmazonClientException ) {
-
-            final AmazonClientException ace = ( AmazonClientException ) t;
-
-            if ( ace.getMessage().contains( "could not get aws access key" ) || ace.getMessage().contains(
-                "could not get aws secret key from system properties" ) ) {
-                //swallow
-                return true;
-            }
-        }
-
-        if( t instanceof AwsPropertiesNotFoundException ){
-            return true;
-        }
-
-        /**
-         * Handle the multiple failure junit trace
-         */
-        if( t instanceof MultipleFailureException ){
-            for(final Throwable failure : ((MultipleFailureException)t).getFailures()){
-                final boolean isMissingCreds = isMissingCredsException( failure );
-
-                if(isMissingCreds){
-                    return true;
-                }
-            }
-        }
-        final Throwable cause = t.getCause();
-
-        if ( cause == null ) {
-            return false;
-        }
-
-
-        return isMissingCredsException( cause );
-    }
-}

http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/rules/NoAWSCredsRule.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/rules/NoAWSCredsRule.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/rules/NoAWSCredsRule.java
new file mode 100644
index 0000000..8606e4e
--- /dev/null
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/rules/NoAWSCredsRule.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.rest.applications.assets.rules;
+
+
+import com.amazonaws.AmazonClientException;
+import com.amazonaws.SDKGlobalConfiguration;
+import org.apache.usergrid.rest.test.resource.AbstractRestIT;
+import org.apache.usergrid.services.exceptions.AwsPropertiesNotFoundException;
+import org.junit.Assume;
+import org.junit.internal.runners.model.MultipleFailureException;
+import org.junit.rules.TestRule;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.io.FileInputStream;
+import java.util.Map;
+import java.util.Properties;
+
+
+/**
+ * Created in an attempt to mark no aws cred tests as ignored.  Blocked by this issue
+ * https://github.com/junit-team/junit/issues/116
+ *
+ * Until then, simply marks as passed, which is a bit dangerous
+ */
+public class NoAWSCredsRule extends AbstractRestIT implements TestRule {
+
+    private static final Logger logger = LoggerFactory.getLogger( NoAWSCredsRule.class );
+
+
+    private Properties properties;
+
+
+    public Statement apply( final Statement base, final Description description ) {
+        return new Statement() {
+            @Override
+            public void evaluate() throws Throwable {
+
+
+                try {
+
+                    final Map<String,Object> properties = getRemoteTestProperties();
+                    final String bucketName = (String)properties.get( "usergrid.binary.bucketname" );
+
+
+                    //TODO: GREY change this so that it checks for the properties, then if it doesn't have them, mark the tests as ignored.
+                    final String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
+                    final String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );
+
+                    if(accessId==null||secretKey==null||bucketName==null){
+                        throw new AwsPropertiesNotFoundException( "Access Keys" );
+                    }
+
+
+
+
+
+                    base.evaluate();
+
+                }
+                catch ( Throwable t ) {
+
+                    if ( !isMissingCredsException( t ) ) {
+                        throw t;
+                    }
+
+                    //do this so our test gets marked as ignored.  Not pretty, but it works
+                    Assume.assumeTrue( false );
+
+
+                }
+            }
+        };
+    }
+
+
+    private boolean isMissingCredsException( final Throwable t ) {
+
+        if ( t instanceof AmazonClientException ) {
+
+            final AmazonClientException ace = ( AmazonClientException ) t;
+
+            if ( ace.getMessage().contains( "could not get aws access key" ) || ace.getMessage().contains(
+                "could not get aws secret key from system properties" ) ) {
+                //swallow
+                return true;
+            }
+        }
+
+        if( t instanceof AwsPropertiesNotFoundException ){
+            return true;
+        }
+
+        /**
+         * Handle the multiple failure junit trace
+         */
+        if( t instanceof MultipleFailureException ){
+            for(final Throwable failure : ((MultipleFailureException)t).getFailures()){
+                final boolean isMissingCreds = isMissingCredsException( failure );
+
+                if(isMissingCreds){
+                    return true;
+                }
+            }
+        }
+        final Throwable cause = t.getCause();
+
+        if ( cause == null ) {
+            return false;
+        }
+
+
+        return isMissingCredsException( cause );
+    }
+}

http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/rules/NoGoogleCredsRule.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/rules/NoGoogleCredsRule.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/rules/NoGoogleCredsRule.java
new file mode 100644
index 0000000..5e3f66e
--- /dev/null
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/rules/NoGoogleCredsRule.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.rest.applications.assets.rules;
+
+
+import com.amazonaws.AmazonClientException;
+import com.amazonaws.SDKGlobalConfiguration;
+import org.apache.usergrid.rest.test.resource.AbstractRestIT;
+import org.apache.usergrid.services.exceptions.AwsPropertiesNotFoundException;
+import org.junit.Assume;
+import org.junit.internal.runners.model.MultipleFailureException;
+import org.junit.rules.TestRule;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.util.Map;
+import java.util.Properties;
+
+
+/**
+ * Created in an attempt to mark no Google cred tests as ignored.  Blocked by this issue
+ * https://github.com/junit-team/junit/issues/116
+ *
+ * Until then, simply marks as passed, which is a bit dangerous
+ */
+public class NoGoogleCredsRule extends AbstractRestIT implements TestRule {
+
+    private static final Logger logger = LoggerFactory.getLogger( NoGoogleCredsRule.class );
+
+
+    @Autowired
+    private Properties properties;
+
+
+    public Statement apply( final Statement base, final Description description ) {
+        return new Statement() {
+            @Override
+            public void evaluate() throws Throwable {
+
+
+                try {
+
+                    final String filename = System.getenv( "GOOGLE_APPLICATION_CREDENTIALS" );
+                    logger.info("cred filename: {}", filename);
+                    // if the file doesn't exist, an exception will be thrown
+                    new FileInputStream(filename);
+
+                    base.evaluate();
+
+                }
+                catch ( Throwable t ) {
+
+                    if ( !isMissingCredsException( t ) ) {
+                        throw t;
+                    }
+
+                    //do this so our test gets marked as ignored.  Not pretty, but it works
+                    Assume.assumeTrue( false );
+
+
+                }
+            }
+        };
+    }
+
+
+    private boolean isMissingCredsException( final Throwable t ) {
+
+        // either no filename was provided or the filename provided doesn't actually exist on the file system
+        if ( t instanceof FileNotFoundException || t instanceof NullPointerException ) {
+                return true;
+            }
+
+
+        /**
+         * Handle the multiple failure junit trace
+         */
+        if( t instanceof MultipleFailureException ){
+            for(final Throwable failure : ((MultipleFailureException)t).getFailures()){
+                final boolean isMissingCreds = isMissingCredsException( failure );
+
+                if(isMissingCreds){
+                    return true;
+                }
+            }
+        }
+        final Throwable cause = t.getCause();
+
+        if ( cause == null ) {
+            return false;
+        }
+
+
+        return isMissingCredsException( cause );
+    }
+}

http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/services/pom.xml
----------------------------------------------------------------------
diff --git a/stack/services/pom.xml b/stack/services/pom.xml
index 8cd1756..b1df1b4 100644
--- a/stack/services/pom.xml
+++ b/stack/services/pom.xml
@@ -267,6 +267,12 @@
         </dependency>
 
         <dependency>
+            <groupId>com.google.cloud</groupId>
+            <artifactId>google-cloud-storage</artifactId>
+            <version>0.11.0-beta</version>
+        </dependency>
+
+        <dependency>
             <groupId>org.apache.jclouds</groupId>
             <artifactId>jclouds-blobstore</artifactId>
         </dependency>

http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/services/src/main/java/org/apache/usergrid/services/assets/BinaryStoreFactory.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/assets/BinaryStoreFactory.java b/stack/services/src/main/java/org/apache/usergrid/services/assets/BinaryStoreFactory.java
new file mode 100644
index 0000000..bd91dba
--- /dev/null
+++ b/stack/services/src/main/java/org/apache/usergrid/services/assets/BinaryStoreFactory.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.services.assets;
+
+import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.services.assets.data.AWSBinaryStore;
+import org.apache.usergrid.services.assets.data.BinaryStore;
+import org.apache.usergrid.services.assets.data.GoogleBinaryStore;
+import org.apache.usergrid.services.assets.data.LocalFileBinaryStore;
+
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.Properties;
+
+import static org.apache.usergrid.management.AccountCreationProps.PROPERTIES_USERGRID_BINARY_UPLOADER;
+
+
+public class BinaryStoreFactory {
+
+    public enum Provider{
+        local,aws,google
+    }
+
+    private EntityManagerFactory entityManagerFactory;
+    private Properties properties;
+    private String reposLocation;
+    private LocalFileBinaryStore localFileBinaryStore;
+    private AWSBinaryStore awsBinaryStore;
+    private GoogleBinaryStore googleCloudStorageBinaryStore;
+
+    public BinaryStoreFactory(Properties properties, EntityManagerFactory entityManagerFactory, String reposLocation) throws IOException, GeneralSecurityException {
+        this.properties = properties;
+        this.entityManagerFactory = entityManagerFactory;
+        this.reposLocation = reposLocation;
+        this.localFileBinaryStore = new LocalFileBinaryStore(properties, entityManagerFactory, reposLocation);
+        this.awsBinaryStore = new AWSBinaryStore(properties, entityManagerFactory, reposLocation);
+        this.googleCloudStorageBinaryStore = new GoogleBinaryStore(properties, entityManagerFactory, reposLocation);
+    }
+
+    public synchronized BinaryStore getBinaryStore(String provider) throws IOException, GeneralSecurityException {
+
+        provider = provider != null? provider.toLowerCase(): "";
+
+        if( provider.isEmpty() ){
+
+            if(properties.getProperty( PROPERTIES_USERGRID_BINARY_UPLOADER ).equals( Provider.local.name() )){
+                return localFileBinaryStore;
+            } else if (properties.getProperty( PROPERTIES_USERGRID_BINARY_UPLOADER ).equals( Provider.google.name() )){
+                return googleCloudStorageBinaryStore;
+            } else{
+                return awsBinaryStore;
+            }
+        }
+
+        if ( provider.equals(Provider.local.name())){
+            return localFileBinaryStore;
+        }
+        if ( provider.equals(Provider.google.name())){
+            return googleCloudStorageBinaryStore;
+        }
+        if( provider.equals(Provider.aws.name())){
+            return awsBinaryStore;
+        }
+
+        // this for backwards compatibility because historically anything other than "local" meant AWS
+        return awsBinaryStore;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/services/src/main/java/org/apache/usergrid/services/assets/data/AWSBinaryStore.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/assets/data/AWSBinaryStore.java b/stack/services/src/main/java/org/apache/usergrid/services/assets/data/AWSBinaryStore.java
new file mode 100644
index 0000000..14da2e1
--- /dev/null
+++ b/stack/services/src/main/java/org/apache/usergrid/services/assets/data/AWSBinaryStore.java
@@ -0,0 +1,320 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.services.assets.data;
+
+
+import java.io.BufferedOutputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Properties;
+import java.util.UUID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.persistence.queue.impl.UsergridAwsCredentialsProvider;
+import org.apache.usergrid.services.exceptions.AwsPropertiesNotFoundException;
+import org.apache.usergrid.utils.StringUtils;
+
+import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.Protocol;
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.regions.Region;
+import com.amazonaws.regions.Regions;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.model.AbortMultipartUploadRequest;
+import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
+import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
+import com.amazonaws.services.s3.model.DeleteObjectRequest;
+import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
+import com.amazonaws.services.s3.model.InitiateMultipartUploadResult;
+import com.amazonaws.services.s3.model.ListMultipartUploadsRequest;
+import com.amazonaws.services.s3.model.MultipartUpload;
+import com.amazonaws.services.s3.model.MultipartUploadListing;
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import com.amazonaws.services.s3.model.PartETag;
+import com.amazonaws.services.s3.model.PutObjectResult;
+import com.amazonaws.services.s3.model.S3Object;
+import com.amazonaws.services.s3.model.UploadPartRequest;
+import com.google.common.primitives.Ints;
+import java.io.ByteArrayInputStream;
+import java.io.FileInputStream;
+import java.io.PushbackInputStream;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.commons.codec.binary.Base64;
+
+
+public class AWSBinaryStore implements BinaryStore {
+
+    private static final Logger logger = LoggerFactory.getLogger(AWSBinaryStore.class );
+    private static final long FIVE_MB = ( FileUtils.ONE_MB * 5 );
+
+    private AmazonS3 s3Client;
+    private String accessId;
+    private String secretKey;
+    private String bucketName;
+    private String regionName;
+
+    private EntityManagerFactory emf;
+    private Properties properties;
+    private String reposLocation;
+
+    public AWSBinaryStore(Properties properties,
+                          EntityManagerFactory entityManagerFactory,
+                          String reposLocation) {
+        this.properties = properties;
+        this.emf = entityManagerFactory;
+        this.reposLocation = reposLocation;
+
+    }
+
+    //TODO: GREY rework how the s3 client works because currently it handles initlization and returning of the client
+    //ideally it should only do one. and the client should be initlized at the beginning of the run.
+    private AmazonS3 getS3Client() throws Exception{
+
+        this.bucketName = properties.getProperty( "usergrid.binary.bucketname" );
+        if(bucketName == null){
+            logger.error( "usergrid.binary.bucketname not properly set so amazon bucket is null" );
+            throw new AwsPropertiesNotFoundException( "usergrid.binary.bucketname" );
+
+        }
+
+        final UsergridAwsCredentialsProvider ugProvider = new UsergridAwsCredentialsProvider();
+        AWSCredentials credentials = ugProvider.getCredentials();
+        ClientConfiguration clientConfig = new ClientConfiguration();
+        clientConfig.setProtocol(Protocol.HTTP);
+
+        s3Client = new AmazonS3Client(credentials, clientConfig);
+        if(regionName != null)
+            s3Client.setRegion( Region.getRegion(Regions.fromName(regionName)) );
+
+        return s3Client;
+    }
+
+    @Override
+    public void write( final UUID appId, final Entity entity, InputStream inputStream ) throws Exception {
+
+        String uploadFileName = AssetUtils.buildAssetKey( appId, entity );
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        long written = IOUtils.copyLarge( inputStream, baos, 0, FIVE_MB );
+
+        byte[] data = baos.toByteArray();
+
+        InputStream awsInputStream = new ByteArrayInputStream(data);
+
+        final Map<String, Object> fileMetadata = AssetUtils.getFileMetadata( entity );
+        fileMetadata.put( AssetUtils.LAST_MODIFIED, System.currentTimeMillis() );
+
+        String mimeType = AssetMimeHandler.get().getMimeType( entity, data );
+
+        Boolean overSizeLimit = false;
+
+        EntityManager em = emf.getEntityManager( appId );
+
+
+        if ( written < FIVE_MB ) { // total smaller than 5mb
+
+            ObjectMetadata om = new ObjectMetadata();
+            om.setContentLength(written);
+            om.setContentType( mimeType );
+            PutObjectResult result = null;
+            result = getS3Client().putObject( bucketName, uploadFileName, awsInputStream, om );
+
+            String md5sum = Hex.encodeHexString( Base64.decodeBase64(result.getContentMd5()) );
+            String eTag = result.getETag();
+
+            fileMetadata.put( AssetUtils.CONTENT_LENGTH, written );
+
+            if(md5sum != null)
+                fileMetadata.put( AssetUtils.CHECKSUM, md5sum );
+            fileMetadata.put( AssetUtils.E_TAG, eTag );
+
+            em.update( entity );
+
+        }
+        else { // bigger than 5mb... dump 5 mb tmp files and upload from them
+            written = 0; //reset written to 0, we still haven't wrote anything in fact
+            int partNumber = 1;
+            int firstByte = 0;
+            Boolean isFirstChunck = true;
+            List<PartETag> partETags = new ArrayList<PartETag>();
+
+
+            //get the s3 client in order to initialize the multipart request
+            getS3Client();
+            InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest( bucketName, uploadFileName );
+            InitiateMultipartUploadResult initResponse = getS3Client().initiateMultipartUpload( initRequest );
+
+
+            InputStream firstChunck = new ByteArrayInputStream(data);
+            PushbackInputStream chunckableInputStream = new PushbackInputStream(inputStream, 1);
+
+            // determine max size file allowed, default to 50mb
+            long maxSizeBytes = 50 * FileUtils.ONE_MB;
+            String maxSizeMbString = properties.getProperty( "usergrid.binary.max-size-mb", "50" );
+            if ( StringUtils.isNumeric( maxSizeMbString )) {
+                maxSizeBytes = Long.parseLong( maxSizeMbString ) * FileUtils.ONE_MB;
+            }
+
+            // always allow files up to 5mb
+            if (maxSizeBytes < 5 * FileUtils.ONE_MB ) {
+                maxSizeBytes = 5 * FileUtils.ONE_MB;
+            }
+
+            while (-1 != (firstByte = chunckableInputStream.read())) {
+                long partSize = 0;
+                chunckableInputStream.unread(firstByte);
+                File tempFile = File.createTempFile( entity.getUuid().toString().concat("-part").concat(String.valueOf(partNumber)), "tmp" );
+
+                tempFile.deleteOnExit();
+                OutputStream os = null;
+                try {
+                    os = new BufferedOutputStream( new FileOutputStream( tempFile.getAbsolutePath() ) );
+
+                    if(isFirstChunck == true) {
+                        partSize = IOUtils.copyLarge( firstChunck, os, 0, ( FIVE_MB ) );
+                        isFirstChunck = false;
+                    }
+                    else {
+                        partSize = IOUtils.copyLarge( chunckableInputStream, os, 0, ( FIVE_MB ) );
+                    }
+                    written += partSize;
+
+                    if(written> maxSizeBytes){
+                        overSizeLimit = true;
+                        logger.error( "OVERSIZED FILE ({}). STARTING ABORT", written );
+                        break;
+                        //set flag here and break out of loop to run abort
+                    }
+                }
+                finally {
+                    IOUtils.closeQuietly( os );
+                }
+
+                FileInputStream chunk = new FileInputStream(tempFile);
+
+                Boolean isLastPart = -1 == (firstByte = chunckableInputStream.read());
+                if(!isLastPart)
+                    chunckableInputStream.unread(firstByte);
+
+                UploadPartRequest uploadRequest = new UploadPartRequest().withUploadId(initResponse.getUploadId())
+                                                                         .withBucketName(bucketName)
+                                                                         .withKey(uploadFileName)
+                                                                         .withInputStream(chunk)
+                                                                         .withPartNumber(partNumber)
+                                                                         .withPartSize(partSize)
+                                                                         .withLastPart(isLastPart);
+                partETags.add( getS3Client().uploadPart(uploadRequest).getPartETag() );
+                partNumber++;
+            }
+
+            //check for flag here then abort.
+            if(overSizeLimit) {
+
+                AbortMultipartUploadRequest abortRequest =
+                    new AbortMultipartUploadRequest( bucketName, uploadFileName, initResponse.getUploadId() );
+
+                ListMultipartUploadsRequest listRequest = new ListMultipartUploadsRequest( bucketName );
+
+                MultipartUploadListing listResult = getS3Client().listMultipartUploads( listRequest );
+
+                //upadte the entity with the error.
+                try {
+                    logger.error("starting update of entity due to oversized asset");
+                    fileMetadata.put( "error", "Asset size is larger than max size of " + maxSizeBytes );
+                    em.update( entity );
+                }
+                catch ( Exception e ) {
+                    logger.error( "Error updating entity with error message", e );
+                }
+
+                int timesIterated = 20;
+                //loop and abort all the multipart uploads
+                while ( listResult.getMultipartUploads().size()!=0  && timesIterated > 0) {
+
+                    getS3Client().abortMultipartUpload( abortRequest );
+                    Thread.sleep( 1000 );
+                    timesIterated--;
+                    listResult = getS3Client().listMultipartUploads( listRequest );
+                    if (logger.isDebugEnabled()) {
+                        logger.debug("Files that haven't been aborted are: {}", listResult.getMultipartUploads().listIterator().toString());
+                    }
+
+                }
+                if ( timesIterated == 0 ){
+                    logger.error( "Files parts that couldn't be aborted in 20 seconds are:" );
+                    Iterator<MultipartUpload> multipartUploadIterator = listResult.getMultipartUploads().iterator();
+                    while(multipartUploadIterator.hasNext()){
+                        logger.error( multipartUploadIterator.next().getKey() );
+                    }
+                }
+            }
+            else {
+                CompleteMultipartUploadRequest request =
+                    new CompleteMultipartUploadRequest( bucketName, uploadFileName, initResponse.getUploadId(),
+                        partETags );
+                CompleteMultipartUploadResult amazonResult = getS3Client().completeMultipartUpload( request );
+                fileMetadata.put( AssetUtils.CONTENT_LENGTH, written );
+                fileMetadata.put( AssetUtils.E_TAG, amazonResult.getETag() );
+                em.update( entity );
+            }
+        }
+    }
+
+
+    @Override
+    public InputStream read( UUID appId, Entity entity, long offset, long length ) throws Exception {
+
+        S3Object object = getS3Client().getObject( bucketName, AssetUtils.buildAssetKey( appId, entity ) );
+
+        byte data[] = null;
+
+        if ( offset == 0 && length == FIVE_MB ) {
+            return object.getObjectContent();
+        }
+        else {
+            object.getObjectContent().read(data, Ints.checkedCast(offset), Ints.checkedCast(length));
+        }
+
+        return new ByteArrayInputStream(data);
+    }
+
+
+    @Override
+    public InputStream read( UUID appId, Entity entity ) throws Exception {
+        return read( appId, entity, 0, FIVE_MB );
+    }
+
+
+    @Override
+    public void delete( UUID appId, Entity entity ) throws Exception {
+        getS3Client().deleteObject(new DeleteObjectRequest(bucketName, AssetUtils.buildAssetKey( appId, entity )));
+    }
+}

http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/services/src/main/java/org/apache/usergrid/services/assets/data/AwsSdkS3BinaryStore.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/assets/data/AwsSdkS3BinaryStore.java b/stack/services/src/main/java/org/apache/usergrid/services/assets/data/AwsSdkS3BinaryStore.java
deleted file mode 100644
index 5f2c041..0000000
--- a/stack/services/src/main/java/org/apache/usergrid/services/assets/data/AwsSdkS3BinaryStore.java
+++ /dev/null
@@ -1,317 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.usergrid.services.assets.data;
-
-
-import java.io.BufferedOutputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Properties;
-import java.util.UUID;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.apache.usergrid.persistence.Entity;
-import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.EntityManagerFactory;
-import org.apache.usergrid.persistence.queue.impl.UsergridAwsCredentialsProvider;
-import org.apache.usergrid.services.exceptions.AwsPropertiesNotFoundException;
-import org.apache.usergrid.utils.StringUtils;
-
-import org.apache.commons.codec.binary.Hex;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOUtils;
-
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.Protocol;
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.regions.Region;
-import com.amazonaws.regions.Regions;
-import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.AmazonS3Client;
-import com.amazonaws.services.s3.model.AbortMultipartUploadRequest;
-import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
-import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
-import com.amazonaws.services.s3.model.DeleteObjectRequest;
-import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
-import com.amazonaws.services.s3.model.InitiateMultipartUploadResult;
-import com.amazonaws.services.s3.model.ListMultipartUploadsRequest;
-import com.amazonaws.services.s3.model.MultipartUpload;
-import com.amazonaws.services.s3.model.MultipartUploadListing;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.model.PartETag;
-import com.amazonaws.services.s3.model.PutObjectResult;
-import com.amazonaws.services.s3.model.S3Object;
-import com.amazonaws.services.s3.model.UploadPartRequest;
-import com.google.common.primitives.Ints;
-import java.io.ByteArrayInputStream;
-import java.io.FileInputStream;
-import java.io.PushbackInputStream;
-import java.util.ArrayList;
-import java.util.List;
-import org.apache.commons.codec.binary.Base64;
-
-
-public class  AwsSdkS3BinaryStore implements BinaryStore {
-
-    private static final Logger logger = LoggerFactory.getLogger(AwsSdkS3BinaryStore.class );
-    private static final long FIVE_MB = ( FileUtils.ONE_MB * 5 );
-
-    private AmazonS3 s3Client;
-    private String accessId;
-    private String secretKey;
-    private String bucketName;
-    private String regionName;
-
-    @Autowired
-    private EntityManagerFactory emf;
-
-    @Autowired
-    private Properties properties;
-
-    public AwsSdkS3BinaryStore( ) {
-    }
-
-    //TODO: GREY rework how the s3 client works because currently it handles initlization and returning of the client
-    //ideally it should only do one. and the client should be initlized at the beginning of the run.
-    private AmazonS3 getS3Client() throws Exception{
-
-        this.bucketName = properties.getProperty( "usergrid.binary.bucketname" );
-        if(bucketName == null){
-            logger.error( "usergrid.binary.bucketname not properly set so amazon bucket is null" );
-            throw new AwsPropertiesNotFoundException( "usergrid.binary.bucketname" );
-
-        }
-
-        final UsergridAwsCredentialsProvider ugProvider = new UsergridAwsCredentialsProvider();
-        AWSCredentials credentials = ugProvider.getCredentials();
-        ClientConfiguration clientConfig = new ClientConfiguration();
-        clientConfig.setProtocol(Protocol.HTTP);
-
-        s3Client = new AmazonS3Client(credentials, clientConfig);
-        if(regionName != null)
-            s3Client.setRegion( Region.getRegion(Regions.fromName(regionName)) );
-
-        return s3Client;
-    }
-
-    @Override
-    public void write( final UUID appId, final Entity entity, InputStream inputStream ) throws Exception {
-
-        String uploadFileName = AssetUtils.buildAssetKey( appId, entity );
-        ByteArrayOutputStream baos = new ByteArrayOutputStream();
-        long written = IOUtils.copyLarge( inputStream, baos, 0, FIVE_MB );
-
-        byte[] data = baos.toByteArray();
-
-        InputStream awsInputStream = new ByteArrayInputStream(data);
-
-        final Map<String, Object> fileMetadata = AssetUtils.getFileMetadata( entity );
-        fileMetadata.put( AssetUtils.LAST_MODIFIED, System.currentTimeMillis() );
-
-        String mimeType = AssetMimeHandler.get().getMimeType( entity, data );
-
-        Boolean overSizeLimit = false;
-
-        EntityManager em = emf.getEntityManager( appId );
-
-
-        if ( written < FIVE_MB ) { // total smaller than 5mb
-
-            ObjectMetadata om = new ObjectMetadata();
-            om.setContentLength(written);
-            om.setContentType( mimeType );
-            PutObjectResult result = null;
-            result = getS3Client().putObject( bucketName, uploadFileName, awsInputStream, om );
-
-            String md5sum = Hex.encodeHexString( Base64.decodeBase64(result.getContentMd5()) );
-            String eTag = result.getETag();
-
-            fileMetadata.put( AssetUtils.CONTENT_LENGTH, written );
-
-            if(md5sum != null)
-                fileMetadata.put( AssetUtils.CHECKSUM, md5sum );
-            fileMetadata.put( AssetUtils.E_TAG, eTag );
-
-            em.update( entity );
-
-        }
-        else { // bigger than 5mb... dump 5 mb tmp files and upload from them
-            written = 0; //reset written to 0, we still haven't wrote anything in fact
-            int partNumber = 1;
-            int firstByte = 0;
-            Boolean isFirstChunck = true;
-            List<PartETag> partETags = new ArrayList<PartETag>();
-
-
-            //get the s3 client in order to initialize the multipart request
-            getS3Client();
-            InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest( bucketName, uploadFileName );
-            InitiateMultipartUploadResult initResponse = getS3Client().initiateMultipartUpload( initRequest );
-
-
-            InputStream firstChunck = new ByteArrayInputStream(data);
-            PushbackInputStream chunckableInputStream = new PushbackInputStream(inputStream, 1);
-
-            // determine max size file allowed, default to 50mb
-            long maxSizeBytes = 50 * FileUtils.ONE_MB;
-            String maxSizeMbString = properties.getProperty( "usergrid.binary.max-size-mb", "50" );
-            if ( StringUtils.isNumeric( maxSizeMbString )) {
-                maxSizeBytes = Long.parseLong( maxSizeMbString ) * FileUtils.ONE_MB;
-            }
-
-            // always allow files up to 5mb
-            if (maxSizeBytes < 5 * FileUtils.ONE_MB ) {
-                maxSizeBytes = 5 * FileUtils.ONE_MB;
-            }
-
-            while (-1 != (firstByte = chunckableInputStream.read())) {
-                long partSize = 0;
-                chunckableInputStream.unread(firstByte);
-                File tempFile = File.createTempFile( entity.getUuid().toString().concat("-part").concat(String.valueOf(partNumber)), "tmp" );
-
-                tempFile.deleteOnExit();
-                OutputStream os = null;
-                try {
-                    os = new BufferedOutputStream( new FileOutputStream( tempFile.getAbsolutePath() ) );
-
-                    if(isFirstChunck == true) {
-                        partSize = IOUtils.copyLarge( firstChunck, os, 0, ( FIVE_MB ) );
-                        isFirstChunck = false;
-                    }
-                    else {
-                        partSize = IOUtils.copyLarge( chunckableInputStream, os, 0, ( FIVE_MB ) );
-                    }
-                    written += partSize;
-
-                    if(written> maxSizeBytes){
-                        overSizeLimit = true;
-                        logger.error( "OVERSIZED FILE ({}). STARTING ABORT", written );
-                        break;
-                        //set flag here and break out of loop to run abort
-                    }
-                }
-                finally {
-                    IOUtils.closeQuietly( os );
-                }
-
-                FileInputStream chunk = new FileInputStream(tempFile);
-
-                Boolean isLastPart = -1 == (firstByte = chunckableInputStream.read());
-                if(!isLastPart)
-                    chunckableInputStream.unread(firstByte);
-
-                UploadPartRequest uploadRequest = new UploadPartRequest().withUploadId(initResponse.getUploadId())
-                                                                         .withBucketName(bucketName)
-                                                                         .withKey(uploadFileName)
-                                                                         .withInputStream(chunk)
-                                                                         .withPartNumber(partNumber)
-                                                                         .withPartSize(partSize)
-                                                                         .withLastPart(isLastPart);
-                partETags.add( getS3Client().uploadPart(uploadRequest).getPartETag() );
-                partNumber++;
-            }
-
-            //check for flag here then abort.
-            if(overSizeLimit) {
-
-                AbortMultipartUploadRequest abortRequest =
-                    new AbortMultipartUploadRequest( bucketName, uploadFileName, initResponse.getUploadId() );
-
-                ListMultipartUploadsRequest listRequest = new ListMultipartUploadsRequest( bucketName );
-
-                MultipartUploadListing listResult = getS3Client().listMultipartUploads( listRequest );
-
-                //upadte the entity with the error.
-                try {
-                    logger.error("starting update of entity due to oversized asset");
-                    fileMetadata.put( "error", "Asset size is larger than max size of " + maxSizeBytes );
-                    em.update( entity );
-                }
-                catch ( Exception e ) {
-                    logger.error( "Error updating entity with error message", e );
-                }
-
-                int timesIterated = 20;
-                //loop and abort all the multipart uploads
-                while ( listResult.getMultipartUploads().size()!=0  && timesIterated > 0) {
-
-                    getS3Client().abortMultipartUpload( abortRequest );
-                    Thread.sleep( 1000 );
-                    timesIterated--;
-                    listResult = getS3Client().listMultipartUploads( listRequest );
-                    if (logger.isDebugEnabled()) {
-                        logger.debug("Files that haven't been aborted are: {}", listResult.getMultipartUploads().listIterator().toString());
-                    }
-
-                }
-                if ( timesIterated == 0 ){
-                    logger.error( "Files parts that couldn't be aborted in 20 seconds are:" );
-                    Iterator<MultipartUpload> multipartUploadIterator = listResult.getMultipartUploads().iterator();
-                    while(multipartUploadIterator.hasNext()){
-                        logger.error( multipartUploadIterator.next().getKey() );
-                    }
-                }
-            }
-            else {
-                CompleteMultipartUploadRequest request =
-                    new CompleteMultipartUploadRequest( bucketName, uploadFileName, initResponse.getUploadId(),
-                        partETags );
-                CompleteMultipartUploadResult amazonResult = getS3Client().completeMultipartUpload( request );
-                fileMetadata.put( AssetUtils.CONTENT_LENGTH, written );
-                fileMetadata.put( AssetUtils.E_TAG, amazonResult.getETag() );
-                em.update( entity );
-            }
-        }
-    }
-
-
-    @Override
-    public InputStream read( UUID appId, Entity entity, long offset, long length ) throws Exception {
-
-        S3Object object = getS3Client().getObject( bucketName, AssetUtils.buildAssetKey( appId, entity ) );
-
-        byte data[] = null;
-
-        if ( offset == 0 && length == FIVE_MB ) {
-            return object.getObjectContent();
-        }
-        else {
-            object.getObjectContent().read(data, Ints.checkedCast(offset), Ints.checkedCast(length));
-        }
-
-        return new ByteArrayInputStream(data);
-    }
-
-
-    @Override
-    public InputStream read( UUID appId, Entity entity ) throws Exception {
-        return read( appId, entity, 0, FIVE_MB );
-    }
-
-
-    @Override
-    public void delete( UUID appId, Entity entity ) throws Exception {
-        getS3Client().deleteObject(new DeleteObjectRequest(bucketName, AssetUtils.buildAssetKey( appId, entity )));
-    }
-}


Mime
View raw message