usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From g...@apache.org
Subject incubator-usergrid git commit: Adding properties check for the asset service endpoints for testing purposes only changed resources to include both local and was endpoint . Increased timeout for loading large files into s3. Removed binary store informatio
Date Thu, 23 Jul 2015 19:08:11 GMT
Repository: incubator-usergrid
Updated Branches:
  refs/heads/USERGRID-869 c8d616a90 -> 050aa0a03


Adding properties check for the asset service endpoints for testing purposes only
changed resources to include both local and was endpoint .
Increased timeout for loading large files into s3.
Removed binary store information from test context as regular context supports both deployments
Added log messages to AwsSdkBinaryStore


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/050aa0a0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/050aa0a0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/050aa0a0

Branch: refs/heads/USERGRID-869
Commit: 050aa0a035fffbb202a45251e48b542d3053a655
Parents: c8d616a
Author: GERey <greyes@apigee.com>
Authored: Thu Jul 23 12:08:08 2015 -0700
Committer: GERey <greyes@apigee.com>
Committed: Thu Jul 23 12:08:08 2015 -0700

----------------------------------------------------------------------
 .../rest/applications/ServiceResource.java      | 26 +++++++++++++++++
 .../main/resources/usergrid-rest-context.xml    | 14 ++++++++-
 .../applications/assets/AwsAssetResourceIT.java | 30 ++++++++++----------
 .../test/resources/usergrid-test-context.xml    |  8 ------
 .../assets/data/AwsSdkS3BinaryStore.java        |  6 ++--
 .../test/resources/usergrid-test-context.xml    |  2 +-
 6 files changed, 59 insertions(+), 27 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/050aa0a0/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
index fc729d4..bd5d89f 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
@@ -83,6 +83,8 @@ import com.sun.jersey.multipart.FormDataMultiPart;
 
 import static javax.ws.rs.core.MediaType.APPLICATION_JSON_TYPE;
 
+import static org.apache.usergrid.management.AccountCreationProps.PROPERTIES_USERGRID_BINARY_UPLOADER;
+
 
 @Component
 @Scope("prototype")
@@ -582,6 +584,14 @@ public class ServiceResource extends AbstractContextResource {
     private JSONWithPadding executeMultiPart( UriInfo ui, String callback, FormDataMultiPart
multiPart,
                                               ServiceAction serviceAction ) throws Exception
{
 
+        //needed for testing
+        if(properties.getProperty( PROPERTIES_USERGRID_BINARY_UPLOADER ).equals( "local"
)){
+            this.binaryStore = localFileBinaryStore;
+        }
+        else{
+            this.binaryStore = awsSdkS3BinaryStore;
+        }
+
         // collect form data values
         List<BodyPart> bodyParts = multiPart.getBodyParts();
         HashMap<String, Object> data = new HashMap<String, Object>();
@@ -638,6 +648,14 @@ public class ServiceResource extends AbstractContextResource {
     @Consumes(MediaType.APPLICATION_OCTET_STREAM)
     public Response uploadDataStream( @Context UriInfo ui, InputStream uploadedInputStream
) throws Exception {
 
+        //needed for testing
+        if(properties.getProperty( PROPERTIES_USERGRID_BINARY_UPLOADER ).equals( "local"
)){
+            this.binaryStore = localFileBinaryStore;
+        }
+        else{
+            this.binaryStore = awsSdkS3BinaryStore;
+        }
+
         ApiResponse response = createApiResponse();
         response.setAction( "get" );
         response.setApplication( services.getApplication() );
@@ -662,6 +680,14 @@ public class ServiceResource extends AbstractContextResource {
 
         LOG.debug( "ServiceResource.executeStreamGet" );
 
+        //Needed for testing
+        if(properties.getProperty( PROPERTIES_USERGRID_BINARY_UPLOADER ).equals( "local"
)){
+            this.binaryStore = localFileBinaryStore;
+        }
+        else{
+            this.binaryStore = awsSdkS3BinaryStore;
+        }
+
         ApiResponse response = createApiResponse();
         response.setAction( "get" );
         response.setApplication( services.getApplication() );

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/050aa0a0/stack/rest/src/main/resources/usergrid-rest-context.xml
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/resources/usergrid-rest-context.xml b/stack/rest/src/main/resources/usergrid-rest-context.xml
index 16f4541..438cd1c 100644
--- a/stack/rest/src/main/resources/usergrid-rest-context.xml
+++ b/stack/rest/src/main/resources/usergrid-rest-context.xml
@@ -65,8 +65,20 @@
 		<property name="queueCapacity" value="25" />
 	</bean>
 
-    <bean id="binaryStore" class="org.apache.usergrid.services.assets.data.LocalFileBinaryStore">
+    <bean id="serviceResource"
+           class="org.apache.usergrid.rest.applications.ServiceResource" scope="prototype">
+        <property name="binaryStore" value="${usergrid.binary.uploader}"/>
+    </bean>
+
+    <bean id="localFileBinaryStore" class="org.apache.usergrid.services.assets.data.LocalFileBinaryStore">
         <property name="reposLocation" value="${usergrid.temp.files}"/>
     </bean>
 
+    <bean id="awsS3BinaryStore" class="org.apache.usergrid.services.assets.data.AwsSdkS3BinaryStore">
+        <constructor-arg name="accessId" value="xx" />
+        <constructor-arg name="secretKey" value="xx" />
+        <constructor-arg name="bucketName" value="xx" />
+    </bean>
+
+
 </beans>

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/050aa0a0/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AwsAssetResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AwsAssetResourceIT.java
b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AwsAssetResourceIT.java
index e512224..2914371 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AwsAssetResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AwsAssetResourceIT.java
@@ -50,6 +50,8 @@ import org.apache.usergrid.setup.ConcurrentProcessSingleton;
 
 import com.sun.jersey.multipart.FormDataMultiPart;
 
+import net.jcip.annotations.NotThreadSafe;
+
 import static org.apache.usergrid.management.AccountCreationProps.PROPERTIES_SYSADMIN_EMAIL;
 import static org.apache.usergrid.management.AccountCreationProps.PROPERTIES_USERGRID_BINARY_UPLOADER;
 import static org.apache.usergrid.utils.MapUtils.hashMap;
@@ -59,18 +61,18 @@ import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 
 
-
+@NotThreadSafe
 public class AwsAssetResourceIT extends AbstractRestIT {
 
     private String access_token;
     private Map<String, Object> originalProperties;
     private Logger LOG = LoggerFactory.getLogger( AwsAssetResourceIT.class );
 
-//    /**
-//     * Mark tests as ignored if no AWS creds are present
-//     */
-//    @Rule
-//    public NoAWSCredsRule awsCredsRule = new NoAWSCredsRule();
+    /**
+     * Mark tests as ignored if no AWS creds are present
+     */
+    @Rule
+    public NoAWSCredsRule awsCredsRule = new NoAWSCredsRule();
 
     @Before
     public void setup(){
@@ -230,7 +232,7 @@ public class AwsAssetResourceIT extends AbstractRestIT {
         ApiResponse postResponse = pathResource( getOrgAppPath( "foos" ) ).post( form );
         UUID assetId = postResponse.getEntities().get(0).getUuid();
         LOG.info( "Waiting for upload to finish..." );
-        Thread.sleep( 2000 );
+        Thread.sleep( 5000 );
 
         // check that entire file was uploaded
 
@@ -251,10 +253,7 @@ public class AwsAssetResourceIT extends AbstractRestIT {
         this.refreshIndex();
 
         // set max file size down to 6mb
-
-        Map<String, String> props = new HashMap<String, String>();
-        props.put( "usergrid.binary.max-size-mb", "6" );
-        pathResource( "testproperties" ).post( props );
+        setTestProperty( "usergrid.binary.max-size-mb","6" );
 
         try {
 
@@ -267,20 +266,21 @@ public class AwsAssetResourceIT extends AbstractRestIT {
 
             String errorMessage = null;
             LOG.info( "Waiting for upload to finish..." );
-            Thread.sleep( 2000 );
+            Thread.sleep( 1000 );
 
             // attempt to get asset entity, it should contain error
 
+            refreshIndex();
             ApiResponse getResponse = pathResource( getOrgAppPath( "bars/" +assetId ) ).get(
ApiResponse.class );
             Map<String, Object> fileMetadata = (Map<String, Object>)getResponse.getEntities().get(0).get("file-metadata");
+            assertNotNull( fileMetadata );
+            assertNotNull( fileMetadata.get( "error" ) );
             assertTrue( fileMetadata.get( "error" ).toString().startsWith( "Asset size "
) );
 
         } finally {
 
             // set max upload size back to default 25mb
-
-            props.put( "usergrid.binary.max-size-mb", "25" );
-            pathResource( "testproperties" ).post( props );
+            setTestProperties( originalProperties );
         }
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/050aa0a0/stack/rest/src/test/resources/usergrid-test-context.xml
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/resources/usergrid-test-context.xml b/stack/rest/src/test/resources/usergrid-test-context.xml
index 6b15b7d..a9fc5a5 100644
--- a/stack/rest/src/test/resources/usergrid-test-context.xml
+++ b/stack/rest/src/test/resources/usergrid-test-context.xml
@@ -44,14 +44,6 @@
         <property name="traceEnabled" value="false"/>
     </bean>
 
-    <bean id="binaryStore" class="org.apache.usergrid.services.assets.data.LocalFileBinaryStore"/>
-
-    <!--<bean id="binaryStore" class="org.apache.usergrid.services.assets.data.S3BinaryStore">-->
-    <!--<constructor-arg name="accessId" value="xx" />-->
-    <!--<constructor-arg name="secretKey" value="xx" />-->
-    <!--<constructor-arg name="bucketName" value="xx" />-->
-    <!--</bean>-->
-
     <bean id="setup" class="org.apache.usergrid.corepersistence.CpSetup">
 
         <constructor-arg ref="entityManagerFactory"/>

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/050aa0a0/stack/services/src/main/java/org/apache/usergrid/services/assets/data/AwsSdkS3BinaryStore.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/assets/data/AwsSdkS3BinaryStore.java
b/stack/services/src/main/java/org/apache/usergrid/services/assets/data/AwsSdkS3BinaryStore.java
index d34a273..e492f42 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/assets/data/AwsSdkS3BinaryStore.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/assets/data/AwsSdkS3BinaryStore.java
@@ -73,7 +73,7 @@ import org.apache.commons.codec.binary.Base64;
 
 public class AwsSdkS3BinaryStore implements BinaryStore {
 
-    private static final Logger LOG = LoggerFactory.getLogger(AwsSdkS3BinaryStore.class );
+    private static final Logger logger = LoggerFactory.getLogger(AwsSdkS3BinaryStore.class
);
     private static final long FIVE_MB = ( FileUtils.ONE_MB * 5 );
 
     private AmazonS3 s3Client;
@@ -197,6 +197,7 @@ public class AwsSdkS3BinaryStore implements BinaryStore {
 
                     if(written> maxSizeBytes){
                         overSizeLimit = true;
+                        logger.error( "OVERSIZED FILE. STARTING ABORT" );
                         break;
                         //set flag here and break out of loop to run abort
                     }
@@ -235,11 +236,12 @@ public class AwsSdkS3BinaryStore implements BinaryStore {
 
                 //upadte the entity with the error.
                 try {
+                    logger.error("starting update of entity due to oversized asset");
                     fileMetadata.put( "error", "Asset size is larger than max size of " +
maxSizeBytes );
                     em.update( entity );
                 }
                 catch ( Exception e ) {
-                    LOG.error( "Error updating entity with error message", e );
+                    logger.error( "Error updating entity with error message", e );
                 }
 
                 //loop and abort all the multipart uploads

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/050aa0a0/stack/services/src/test/resources/usergrid-test-context.xml
----------------------------------------------------------------------
diff --git a/stack/services/src/test/resources/usergrid-test-context.xml b/stack/services/src/test/resources/usergrid-test-context.xml
index f6d750d..ec1e2d2 100644
--- a/stack/services/src/test/resources/usergrid-test-context.xml
+++ b/stack/services/src/test/resources/usergrid-test-context.xml
@@ -51,7 +51,7 @@
         <constructor-arg ref="cassandraCluster"/>
     </bean>
 
-    <bean id="binaryStore" class="org.apache.usergrid.services.assets.data.LocalFileBinaryStore">
+    <bean id="localFileBinaryStore" class="org.apache.usergrid.services.assets.data.LocalFileBinaryStore">
         <property name="reposLocation" value="${usergrid.temp.files}"/>
     </bean>
 


Mime
View raw message