usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From snoopd...@apache.org
Subject [04/27] git commit: Added organization level exports with a different applications file for each application.
Date Tue, 01 Apr 2014 01:20:56 GMT
Added organization level exports with a different applications file for each application.


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/844fd5d6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/844fd5d6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/844fd5d6

Branch: refs/pull/92/merge
Commit: 844fd5d6eb1fc8416f4836d2abca30abc9d9394a
Parents: 2cd1d5d
Author: grey <greyes@apigee.com>
Authored: Mon Mar 31 00:32:57 2014 -0700
Committer: grey <greyes@apigee.com>
Committed: Mon Mar 31 00:32:57 2014 -0700

----------------------------------------------------------------------
 .../organizations/OrganizationResource.java     |  81 ++++++++++++
 .../rest/management/ManagementResourceIT.java   |   2 +-
 .../management/export/ExportServiceImpl.java    | 129 +++++++++++++++++--
 .../cassandra/ManagementServiceIT.java          |  89 +++++++------
 4 files changed, 253 insertions(+), 48 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/844fd5d6/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
index b57d02a..05312ab 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
@@ -17,7 +17,9 @@
 package org.apache.usergrid.rest.management.organizations;
 
 
+import java.util.HashMap;
 import java.util.Map;
+import java.util.UUID;
 
 import javax.ws.rs.Consumes;
 import javax.ws.rs.DefaultValue;
@@ -29,20 +31,29 @@ import javax.ws.rs.Produces;
 import javax.ws.rs.QueryParam;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
 import javax.ws.rs.core.UriInfo;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.annotation.Scope;
 import org.springframework.stereotype.Component;
+
+import org.apache.amber.oauth2.common.exception.OAuthSystemException;
+import org.apache.amber.oauth2.common.message.OAuthResponse;
+
 import org.apache.usergrid.management.ActivationState;
 import org.apache.usergrid.management.OrganizationInfo;
+import org.apache.usergrid.management.export.ExportService;
 import org.apache.usergrid.rest.AbstractContextResource;
 import org.apache.usergrid.rest.ApiResponse;
+import org.apache.usergrid.rest.applications.ServiceResource;
 import org.apache.usergrid.rest.exceptions.RedirectionException;
 import org.apache.usergrid.rest.management.organizations.applications.ApplicationsResource;
 import org.apache.usergrid.rest.management.organizations.users.UsersResource;
 import org.apache.usergrid.rest.security.annotations.RequireOrganizationAccess;
+import org.apache.usergrid.rest.utils.JSONPUtils;
 import org.apache.usergrid.security.oauth.ClientCredentialsInfo;
 import org.apache.usergrid.security.tokens.exceptions.TokenException;
 import org.apache.usergrid.services.ServiceResults;
@@ -50,6 +61,11 @@ import org.apache.usergrid.services.ServiceResults;
 import com.sun.jersey.api.json.JSONWithPadding;
 import com.sun.jersey.api.view.Viewable;
 
+import static javax.servlet.http.HttpServletResponse.SC_ACCEPTED;
+import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
+import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
+import static javax.ws.rs.core.MediaType.APPLICATION_JSON;
+
 
 @Component("org.apache.usergrid.rest.management.organizations.OrganizationResource")
 @Scope("prototype")
@@ -61,6 +77,9 @@ public class OrganizationResource extends AbstractContextResource {
 
     private static final Logger logger = LoggerFactory.getLogger( OrganizationsResource.class
);
 
+    @Autowired
+    protected ExportService exportService;
+
     OrganizationInfo organization;
 
 
@@ -252,4 +271,66 @@ public class OrganizationResource extends AbstractContextResource {
 
         return new JSONWithPadding( response, callback );
     }
+
+    @POST
+    @Path("export")
+    @Consumes(APPLICATION_JSON)
+    @RequireOrganizationAccess
+    public Response exportPostJson( @Context UriInfo ui,Map<String, Object> json,
+                                    @QueryParam("callback") @DefaultValue("") String callback
)
+            throws OAuthSystemException {
+
+
+        OAuthResponse response = null;
+        UUID jobUUID = null;
+        Map<String, String> uuidRet = new HashMap<String, String>();
+
+        Map<String,Object> properties;
+        Map<String, Object> storage_info;
+
+        try {
+            if((properties = ( Map<String, Object> )  json.get( "properties" )) ==
null){
+                throw new NullPointerException("Could not find 'properties'");
+            }
+            storage_info = ( Map<String, Object> ) properties.get( "storage_info" );
+            String storage_provider = ( String ) properties.get( "storage_provider" );
+            if(storage_provider == null) {
+                throw new NullPointerException( "Could not find field 'storage_provider'"
);
+            }
+            if(storage_info == null) {
+                throw new NullPointerException( "Could not find field 'storage_info'" );
+            }
+
+
+            String bucketName = ( String ) storage_info.get( "bucket_location" );
+            String accessId = ( String ) storage_info.get( "s3_access_id" );
+            String secretKey = ( String ) storage_info.get( "s3_key" );
+
+            if(bucketName == null) {
+                throw new NullPointerException( "Could not find field 'bucketName'" );
+            }
+            if(accessId == null) {
+                throw new NullPointerException( "Could not find field 's3_access_id'" );
+            }
+            if(secretKey == null) {
+                throw new NullPointerException( "Could not find field 's3_key'" );
+            }
+
+            json.put( "organizationId",organization.getUuid());
+
+            jobUUID = exportService.schedule( json );
+            uuidRet.put( "Export Entity", jobUUID.toString() );
+        }
+        catch ( NullPointerException e ) {
+            return Response.status( SC_BAD_REQUEST ).type( JSONPUtils.jsonMediaType( callback
) )
+                           .entity( ServiceResource.wrapWithCallback( e.getMessage(), callback
) ).build();
+        }
+        catch ( Exception e ) {
+            //TODO:throw descriptive error message and or include on in the response
+            //TODO:fix below, it doesn't work if there is an exception. Make it look like
the OauthResponse.
+            return Response.status( SC_INTERNAL_SERVER_ERROR ).type( JSONPUtils.jsonMediaType(
callback ) )
+                           .entity( ServiceResource.wrapWithCallback( e.getMessage(), callback
) ).build();
+        }
+        return Response.status( SC_ACCEPTED ).entity( uuidRet ).build();
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/844fd5d6/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
index 9874bd4..d117e57 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
@@ -771,7 +771,7 @@ public class ManagementResourceIT extends AbstractRestIT {
         uuid = uuid.replaceAll( "\"", "" );
 
         try {
-            node = resource().path( "/management/orgs/test-organization/export/" + uuid )
+            node = resource().path( "/management/orgs/test-organization/apps/exporters/export/"
+ uuid )
                              .queryParam( "access_token", superAdminToken() ).accept( MediaType.APPLICATION_JSON
)
                              .type( MediaType.APPLICATION_JSON_TYPE ).get( JsonNode.class
);
         }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/844fd5d6/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
index b599854..053a4e3 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
@@ -48,6 +48,8 @@ import org.apache.usergrid.persistence.Results;
 import org.apache.usergrid.persistence.entities.Export;
 import org.apache.usergrid.persistence.entities.JobData;
 
+import com.google.common.collect.BiMap;
+
 
 /**
  * Need to refactor out the mutliple orgs being take , need to factor out the multiple apps
it will just be the one app
@@ -78,9 +80,12 @@ public class ExportServiceImpl implements ExportService {
 
     private S3Export s3Export;
 
+    private String defaultAppExportname = "exporters";
+
 
     @Override
     public UUID schedule( final Map<String, Object> config ) throws Exception {
+        ApplicationInfo defaultExportApp = null;
 
         if ( config == null ) {
             logger.error( "export information cannot be null" );
@@ -88,8 +93,11 @@ public class ExportServiceImpl implements ExportService {
         }
 
         if ( config.get( "applicationId" ) == null ) {
-            logger.error( "application information from export info could not be found" );
-            return null;
+            defaultExportApp =
+                    managementService.createApplication( ( UUID ) config.get( "organizationId"
), defaultAppExportname );
+            config.put( "applicationId", defaultExportApp.getId() );
+            //logger.error( "application information from export info could not be found"
);
+            //return null;
         }
 
         EntityManager em = null;
@@ -169,24 +177,29 @@ public class ExportServiceImpl implements ExportService {
     public void doExport( final JobExecution jobExecution ) throws Exception {
         Map<String, Object> config = ( Map<String, Object> ) jobExecution.getJobData().getProperty(
"exportInfo" );
 
+        UUID scopedAppId = ( UUID ) config.get( "applicationId" );
+
         if ( config == null ) {
             logger.error( "Export Information passed through is null" );
             return;
         }
         //get the entity manager for the application, and the entity that this Export corresponds
to.
         UUID exportId = ( UUID ) jobExecution.getJobData().getProperty( EXPORT_ID );
-        if ( config.get( "applicationId" ) == null ) {
+        if ( scopedAppId == null ) {
             logger.error( "Export Information application uuid is null" );
             return;
         }
-        EntityManager em = emf.getEntityManager( ( UUID ) config.get( "applicationId" ) );
+        EntityManager em = emf.getEntityManager( scopedAppId );
         Export export = em.get( exportId, Export.class );
 
         //update the entity state to show that the job has officially started.
         export.setState( Export.State.STARTED );
         em.update( export );
 
-        if ( config.get( "collectionName" ) == null ) {
+        if ( em.getApplication().getApplicationName().equals( "exporters" ) ) {
+            exportApplicationsFromOrg( ( UUID ) config.get( "organizationId" ), config, jobExecution
);
+        }
+        else if ( config.get( "collectionName" ) == null ) {
             //exports all the applications for a given organization.
 
             if ( config.get( "organizationId" ) == null ) {
@@ -195,7 +208,7 @@ public class ExportServiceImpl implements ExportService {
                 em.update( export );
                 return;
             }
-            exportApplicationsForOrg( ( UUID ) config.get( "organizationId" ), ( UUID ) config.get(
"applicationId" ),
+            exportApplicationFromOrg( ( UUID ) config.get( "organizationId" ), ( UUID ) config.get(
"applicationId" ),
                     config, jobExecution );
         }
         else {
@@ -207,7 +220,7 @@ public class ExportServiceImpl implements ExportService {
                     em.update( export );
                     return;
                 }
-                exportApplicationForOrg( ( UUID ) config.get( "organizationId" ),
+                exportCollectionFromOrgApp( ( UUID ) config.get( "organizationId" ),
                         ( UUID ) config.get( "applicationId" ), config, jobExecution );
             }
             catch ( Exception e ) {
@@ -254,9 +267,99 @@ public class ExportServiceImpl implements ExportService {
 
 
     /**
-     * Exports all applications for the given organization.
+     * Exports All Applications from an Organization
      */
-    private void exportApplicationsForOrg( UUID organizationUUID, UUID applicationId, final
Map<String, Object> config,
+    private void exportApplicationsFromOrg( UUID organizationUUID, final Map<String, Object>
config,
+                                            final JobExecution jobExecution ) throws Exception
{
+
+        //retrieves export entity
+        UUID exportId = ( UUID ) jobExecution.getJobData().getProperty( EXPORT_ID );
+        EntityManager exportManager = emf.getEntityManager( ( UUID ) config.get( "applicationId"
) );
+        Export export = exportManager.get( exportId, Export.class );
+        String appFileName = null;
+
+        //sets up a output stream for s3 backup.
+//        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+//        JsonGenerator jg = getJsonGenerator( baos );
+
+        BiMap<UUID, String> applications = managementService.getApplicationsForOrganization(
organizationUUID );
+
+        for ( Map.Entry<UUID, String> application : applications.entrySet() ) {
+            ByteArrayOutputStream baos = new ByteArrayOutputStream();
+            JsonGenerator jg = getJsonGenerator( baos );
+            if(application.getValue().equals(
+                    managementService.getOrganizationByUuid( organizationUUID ).getName()
+ "/exporters" )){
+                continue;
+            }
+
+            appFileName = prepareOutputFileName( "application", application.getValue(), null
);
+
+
+            EntityManager em = emf.getEntityManager( application.getKey() );
+
+            jg.writeStartArray();
+
+            Map<String, Object> metadata = em.getApplicationCollectionMetadata();
+            long starting_time = System.currentTimeMillis();
+
+            // Loop through the collections. This is the only way to loop
+            // through the entities in the application (former namespace).
+            //could support queries, just need to implement that in the rest endpoint.
+            for ( String collectionName : metadata.keySet() ) {
+                if ( collectionName.equals( "exports" ) ) {
+                    continue;
+                }
+                //if the collection you are looping through doesn't match the name of the
one you want. Don't export it.
+
+                if ( ( config.get( "collectionName" ) == null ) || collectionName
+                        .equals( config.get( "collectionName" ) ) ) {
+                    //Query entity manager for the entities in a collection
+                    Query query = new Query();
+                    query.setLimit( MAX_ENTITY_FETCH );
+                    query.setResultsLevel( Results.Level.ALL_PROPERTIES );
+                    Results entities = em.searchCollection( em.getApplicationRef(), collectionName,
query );
+
+                    //pages through the query and backs up all results.
+                    PagingResultsIterator itr = new PagingResultsIterator( entities );
+                    for ( Object e : itr ) {
+                        starting_time = checkTimeDelta( starting_time, jobExecution );
+                        Entity entity = ( Entity ) e;
+                        jg.writeStartObject();
+                        jg.writeFieldName( "Metadata" );
+                        jg.writeObject( entity );
+                        saveCollectionMembers( jg, em, ( String ) config.get( "collectionName"
), entity );
+                        jg.writeEndObject();
+                    }
+                 }
+            }
+        //}
+
+
+
+        // Close writer and file for this application.
+        jg.writeEndArray();
+
+        jg.close();
+        baos.flush();
+        baos.close();
+
+        //sets up the Inputstream for copying the method to s3.
+        InputStream is = new ByteArrayInputStream( baos.toByteArray() );
+        try {
+            s3Export.copyToS3( is, config, appFileName );
+        }
+        catch ( Exception e ) {
+            export.setState( Export.State.FAILED );
+            return;
+        }
+        }
+    }
+
+
+    /**
+     * Exports a specific applications from an organization
+     */
+    private void exportApplicationFromOrg( UUID organizationUUID, UUID applicationId, final
Map<String, Object> config,
                                            final JobExecution jobExecution ) throws Exception
{
 
         //retrieves export entity
@@ -328,9 +431,13 @@ public class ExportServiceImpl implements ExportService {
     }
 
 
+    /**
+     * Exports a specific collection from an org-app combo.
+     */
     //might be confusing, but uses the /s/ inclusion or exclusion nomenclature.
-    private void exportApplicationForOrg( UUID organizationUUID, UUID applicationUUID, final
Map<String, Object> config,
-                                          final JobExecution jobExecution ) throws Exception
{
+    private void exportCollectionFromOrgApp( UUID organizationUUID, UUID applicationUUID,
+                                             final Map<String, Object> config, final
JobExecution jobExecution )
+            throws Exception {
 
         //retrieves export entity
         UUID exportId = ( UUID ) jobExecution.getJobData().getProperty( EXPORT_ID );

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/844fd5d6/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
index f1dd4bb..fa8dd0c 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
@@ -1589,40 +1589,43 @@ public class ManagementServiceIT {
 
         OrganizationInfo orgMade = null;
         ApplicationInfo appMade = null;
-        for(int i = 0; i < 100; i++) {
-            orgMade =setup.getMgmtSvc().createOrganization( "minorboss"+i,adminUser,true
);
-            appMade = setup.getMgmtSvc().createApplication( orgMade.getUuid(), "superapp"+i);
-
-            EntityManager customMaker = setup.getEmf().getEntityManager( appMade.getId()
);
-            customMaker.createApplicationCollection( "superappCol"+i );
-            //intialize user object to be posted
-            Map<String, Object> entityLevelProperties = null;
-            Entity[] entNotCopied;
-            entNotCopied = new Entity[20];
-            //creates entities
-            for ( int index = 0; index < 20; index++ ) {
-                entityLevelProperties = new LinkedHashMap<String, Object>();
-                entityLevelProperties.put( "username", "bobso" + index );
-                entityLevelProperties.put( "email", "derp" + index + "@anuff.com" );
-                entNotCopied[index] = customMaker.create( "superappCol", entityLevelProperties
);
+        for ( int i = 0; i < 5; i++ ) {
+            orgMade = setup.getMgmtSvc().createOrganization( "minorboss" + i, adminUser,
true );
+            for ( int j = 0; j < 5; j++ ) {
+                appMade = setup.getMgmtSvc().createApplication( orgMade.getUuid(), "superapp"
+ j );
+
+                EntityManager customMaker = setup.getEmf().getEntityManager( appMade.getId()
);
+                customMaker.createApplicationCollection( "superappCol" + j );
+                //intialize user object to be posted
+                Map<String, Object> entityLevelProperties = null;
+                Entity[] entNotCopied;
+                entNotCopied = new Entity[1];
+                //creates entities
+                for ( int index = 0; index < 1; index++ ) {
+                    entityLevelProperties = new LinkedHashMap<String, Object>();
+                    entityLevelProperties.put( "derp","bacon" );
+//                    entityLevelProperties.put( "username", "bobso" + index );
+//                    entityLevelProperties.put( "email", "derp" + index + "@anuff.com" );
+                    entNotCopied[index] = customMaker.create( "superappCol"+j, entityLevelProperties
);
+                }
             }
         }
 
         payload.put("organizationId",orgMade.getUuid());
 
-        EntityManager em = setup.getEmf().getEntityManager( appMade.getId() );
+        //EntityManager em = setup.getEmf().getEntityManager( appMade.getId() );
         //intialize user object to be posted
-        Map<String, Object> userProperties = null;
-        Entity[] entity;
-        entity = new Entity[100];
-        //creates entities
-        for ( int i = 0; i < 100; i++ ) {
-            userProperties = new LinkedHashMap<String, Object>();
-            userProperties.put( "username", "bido" + i );
-            userProperties.put( "email", "bido" + i + "@anuff.com" );
-
-            entity[i] = em.create( "user", userProperties );
-        }
+//        Map<String, Object> userProperties = null;
+//        Entity[] entity;
+//        entity = new Entity[100];
+//        //creates entities
+//        for ( int i = 0; i < 100; i++ ) {
+//            userProperties = new LinkedHashMap<String, Object>();
+//            userProperties.put( "username", "bido" + i );
+//            userProperties.put( "email", "bido" + i + "@anuff.com" );
+//
+//            entity[i] = em.create( "user", userProperties );
+//        }
 
         UUID exportUUID = exportService.schedule( payload );
         assertNotNull( exportUUID );
@@ -1637,8 +1640,15 @@ public class ManagementServiceIT {
         JobExecution jobExecution = mock( JobExecution.class );
         when( jobExecution.getJobData() ).thenReturn( jobData );
 
+        //the exporters app is never returned, and I'm not even sure if that is the way we
want to go
+        // I feel like there should be a better way than having to store data for the user.
         exportService.doExport( jobExecution );
-        while (!exportService.getState( appMade.getId(),exportUUID ).equals("FINISHED"));
+        //BiMap<UUID,String> mapper = setup.getMgmtSvc().getApplicationsForOrganization(orgMade.getUuid());
+        //mapper.get( "" )
+//        while (!exportService.getState( ,exportUUID )
+//                             .equals("FINISHED"));
+
+        Thread.sleep( 3000 );
 
         String bucketName = System.getProperty( "bucketName" );
         String accessId = System.getProperty( "accessKey" );
@@ -1662,17 +1672,24 @@ public class ManagementServiceIT {
 
 
             blobStore = context.getBlobStore();
-            if(!blobStore.blobExists( bucketName,s3Export.getFilename()  )){
-                blobStore.deleteContainer( bucketName );
-                assert(false);
-            }
+//            if(!blobStore.blobExists( bucketName,s3Export.getFilename()  )){
+//                blobStore.deleteContainer( bucketName );
+//                assert(false);
+//            }
+            //Grab Number of files
             Long numOfFiles = blobStore.countBlobs( bucketName );
-            Long numWeWant = Long.valueOf( 100 );
-            assertEquals( numOfFiles, numWeWant );
-
+            //delete container containing said files
             bo = blobStore.getBlob( bucketName,s3Export.getFilename()   );
+            blobStore.deleteContainer( bucketName );
+            Long numWeWant = Long.valueOf( 5 );
+            //asserts that the correct number of files was transferred over
+            assertEquals( numWeWant, numOfFiles);
+
+
 
         }catch(Exception e) {
+            blobStore.deleteContainer( bucketName );
+            e.printStackTrace();
             assert(false);
         }
 


Mime
View raw message