usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From snoopd...@apache.org
Subject [20/50] [abbrv] git commit: Revised response message and added ability for user to get back Json response with UUID of job. Added test that checks to make sure you are getting a UUID back. Removed extra spaces in exportInfo class. Uncomplicated imports i
Date Tue, 25 Feb 2014 06:21:34 GMT
Revised response message and added ability for user to get back Json response with UUID of
job.
Added test that checks to make sure you are getting a UUID back.
Removed extra spaces in exportInfo class.
Uncomplicated imports in ManagementService.
Removed unused method processExportData.
ExportJob has the heartbeat in to let server know the job is still continuing.
Added setter method so that we could mock the ExportService.
Added getJobUUID to simplify the sending of the UUID to the user.
Added setS3Export method to be able to mock the transfer of data to s3.
Removed the copyToS3 method to a Impl method that would actually copy a file to s3.
Added spring wiring to find s3ExportImpl class.
Added ExportService to ServiceITSetup to allow calling of the service from setup.
Added a doJob test that correctly mocks the services from job to export.


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/68bd76be
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/68bd76be
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/68bd76be

Branch: refs/pull/60/head
Commit: 68bd76be3f36f0098777a5696305526abcbb962c
Parents: e8d47ba
Author: George Reyes <ApigeeCorporation@George-Reyess-MacBook-Pro.local>
Authored: Wed Feb 12 11:27:21 2014 -0800
Committer: George Reyes <ApigeeCorporation@George-Reyess-MacBook-Pro.local>
Committed: Wed Feb 12 11:27:21 2014 -0800

----------------------------------------------------------------------
 .../rest/management/ManagementResource.java     | 45 +++++----
 .../rest/management/ManagementResourceIT.java   | 46 +++++++++
 .../org/usergrid/management/ExportInfo.java     |  3 -
 .../usergrid/management/ManagementService.java  | 12 ++-
 .../cassandra/ManagementServiceImpl.java        |  8 --
 .../usergrid/management/export/ExportJob.java   | 10 ++
 .../management/export/ExportService.java        | 10 ++
 .../management/export/ExportServiceImpl.java    | 98 +++++---------------
 .../usergrid/management/export/S3Export.java    | 14 +++
 .../management/export/S3ExportImpl.java         | 83 +++++++++++++++++
 .../resources/usergrid-services-context.xml     |  2 +
 .../test/java/org/usergrid/ServiceITSetup.java  |  3 +
 .../java/org/usergrid/ServiceITSetupImpl.java   |  6 ++
 .../cassandra/ManagementServiceIT.java          | 89 +++++++++++++++++-
 14 files changed, 320 insertions(+), 109 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68bd76be/stack/rest/src/main/java/org/usergrid/rest/management/ManagementResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/usergrid/rest/management/ManagementResource.java
b/stack/rest/src/main/java/org/usergrid/rest/management/ManagementResource.java
index 65be6c3..8f802bd 100644
--- a/stack/rest/src/main/java/org/usergrid/rest/management/ManagementResource.java
+++ b/stack/rest/src/main/java/org/usergrid/rest/management/ManagementResource.java
@@ -17,7 +17,9 @@ package org.usergrid.rest.management;
 
 
 import java.net.URLEncoder;
+import java.util.HashMap;
 import java.util.Map;
+import java.util.UUID;
 
 import javax.ws.rs.Consumes;
 import javax.ws.rs.DefaultValue;
@@ -61,8 +63,10 @@ import org.apache.shiro.codec.Base64;
 
 import com.sun.jersey.api.view.Viewable;
 
+import static javax.servlet.http.HttpServletResponse.SC_ACCEPTED;
 import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
 import static javax.servlet.http.HttpServletResponse.SC_FORBIDDEN;
+import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
 import static javax.ws.rs.core.MediaType.APPLICATION_FORM_URLENCODED;
 import static javax.ws.rs.core.MediaType.APPLICATION_JSON;
@@ -449,39 +453,46 @@ public class ManagementResource extends AbstractContextResource {
 
     @POST
     @Path( "export" )
+   // @RequireAdminUserAccess
     @Consumes(MediaType.APPLICATION_JSON)
-    public Response exportPostJson (@Context UriInfo ui,@HeaderParam( "Authorization" ) String
authorization,
+    public Response exportPostJson (@Context UriInfo ui,
+                                   // @QueryParam( "access_token" ) String access_token,
                                     Map<String, Object> json,
-                                    @QueryParam( "callback" ) @DefaultValue( "" ) String
callback){
+                                    @QueryParam( "callback" ) @DefaultValue( "" ) String
callback)
+            throws OAuthSystemException {
 
 
         OAuthResponse response = null;
-        String client_id;
-        String client_secret;
+        UUID jobUUID = null;
+        Map<String, String> uuidRet = new HashMap<String, String>(  );
 
         try {
 
         //parse the json into some useful object (the config params)
-        ExportInfo objEx = new ExportInfo(json);
+            ExportInfo objEx = new ExportInfo(json);
+            exportService.schedule(objEx);
+            jobUUID = exportService.getJobUUID();
+            uuidRet.put( "jobUUID", jobUUID.toString() );
 
-        exportService.schedule(objEx);
-       // exportService.doExport( objEx );
+        }
+        catch (NullPointerException e) {
+            OAuthResponse errorMsg = OAuthResponse.errorResponse( SC_BAD_REQUEST )
+                         .setErrorDescription( "Job Not Created" )
+                         .buildJSONMessage();
+
+            return Response.status( errorMsg.getResponseStatus() ).type( jsonMediaType( callback
) )
+                           .entity( wrapWithCallback( errorMsg.getBody(), callback ) ).build();
         }
         catch (Exception e) {
             //TODO:throw descriptive error message and or include on in the response
             //TODO:fix below, it doesn't work if there is an exception. Make it look like
the OauthResponse.
-            return Response.status( SC_BAD_REQUEST ).build();
+            return Response.status( SC_INTERNAL_SERVER_ERROR ).build();
         }
 
-        //TODO: make schedule or doExport return a response? Or create one.
-        try {
-            response = OAuthResponse.status( SC_OK ).buildJSONMessage();
-        }
-        catch ( OAuthSystemException e ) {
-            e.printStackTrace();
-        }
-        return Response.status( response.getResponseStatus() ).type( jsonMediaType( callback
) )
-                       .entity( wrapWithCallback( response.getBody(), callback ) ).build();
+        return Response.status(SC_ACCEPTED).entity(uuidRet).build();
+
+        //Response.status( response.getResponseStatus() ).type( jsonMediaType( callback )
)
+                 //      .entity( wrapWithCallback( "", callback ) ).build();
     }
 
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68bd76be/stack/rest/src/test/java/org/usergrid/rest/management/ManagementResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/usergrid/rest/management/ManagementResourceIT.java
b/stack/rest/src/test/java/org/usergrid/rest/management/ManagementResourceIT.java
index a8278bc..87a1d18 100644
--- a/stack/rest/src/test/java/org/usergrid/rest/management/ManagementResourceIT.java
+++ b/stack/rest/src/test/java/org/usergrid/rest/management/ManagementResourceIT.java
@@ -18,6 +18,7 @@ package org.usergrid.rest.management;
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.UUID;
 
 import javax.ws.rs.core.MediaType;
 
@@ -545,6 +546,7 @@ public class ManagementResourceIT extends AbstractRestIT {
         storage_info.put( "s3_accessId","insert access id here");
         storage_info.put( "bucket_location","insert bucket name here");
 
+
         properties.put( "storage_provider","s3");
         properties.put( "storage_info",storage_info);
 
@@ -601,4 +603,48 @@ public class ManagementResourceIT extends AbstractRestIT {
 
         assertEquals( Status.OK, responseStatus );
     }
+//tests that you need to accomplish.
+    //make sure you can call the endpoint
+    //make sure you can return the uuid back to the user
+    //make sure you can mock the s3 testing
+    @Test
+    public void exportUUIDRetTest() throws Exception {
+        Status responseStatus = Status.ACCEPTED;
+        String uuid;
+        UUID jobUUID = null;
+        JsonNode node = null;
+
+
+        HashMap<String, Object> payload = new HashMap<String, Object>();
+        Map<String, Object> properties = new HashMap<String, Object>();
+        Map<String, Object> storage_info = new HashMap<String, Object>();
+        //TODO: make sure to put a valid admin token here.
+        //storage_info.put( "admin_token","insert_token_data_here" );
+        //TODO: always put dummy values here and ignore this test.
+        //TODO: add a ret for when s3 values are invalid.
+        storage_info.put( "s3_key","insert key here" );
+        storage_info.put( "s3_accessId","insert access id here");
+        storage_info.put( "bucket_location","insert bucket name here");
+
+
+        properties.put( "storage_provider","s3");
+        properties.put( "storage_info",storage_info);
+
+        payload.put( "path", "test-organization/test-app/user");
+        payload.put( "properties", properties);
+
+        try {
+            node = resource().path( "/management/export" ).queryParam( "access_token", adminAccessToken
)
+                             .accept( MediaType.APPLICATION_JSON )
+                             .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class,
payload );
+
+        }
+        catch ( UniformInterfaceException uie ) {
+            responseStatus = uie.getResponse().getClientResponseStatus();
+        }
+
+        assertEquals( Status.ACCEPTED, responseStatus );
+        assertNotNull( node.get( "jobUUID" ) );
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68bd76be/stack/services/src/main/java/org/usergrid/management/ExportInfo.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/usergrid/management/ExportInfo.java b/stack/services/src/main/java/org/usergrid/management/ExportInfo.java
index 1a272e4..36df16c 100644
--- a/stack/services/src/main/java/org/usergrid/management/ExportInfo.java
+++ b/stack/services/src/main/java/org/usergrid/management/ExportInfo.java
@@ -69,7 +69,4 @@ public class ExportInfo { //extends Entity something {
     public String getBucket_location () { return bucket_location; }
 
     public String getS3_accessId () { return s3_accessId; }
-
-
-
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68bd76be/stack/services/src/main/java/org/usergrid/management/ManagementService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/usergrid/management/ManagementService.java b/stack/services/src/main/java/org/usergrid/management/ManagementService.java
index 5995a1b..d3926f5 100644
--- a/stack/services/src/main/java/org/usergrid/management/ManagementService.java
+++ b/stack/services/src/main/java/org/usergrid/management/ManagementService.java
@@ -16,7 +16,12 @@
 package org.usergrid.management;
 
 
-import com.google.common.collect.BiMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.UUID;
+
 import org.usergrid.persistence.CredentialsInfo;
 import org.usergrid.persistence.Entity;
 import org.usergrid.persistence.EntityRef;
@@ -28,7 +33,7 @@ import org.usergrid.security.oauth.AccessInfo;
 import org.usergrid.security.shiro.PrincipalCredentialsToken;
 import org.usergrid.services.ServiceResults;
 
-import java.util.*;
+import com.google.common.collect.BiMap;
 
 
 public interface ManagementService {
@@ -311,7 +316,4 @@ public interface ManagementService {
 
     /** For testing purposes only */
     public Properties getProperties();
-
-
-    public JobInfo processExportData( ExportInfo exportData );
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68bd76be/stack/services/src/main/java/org/usergrid/management/cassandra/ManagementServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/usergrid/management/cassandra/ManagementServiceImpl.java
b/stack/services/src/main/java/org/usergrid/management/cassandra/ManagementServiceImpl.java
index cb28cf7..f7cc99c 100644
--- a/stack/services/src/main/java/org/usergrid/management/cassandra/ManagementServiceImpl.java
+++ b/stack/services/src/main/java/org/usergrid/management/cassandra/ManagementServiceImpl.java
@@ -36,8 +36,6 @@ import org.usergrid.locking.LockManager;
 import org.usergrid.management.AccountCreationProps;
 import org.usergrid.management.ActivationState;
 import org.usergrid.management.ApplicationInfo;
-import org.usergrid.management.ExportInfo;
-import org.usergrid.management.JobInfo;
 import org.usergrid.management.ManagementService;
 import org.usergrid.management.OrganizationInfo;
 import org.usergrid.management.OrganizationOwnerInfo;
@@ -2908,10 +2906,4 @@ public class ManagementServiceImpl implements ManagementService {
         // TODO Auto-generated method stub
         return null;
     }
-
-    public JobInfo processExportData ( ExportInfo exportData ) {
-        //TODO: Generate proper method for having export info translated into jobinfo.
-
-        return null;
-    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68bd76be/stack/services/src/main/java/org/usergrid/management/export/ExportJob.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/usergrid/management/export/ExportJob.java b/stack/services/src/main/java/org/usergrid/management/export/ExportJob.java
index 4f57b5f..bd72c6e 100644
--- a/stack/services/src/main/java/org/usergrid/management/export/ExportJob.java
+++ b/stack/services/src/main/java/org/usergrid/management/export/ExportJob.java
@@ -5,6 +5,7 @@ import java.util.UUID;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 import org.usergrid.batch.JobExecution;
 import org.usergrid.batch.job.OnlyOnceJob;
@@ -20,6 +21,7 @@ import org.usergrid.persistence.entities.JobData;
 public class ExportJob extends OnlyOnceJob {
     private static final Logger logger = LoggerFactory.getLogger( ExportJob.class );
 
+    @Autowired
     ExportService exportService;
 
     public ExportJob() {
@@ -35,6 +37,8 @@ public class ExportJob extends OnlyOnceJob {
         //this is probably the state info that todd mentioned
         ExportInfo config = (ExportInfo) jobData.getProperty( "exportInfo" );
 
+        jobExecution.heartbeat();
+//pass in jobExecution so that you can call the heartbeat in the do export method.
         exportService.doExport( config );
 
         logger.info( "executed ExportJob completed normally" );
@@ -45,4 +49,10 @@ public class ExportJob extends OnlyOnceJob {
         //return arbitrary number
         return 100;
     }
+
+
+    @Autowired
+    public void setExportService( final ExportService exportService ) {
+        this.exportService = exportService;
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68bd76be/stack/services/src/main/java/org/usergrid/management/export/ExportService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/usergrid/management/export/ExportService.java
b/stack/services/src/main/java/org/usergrid/management/export/ExportService.java
index 550e8c5..e2e5d54 100644
--- a/stack/services/src/main/java/org/usergrid/management/export/ExportService.java
+++ b/stack/services/src/main/java/org/usergrid/management/export/ExportService.java
@@ -1,6 +1,8 @@
 package org.usergrid.management.export;
 
 
+import java.util.UUID;
+
 import org.usergrid.management.ExportInfo;
 
 
@@ -23,4 +25,12 @@ public interface ExportService {
      */
     void doExport(ExportInfo config) throws Exception;
 
+    /**
+     * Returns the UUID to the user
+     * @param
+     */
+    UUID getJobUUID();
+
+    void setS3Export(S3Export s3Export);
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68bd76be/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java
b/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java
index b0d8cf6..c40f7f1 100644
--- a/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java
+++ b/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java
@@ -7,7 +7,6 @@ import java.io.PrintWriter;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Properties;
 import java.util.Set;
 import java.util.UUID;
 
@@ -15,17 +14,9 @@ import org.codehaus.jackson.JsonFactory;
 import org.codehaus.jackson.JsonGenerator;
 import org.codehaus.jackson.impl.DefaultPrettyPrinter;
 import org.codehaus.jackson.map.ObjectMapper;
-import org.jclouds.ContextBuilder;
-import org.jclouds.blobstore.AsyncBlobStore;
-import org.jclouds.blobstore.BlobStoreContext;
-import org.jclouds.blobstore.domain.Blob;
-import org.jclouds.blobstore.domain.BlobBuilder;
-import org.jclouds.blobstore.options.PutOptions;
-import org.jclouds.http.config.JavaUrlHttpCommandExecutorServiceModule;
-import org.jclouds.logging.log4j.config.Log4JLoggingModule;
-import org.jclouds.netty.config.NettyPayloadModule;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.usergrid.batch.service.SchedulerService;
 import org.usergrid.management.ExportInfo;
 import org.usergrid.management.ManagementService;
@@ -37,11 +28,9 @@ import org.usergrid.persistence.Query;
 import org.usergrid.persistence.Results;
 import org.usergrid.persistence.cassandra.CassandraService;
 import org.usergrid.persistence.entities.JobData;
+import org.usergrid.persistence.entities.JobStat;
 
 import com.google.common.collect.BiMap;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.inject.Module;
 
 
 /**
@@ -53,7 +42,6 @@ public class ExportServiceImpl implements ExportService{
 
     private static final Logger logger = LoggerFactory.getLogger( ExportServiceImpl.class
);
     //dependency injection
-    //inject scheduler - needs to be auto wired
     private SchedulerService sch;
 
     //injected the Entity Manager Factory
@@ -63,7 +51,6 @@ public class ExportServiceImpl implements ExportService{
     //inject Management Service to access Organization Data
     private ManagementService managementService;
 
-
     //Maximum amount of entities retrieved in a single go.
     public static final int MAX_ENTITY_FETCH = 100;
 
@@ -77,6 +64,10 @@ public class ExportServiceImpl implements ExportService{
 
     private String filename = PATH_REPLACEMENT;
 
+    private UUID jobUUID;
+
+    private S3Export s3Export;
+
     //TODO: Todd, do I refactor most of the methods out to just leave schedule and doExport
much like
     //the exporting toolbase class?
 
@@ -106,18 +97,22 @@ public class ExportServiceImpl implements ExportService{
         //TODO: parse path and make sure all the things you need actually exist. then throw
         // good error messages when not found.
 
-        //  managementService.getOrganizationByName(  )
         //validate user has access key to org (rather valid user has admin access token)
             //this is token validation
-        //schedule the job
         JobData jobData = new JobData();
 
-        jobData.setProperty( "exportId", "0001" );//TODO: store uuid here, give export job
uuid.
         jobData.setProperty( "exportInfo",config );
         long soonestPossible = System.currentTimeMillis() + 250; //sch grace period
-        sch.createJob( "exportJob",soonestPossible, jobData );
-
+        JobData retJobData = sch.createJob( "exportJob",soonestPossible, jobData );
+        jobUUID = retJobData.getUuid();
 
+        try {
+            JobStat merp = sch.getStatsForJob( "exportJob", retJobData.getUuid() );
+            System.out.println("hi");
+        }
+        catch ( Exception e ) {
+            logger.error( "could not get stats for job" );
+        }
     }
 
 
@@ -127,15 +122,12 @@ public class ExportServiceImpl implements ExportService{
         Map<UUID, String> organizations = getOrgs();
         for ( Map.Entry<UUID, String> organization : organizations.entrySet() ) {
 
-//            if ( organization.equals( properties.getProperty( "usergrid.test-account.organization"
) ) ) {
-//                // Skip test data from being exported.
-//                continue;
-//            }
-
             exportApplicationsForOrg( organization , config );
         }
     }
 
+
+
     private Map<UUID, String> getOrgs() throws Exception {
         // Loop through the organizations
        // TODO:this will come from the orgs in schedule when you do the validations. delete
orgId
@@ -198,6 +190,10 @@ public class ExportServiceImpl implements ExportService{
         this.managementService = managementService;
     }
 
+    public UUID getJobUUID () {
+        return jobUUID;
+    }
+
 
     private void exportApplicationsForOrg( Map.Entry<UUID, String> organization,final
ExportInfo config ) throws Exception {
 
@@ -316,7 +312,7 @@ public class ExportServiceImpl implements ExportService{
             // Close writer and file for this application.
             jg.writeEndArray();
             jg.close();
-            copyToS3( appFileName , config );
+            s3Export.copyToS3( appFileName, config );
             //below line doesn't copy very good data anyways.
             //copyToS3( collectionsFilename, config );
         }
@@ -419,56 +415,12 @@ public class ExportServiceImpl implements ExportService{
         return outputFileName;
     }
 
-    private void copyToS3( String fileName , final ExportInfo exportInfo) {
-
-        Logger logger = LoggerFactory.getLogger( ExportServiceImpl.class );
-        /*won't need any of the properties as I have the export info*/
-        String bucketName = exportInfo.getBucket_location();
-        String accessId = exportInfo.getS3_accessId();
-        String secretKey = exportInfo.getS3_key();
-
-        Properties overrides = new Properties();
-        overrides.setProperty( "s3" + ".identity", accessId );
-        overrides.setProperty( "s3" + ".credential", secretKey );
-
-        final Iterable<? extends Module> MODULES = ImmutableSet
-                .of( new JavaUrlHttpCommandExecutorServiceModule(), new Log4JLoggingModule(),
new NettyPayloadModule
-                        () );
-
-        BlobStoreContext context =
-                ContextBuilder.newBuilder( "s3" ).credentials( accessId, secretKey ).modules(
MODULES )
-                              .overrides( overrides ).buildView( BlobStoreContext.class );
-
-        // Create Container (the bucket in s3)
-        try {
-            AsyncBlobStore blobStore = context.getAsyncBlobStore(); // it can be changed
to sync
-            // BlobStore (returns false if it already exists)
-            ListenableFuture<Boolean> container = blobStore.createContainerInLocation(
null, bucketName );
-            if ( container.get() ) {
-                logger.info( "Created bucket " + bucketName );
-            }
-        }
-        catch ( Exception ex ) {
-            logger.error( "Could not start binary service: {}", ex.getMessage() );
-            throw new RuntimeException( ex );
-        }
+    @Autowired
+    @Override
+    public void setS3Export (S3Export s3Export) { this.s3Export = s3Export; }
 
-        try {
-            File file = new File( fileName );
-            AsyncBlobStore blobStore = context.getAsyncBlobStore();
-            BlobBuilder blobBuilder =
-                    blobStore.blobBuilder( file.getName() ).payload( file ).calculateMD5().contentType(
"text/plain" )
-                             .contentLength( file.length() );
 
-            Blob blob = blobBuilder.build();
 
-            ListenableFuture<String> futureETag = blobStore.putBlob( bucketName, blob,
PutOptions.Builder.multipart() );
 
-            logger.info( "Uploaded file etag=" + futureETag.get() );
-        }
-        catch ( Exception e ) {
-            logger.error( "Error uploading to blob store", e );
-        }
-    }
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68bd76be/stack/services/src/main/java/org/usergrid/management/export/S3Export.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/usergrid/management/export/S3Export.java b/stack/services/src/main/java/org/usergrid/management/export/S3Export.java
new file mode 100644
index 0000000..6895da5
--- /dev/null
+++ b/stack/services/src/main/java/org/usergrid/management/export/S3Export.java
@@ -0,0 +1,14 @@
+package org.usergrid.management.export;
+
+
+import org.usergrid.management.ExportInfo;
+
+
+/**
+ *
+ *
+ */
+public interface S3Export {
+    void copyToS3( String fileName, ExportInfo exportInfo );
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68bd76be/stack/services/src/main/java/org/usergrid/management/export/S3ExportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/usergrid/management/export/S3ExportImpl.java
b/stack/services/src/main/java/org/usergrid/management/export/S3ExportImpl.java
new file mode 100644
index 0000000..fc8e281
--- /dev/null
+++ b/stack/services/src/main/java/org/usergrid/management/export/S3ExportImpl.java
@@ -0,0 +1,83 @@
+package org.usergrid.management.export;
+
+
+import java.io.File;
+import java.util.Properties;
+
+import org.jclouds.ContextBuilder;
+import org.jclouds.blobstore.AsyncBlobStore;
+import org.jclouds.blobstore.BlobStoreContext;
+import org.jclouds.blobstore.domain.Blob;
+import org.jclouds.blobstore.domain.BlobBuilder;
+import org.jclouds.blobstore.options.PutOptions;
+import org.jclouds.http.config.JavaUrlHttpCommandExecutorServiceModule;
+import org.jclouds.logging.log4j.config.Log4JLoggingModule;
+import org.jclouds.netty.config.NettyPayloadModule;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.usergrid.management.ExportInfo;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.inject.Module;
+
+
+/**
+ *
+ *
+ */
+public class S3ExportImpl implements S3Export {
+    @Override
+    public void copyToS3( String fileName, final ExportInfo exportInfo ) {
+
+        Logger logger = LoggerFactory.getLogger( ExportServiceImpl.class );
+        /*won't need any of the properties as I have the export info*/
+        String bucketName = exportInfo.getBucket_location();
+        String accessId = exportInfo.getS3_accessId();
+        String secretKey = exportInfo.getS3_key();
+
+        Properties overrides = new Properties();
+        overrides.setProperty( "s3" + ".identity", accessId );
+        overrides.setProperty( "s3" + ".credential", secretKey );
+
+        final Iterable<? extends Module> MODULES = ImmutableSet
+                .of( new JavaUrlHttpCommandExecutorServiceModule(), new Log4JLoggingModule(),
new NettyPayloadModule
+                        () );
+
+        BlobStoreContext context =
+                ContextBuilder.newBuilder( "s3" ).credentials( accessId, secretKey ).modules(
MODULES )
+                              .overrides( overrides ).buildView( BlobStoreContext.class );
+
+        // Create Container (the bucket in s3)
+        try {
+            AsyncBlobStore blobStore = context.getAsyncBlobStore(); // it can be changed
to sync
+            // BlobStore (returns false if it already exists)
+            ListenableFuture<Boolean> container = blobStore.createContainerInLocation(
null, bucketName );
+            if ( container.get() ) {
+                logger.info( "Created bucket " + bucketName );
+            }
+        }
+        catch ( Exception ex ) {
+            logger.error( "Could not start binary service: {}", ex.getMessage() );
+            throw new RuntimeException( ex );
+        }
+
+        try {
+            File file = new File( fileName );
+            AsyncBlobStore blobStore = context.getAsyncBlobStore();
+            BlobBuilder blobBuilder =
+                    blobStore.blobBuilder( file.getName() ).payload( file ).calculateMD5().contentType(
"text/plain" )
+                             .contentLength( file.length() );
+
+            Blob blob = blobBuilder.build();
+
+            ListenableFuture<String> futureETag = blobStore.putBlob( bucketName, blob,
PutOptions.Builder.multipart() );
+
+            logger.info( "Uploaded file etag=" + futureETag.get() );
+        }
+        catch ( Exception e ) {
+            logger.error( "Error uploading to blob store", e );
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68bd76be/stack/services/src/main/resources/usergrid-services-context.xml
----------------------------------------------------------------------
diff --git a/stack/services/src/main/resources/usergrid-services-context.xml b/stack/services/src/main/resources/usergrid-services-context.xml
index a13eca5..05af9b5 100644
--- a/stack/services/src/main/resources/usergrid-services-context.xml
+++ b/stack/services/src/main/resources/usergrid-services-context.xml
@@ -71,4 +71,6 @@
     <property name="sch" ref="schedulerService"/>
   </bean>
 
+  <bean id="s3Export" class="org.usergrid.management.export.S3ExportImpl" />
+
 </beans>

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68bd76be/stack/services/src/test/java/org/usergrid/ServiceITSetup.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/usergrid/ServiceITSetup.java b/stack/services/src/test/java/org/usergrid/ServiceITSetup.java
index fbfb923..7cd370a 100644
--- a/stack/services/src/test/java/org/usergrid/ServiceITSetup.java
+++ b/stack/services/src/test/java/org/usergrid/ServiceITSetup.java
@@ -5,6 +5,7 @@ import java.util.Properties;
 
 import org.usergrid.management.ApplicationCreator;
 import org.usergrid.management.ManagementService;
+import org.usergrid.management.export.ExportService;
 import org.usergrid.security.providers.SignInProviderFactory;
 import org.usergrid.security.tokens.TokenService;
 import org.usergrid.services.ServiceManagerFactory;
@@ -21,6 +22,8 @@ public interface ServiceITSetup extends CoreITSetup {
 
     Properties getProps();
 
+    ExportService getExportService();
+
     /**
      * Convenience method to set a property in the Properties object returned by getProps();
      *

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68bd76be/stack/services/src/test/java/org/usergrid/ServiceITSetupImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/usergrid/ServiceITSetupImpl.java b/stack/services/src/test/java/org/usergrid/ServiceITSetupImpl.java
index cf6068f..026b61d 100644
--- a/stack/services/src/test/java/org/usergrid/ServiceITSetupImpl.java
+++ b/stack/services/src/test/java/org/usergrid/ServiceITSetupImpl.java
@@ -11,6 +11,7 @@ import org.springframework.beans.factory.config.PropertiesFactoryBean;
 import org.usergrid.cassandra.CassandraResource;
 import org.usergrid.management.ApplicationCreator;
 import org.usergrid.management.ManagementService;
+import org.usergrid.management.export.ExportService;
 import org.usergrid.persistence.cassandra.CassandraService;
 import org.usergrid.security.providers.SignInProviderFactory;
 import org.usergrid.security.tokens.TokenService;
@@ -27,6 +28,7 @@ public class ServiceITSetupImpl extends CoreITSetupImpl implements ServiceITSetu
     private TokenService tokenService;
     private SignInProviderFactory providerFactory;
     private Properties properties;
+    private ExportService exportService;
 
 
     public ServiceITSetupImpl( CassandraResource cassandraResource ) {
@@ -48,6 +50,7 @@ public class ServiceITSetupImpl extends CoreITSetupImpl implements ServiceITSetu
         providerFactory = cassandraResource.getBean( SignInProviderFactory.class );
         properties = cassandraResource.getBean( PropertiesFactoryBean.class ).getObject();
         smf = cassandraResource.getBean( ServiceManagerFactory.class );
+        exportService = cassandraResource.getBean( ExportService.class );
 
         LOG.info( "Test setup complete..." );
     }
@@ -109,6 +112,9 @@ public class ServiceITSetupImpl extends CoreITSetupImpl implements ServiceITSetu
         return properties;
     }
 
+    @Override
+    public ExportService getExportService() { return exportService; }
+
 
     @Override
     public Object set( String key, String value ) {

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68bd76be/stack/services/src/test/java/org/usergrid/management/cassandra/ManagementServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/usergrid/management/cassandra/ManagementServiceIT.java
b/stack/services/src/test/java/org/usergrid/management/cassandra/ManagementServiceIT.java
index a27ebb2..80da981 100644
--- a/stack/services/src/test/java/org/usergrid/management/cassandra/ManagementServiceIT.java
+++ b/stack/services/src/test/java/org/usergrid/management/cassandra/ManagementServiceIT.java
@@ -1,6 +1,7 @@
 package org.usergrid.management.cassandra;
 
 
+import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -15,14 +16,20 @@ import org.slf4j.LoggerFactory;
 import org.usergrid.ServiceITSetup;
 import org.usergrid.ServiceITSetupImpl;
 import org.usergrid.ServiceITSuite;
+import org.usergrid.batch.JobExecution;
 import org.usergrid.cassandra.CassandraResource;
 import org.usergrid.cassandra.ClearShiroSubject;
 import org.usergrid.cassandra.Concurrent;
+import org.usergrid.count.SimpleBatcher;
+import org.usergrid.management.ExportInfo;
 import org.usergrid.management.OrganizationInfo;
 import org.usergrid.management.UserInfo;
+import org.usergrid.management.export.ExportJob;
+import org.usergrid.management.export.S3Export;
 import org.usergrid.persistence.CredentialsInfo;
 import org.usergrid.persistence.Entity;
 import org.usergrid.persistence.EntityManager;
+import org.usergrid.persistence.entities.JobData;
 import org.usergrid.persistence.entities.User;
 import org.usergrid.security.AuthPrincipalType;
 import org.usergrid.security.crypto.command.Md5HashCommand;
@@ -32,18 +39,17 @@ import org.usergrid.security.tokens.exceptions.InvalidTokenException;
 import org.usergrid.utils.JsonUtils;
 import org.usergrid.utils.UUIDUtils;
 
-import org.usergrid.count.SimpleBatcher;
-
 import static org.apache.commons.codec.binary.Base64.encodeBase64URLSafeString;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 import static org.usergrid.persistence.Schema.DICTIONARY_CREDENTIALS;
 import static org.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
 
-
 /** @author zznate */
 @Concurrent()
 public class ManagementServiceIT {
@@ -726,4 +732,81 @@ public class ManagementServiceIT {
 
         assertEquals( userId, authedUser.getUuid() );
     }
+
+    /*Make this test the do export test and verify that it works using a mock method. */
+    //This test really should be called testDoExport as it mocks out sending it to s3.
+    @Test
+    public void testS3Export() throws Exception {
+
+        HashMap<String, Object> payload = new HashMap<String, Object>();
+        Map<String, Object> properties = new HashMap<String, Object>();
+        Map<String, Object> storage_info = new HashMap<String, Object>();
+        storage_info.put( "admin_token","insert_token_data_here" );
+        //TODO: always put dummy values here and ignore this test.
+        //TODO: add a ret for when s3 values are invalid.
+        storage_info.put( "s3_key","insert key here" );
+        storage_info.put( "s3_accessId","insert access id here");
+        storage_info.put( "bucket_location","insert bucket name here");
+
+
+        properties.put( "storage_provider","s3");
+        properties.put( "storage_info",storage_info);
+
+        payload.put( "path", "test-organization/test-app/user");
+        payload.put( "properties", properties);
+
+
+        ExportInfo exportInfo = new ExportInfo(payload);
+
+
+        S3Export s3Export = mock( S3Export.class );
+
+        setup.getExportService().setS3Export( s3Export );
+        setup.getExportService().doExport( exportInfo );
+
+    }
+
+    @Test
+    public void testExportDoJob() throws Exception {
+
+        //ExportService exportService = mock( ExportService.class );
+        HashMap<String, Object> payload = new HashMap<String, Object>();
+        Map<String, Object> properties = new HashMap<String, Object>();
+        Map<String, Object> storage_info = new HashMap<String, Object>();
+        storage_info.put( "admin_token","insert_token_data_here" );
+        //TODO: always put dummy values here and ignore this test.
+        //TODO: add a ret for when s3 values are invalid.
+        storage_info.put( "s3_key","insert key here" );
+        storage_info.put( "s3_accessId","insert access id here");
+        storage_info.put( "bucket_location","insert bucket name here");
+
+
+        properties.put( "storage_provider","s3");
+        properties.put( "storage_info",storage_info);
+
+        payload.put( "path", "test-organization/test-app/user");
+        payload.put( "properties", properties);
+
+
+        ExportInfo exportInfo = new ExportInfo(payload);
+
+
+        //ExportJob job = new ExportJob();
+        //ExportInfo exportInfo;
+
+        JobData jobData = new JobData();
+        jobData.setProperty( "jobName", "exportJob" );
+        jobData.setProperty( "ExportInfo", exportInfo ); //this needs to be populated with
properties of export info
+
+        JobExecution jobExecution = mock ( JobExecution.class);
+
+        when( jobExecution.getJobData() ).thenReturn( jobData );
+
+        ExportJob job = new ExportJob();
+        S3Export s3Export = mock( S3Export.class );
+        setup.getExportService().setS3Export( s3Export );
+        job.setExportService( setup.getExportService() );
+        job.doJob( jobExecution );
+
+    }
 }


Mime
View raw message