usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From snoopd...@apache.org
Subject [09/50] [abbrv] git commit: I uncommented out the schedule, but that method has nothing in it and doesn’t affect usage currently. Added field s3_accessId that will replace s3_token ( not currently but next commit). Changed some of the file name credentia
Date Tue, 25 Feb 2014 06:21:23 GMT
I uncommented out the schedule, but that method has nothing in it and doesn’t affect usage
currently.
Added field s3_accessId that will replace s3_token ( not currently but next commit).
Changed some of the file name credentials so that I could work using hardcoded paths.
Shamelessly took copytToS3 method form WarehouseExport . Added in export info for AWS credentials.
Functions as intended.


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/75ccff1f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/75ccff1f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/75ccff1f

Branch: refs/pull/60/head
Commit: 75ccff1fc193f7862ae96223ef201114807fe584
Parents: a57f6cd
Author: GERey <greyes@apigee.com>
Authored: Thu Feb 6 11:29:54 2014 -0800
Committer: GERey <greyes@apigee.com>
Committed: Thu Feb 6 11:29:54 2014 -0800

----------------------------------------------------------------------
 .../rest/management/ManagementResource.java     |  2 +-
 .../rest/management/ManagementResourceIT.java   |  7 +-
 .../org/usergrid/management/ExportInfo.java     |  4 +
 .../management/export/ExportServiceImpl.java    | 92 +++++++++++++++++---
 4 files changed, 91 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/75ccff1f/stack/rest/src/main/java/org/usergrid/rest/management/ManagementResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/usergrid/rest/management/ManagementResource.java
b/stack/rest/src/main/java/org/usergrid/rest/management/ManagementResource.java
index 926cacb..f6c66da 100644
--- a/stack/rest/src/main/java/org/usergrid/rest/management/ManagementResource.java
+++ b/stack/rest/src/main/java/org/usergrid/rest/management/ManagementResource.java
@@ -461,7 +461,7 @@ public class ManagementResource extends AbstractContextResource {
         ExportInfo objEx = new ExportInfo(json);
 
 
-        exportService.schedule(objEx);
+        //exportService.schedule(objEx);
         exportService.doExport( objEx );
         }
         catch (Exception e) {

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/75ccff1f/stack/rest/src/test/java/org/usergrid/rest/management/ManagementResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/usergrid/rest/management/ManagementResourceIT.java
b/stack/rest/src/test/java/org/usergrid/rest/management/ManagementResourceIT.java
index 2285476..e0d7029 100644
--- a/stack/rest/src/test/java/org/usergrid/rest/management/ManagementResourceIT.java
+++ b/stack/rest/src/test/java/org/usergrid/rest/management/ManagementResourceIT.java
@@ -561,10 +561,11 @@ public class ManagementResourceIT extends AbstractRestIT {
         HashMap<String, Object> payload = new HashMap<String, Object>();
         Map<String, Object> properties = new HashMap<String, Object>();
         Map<String, Object> storage_info = new HashMap<String, Object>();
-
+        //TODO: always put dummy values here and ignore this test.
         storage_info.put( "s3_token","insert_token_data_here" );
-        storage_info.put( "s3_key","insert_secret_here" );
-        storage_info.put( "bucket_location","insert_bucket_location_here");
+        storage_info.put( "s3_key","insert key here" );
+        storage_info.put( "s3_accessId","insert access id here");
+        storage_info.put( "bucket_location","insert bucket name here");
 
         properties.put( "storage_provider","s3");
         properties.put( "storage_info",storage_info);

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/75ccff1f/stack/services/src/main/java/org/usergrid/management/ExportInfo.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/usergrid/management/ExportInfo.java b/stack/services/src/main/java/org/usergrid/management/ExportInfo.java
index ee3f06b..e1dfb8e 100644
--- a/stack/services/src/main/java/org/usergrid/management/ExportInfo.java
+++ b/stack/services/src/main/java/org/usergrid/management/ExportInfo.java
@@ -12,6 +12,7 @@ public class ExportInfo {
     private Map<String, Object> properties;
     private String storage_provider;
     private Map<String, Object> storage_info;
+    private String s3_accessId;
     private String s3_token;
     private String s3_key;
     private String bucket_location;
@@ -21,6 +22,7 @@ public class ExportInfo {
         properties = (Map) exportData.get("properties");
         storage_provider = (String) properties.get ("storage_provider");
         storage_info = (Map) properties.get("storage_info");
+        s3_accessId = (String) storage_info.get("s3_accessId");
         s3_token = (String) storage_info.get("s3_token");
         s3_key = (String) storage_info.get("s3_key");
         bucket_location = (String) storage_info.get("bucket_location");
@@ -48,6 +50,8 @@ public class ExportInfo {
 
     public String getBucket_location () { return bucket_location; }
 
+    public String getS3_accessId () { return s3_accessId; }
+
 
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/75ccff1f/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java
b/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java
index 4ce635e..f980728 100644
--- a/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java
+++ b/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java
@@ -7,6 +7,7 @@ import java.io.PrintWriter;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Properties;
 import java.util.Set;
 import java.util.UUID;
 
@@ -14,6 +15,15 @@ import org.codehaus.jackson.JsonFactory;
 import org.codehaus.jackson.JsonGenerator;
 import org.codehaus.jackson.impl.DefaultPrettyPrinter;
 import org.codehaus.jackson.map.ObjectMapper;
+import org.jclouds.ContextBuilder;
+import org.jclouds.blobstore.AsyncBlobStore;
+import org.jclouds.blobstore.BlobStoreContext;
+import org.jclouds.blobstore.domain.Blob;
+import org.jclouds.blobstore.domain.BlobBuilder;
+import org.jclouds.blobstore.options.PutOptions;
+import org.jclouds.http.config.JavaUrlHttpCommandExecutorServiceModule;
+import org.jclouds.logging.log4j.config.Log4JLoggingModule;
+import org.jclouds.netty.config.NettyPayloadModule;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.usergrid.batch.service.SchedulerService;
@@ -28,6 +38,9 @@ import org.usergrid.persistence.Results;
 import org.usergrid.persistence.cassandra.CassandraService;
 
 import com.google.common.collect.BiMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.inject.Module;
 
 
 /**
@@ -57,7 +70,9 @@ public class ExportServiceImpl implements ExportService{
 
     protected long startTime = System.currentTimeMillis();
 
-    protected static final String PATH_REPLACEMENT = "USERGIRD-PATH-BACKSLASH";
+    protected static final String PATH_REPLACEMENT = "/Users/ApigeeCorporation/";
+
+    private String filename = PATH_REPLACEMENT;
 
     //TODO: Todd, do I refactor most of the methods out to just leave schedule and doExport
much like
     //the exporting toolbase class?
@@ -85,7 +100,7 @@ public class ExportServiceImpl implements ExportService{
 //                continue;
 //            }
 
-            exportApplicationsForOrg( organization );
+            exportApplicationsForOrg( organization , config );
         }
     }
 
@@ -151,7 +166,7 @@ public class ExportServiceImpl implements ExportService{
     }
 
 
-    private void exportApplicationsForOrg( Map.Entry<UUID, String> organization ) throws
Exception {
+    private void exportApplicationsForOrg( Map.Entry<UUID, String> organization,final
ExportInfo config ) throws Exception {
 
         Logger logger = LoggerFactory.getLogger( ExportServiceImpl.class );
 
@@ -165,9 +180,10 @@ public class ExportServiceImpl implements ExportService{
             logger.info( application.getValue() + " : " + application.getKey() );
 
             // Get the JSon serializer.
-           // JsonGenerator jg = getJsonGenerator( createOutputFile( "application", application.getValue()
) );
+            //Creates the applications folder
+            JsonGenerator jg = getJsonGenerator( createOutputFile( "application", application.getValue()
) );
 
-            JsonGenerator jg = getJsonGenerator( new File( "/Users/ApigeeCorporation/derp.txt"
));
+            //JsonGenerator jg = getJsonGenerator( new File( "/Users/ApigeeCorporation/derp.txt"
));
 
 
 
@@ -213,6 +229,7 @@ public class ExportServiceImpl implements ExportService{
 
             // Create a GENERATOR for the application collections.
             JsonGenerator collectionsJg = getJsonGenerator( createOutputFile( "collections",
application.getValue() ) );
+
             collectionsJg.writeStartObject();
 
             Map<String, Object> metadata = em.getApplicationCollectionMetadata();
@@ -259,6 +276,7 @@ public class ExportServiceImpl implements ExportService{
             // Close writer and file for this application.
             jg.writeEndArray();
             jg.close();
+            copyToS3( filename , config );
         }
     }
 
@@ -318,6 +336,7 @@ public class ExportServiceImpl implements ExportService{
 
 
     protected JsonGenerator getJsonGenerator( File outFile ) throws IOException {
+        //TODO:shouldn't the below be UTF-16?
         PrintWriter out = new PrintWriter( outFile, "UTF-8" );
         JsonGenerator jg = jsonFactory.createJsonGenerator( out );
         jg.setPrettyPrinter( new DefaultPrettyPrinter() );
@@ -326,8 +345,7 @@ public class ExportServiceImpl implements ExportService{
     }
 
     protected File createOutputFile( String type, String name ) {
-        return new File ("helper");
-       // return new File( outputDir, prepareOutputFileName( type, name ) );
+        return new File(prepareOutputFileName( type, name ) );
     }
 
 
@@ -337,23 +355,77 @@ public class ExportServiceImpl implements ExportService{
      * @return the file name concatenated with the type and the name of the collection
      */
     protected String prepareOutputFileName( String type, String name ) {
-        name = name.replace( "/", PATH_REPLACEMENT );
+        //name = name.replace( "/", PATH_REPLACEMENT );
         // Add application and timestamp
         StringBuilder str = new StringBuilder();
         // str.append(baseOutputFileName);
         // str.append(".");
-        str.append( type );
-        str.append( "." );
+        str.append( PATH_REPLACEMENT );
+        //str.append( type );
+        //str.append( "." );
         str.append( name );
         str.append( "." );
         str.append( startTime );
         str.append( ".json" );
 
         String outputFileName = str.toString();
+        filename = outputFileName;
 
         //logger.info( "Creating output filename:" + outputFileName );
 
         return outputFileName;
     }
 
+    private void copyToS3( String fileName , final ExportInfo exportInfo) {
+
+        Logger logger = LoggerFactory.getLogger( ExportServiceImpl.class );
+        /*won't need any of the properties as I have the export info*/
+        String bucketName = exportInfo.getBucket_location();
+        String accessId = exportInfo.getS3_accessId();
+        String secretKey = exportInfo.getS3_key();
+
+        Properties overrides = new Properties();
+        overrides.setProperty( "s3" + ".identity", accessId );
+        overrides.setProperty( "s3" + ".credential", secretKey );
+
+        final Iterable<? extends Module> MODULES = ImmutableSet
+                .of( new JavaUrlHttpCommandExecutorServiceModule(), new Log4JLoggingModule(),
new NettyPayloadModule
+                        () );
+
+        BlobStoreContext context =
+                ContextBuilder.newBuilder( "s3" ).credentials( accessId, secretKey ).modules(
MODULES )
+                              .overrides( overrides ).buildView( BlobStoreContext.class );
+
+        // Create Container (the bucket in s3)
+        try {
+            AsyncBlobStore blobStore = context.getAsyncBlobStore(); // it can be changed
to sync
+            // BlobStore (returns false if it already exists)
+            ListenableFuture<Boolean> container = blobStore.createContainerInLocation(
null, bucketName );
+            if ( container.get() ) {
+                logger.info( "Created bucket " + bucketName );
+            }
+        }
+        catch ( Exception ex ) {
+            logger.error( "Could not start binary service: {}", ex.getMessage() );
+            throw new RuntimeException( ex );
+        }
+
+        try {
+            File file = new File( fileName );
+            AsyncBlobStore blobStore = context.getAsyncBlobStore();
+            BlobBuilder blobBuilder =
+                    blobStore.blobBuilder( file.getName() ).payload( file ).calculateMD5().contentType(
"text/plain" )
+                             .contentLength( file.length() );
+
+            Blob blob = blobBuilder.build();
+
+            ListenableFuture<String> futureETag = blobStore.putBlob( bucketName, blob,
PutOptions.Builder.multipart() );
+
+            logger.info( "Uploaded file etag=" + futureETag.get() );
+        }
+        catch ( Exception e ) {
+            logger.error( "Error uploading to blob store", e );
+        }
+    }
+
 }


Mime
View raw message