usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From toddn...@apache.org
Subject [24/50] [abbrv] incubator-usergrid git commit: Merge branch 'two-dot-o-dev' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into two-dot-o-dev
Date Mon, 01 Jun 2015 21:48:33 GMT
Merge branch 'two-dot-o-dev' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid
into two-dot-o-dev


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/265f84bf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/265f84bf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/265f84bf

Branch: refs/heads/USERGRID-628
Commit: 265f84bf6268b2a4cdb7a224be7b4d534b399df9
Parents: a0d9a61 ceadc6c
Author: Dave Johnson <dmjohnson@apigee.com>
Authored: Thu May 28 15:41:36 2015 -0400
Committer: Dave Johnson <dmjohnson@apigee.com>
Committed: Thu May 28 15:41:36 2015 -0400

----------------------------------------------------------------------
 .../corepersistence/CpEntityManagerFactory.java        |  9 +++++++--
 stack/pom.xml                                          |  2 +-
 stack/rest/src/test/resources/log4j.properties         |  3 ++-
 stack/services/pom.xml                                 | 10 ++++++++++
 .../usergrid/management/export/S3ExportImpl.java       |  7 +++++++
 .../usergrid/services/assets/data/S3BinaryStore.java   | 13 +++++++++++--
 .../apache/usergrid/management/importer/S3Upload.java  |  9 ++++++++-
 .../usergrid/setup/ConcurrentProcessSingleton.java     |  8 ++++++--
 8 files changed, 52 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/265f84bf/stack/pom.xml
----------------------------------------------------------------------
diff --cc stack/pom.xml
index ecdefb1,ce78baf..349869c
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@@ -105,7 -105,7 +105,7 @@@
        <hector-test-version>1.1-4</hector-test-version>
        <jackson-version>1.9.9</jackson-version>
        <jackson-2-version>2.3.3</jackson-2-version>
--      <jclouds.version>1.8.0</jclouds.version>
++      <jclouds.version>1.9.0</jclouds.version>
        <jersey-version>1.18.1</jersey-version>
        <junit-version>4.12</junit-version>
        <log4j-version>1.2.16</log4j-version>

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/265f84bf/stack/rest/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --cc stack/rest/src/test/resources/log4j.properties
index 27e35c8,27e35c8..e0aa09d
--- a/stack/rest/src/test/resources/log4j.properties
+++ b/stack/rest/src/test/resources/log4j.properties
@@@ -27,6 -27,6 +27,7 @@@ log4j.appender.stdout.layout=org.apache
  log4j.appender.stdout.layout.ConversionPattern=%d %p (%t) %c{1} - %m%n
  
  log4j.logger.org.apache.usergrid=ERROR
++log4j.logger.org.apache.usergrid.setup=DEBUG
  #log4j.logger.org.apache.usergrid.cassandra=DEBUG
  #log4j.logger.org.apache.usergrid.persistence.cassandra=DEBUG
  
@@@ -66,7 -66,7 +67,7 @@@ log4j.logger.org.apache.usergrid.rest.f
  #log4j.logger.org.apache.usergrid.services.notifiers=DEBUG
  #log4j.logger.org.apache.usergrid.services.groups.users.devices=DEBUG
  
--log4j.logger.org.apache.usergrid.rest=INFO
++log4j.logger.org.apache.usergrid.rest=DEBUG
  log4j.logger.org.apache.usergrid.rest.exceptions=DEBUG
  #log4j.logger.org.apache.usergrid.rest.NotificationsIT=DEBUG
  #log4j.logger.org.apache.usergrid.cassandra.CassandraResource=DEBUG

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/265f84bf/stack/services/pom.xml
----------------------------------------------------------------------
diff --cc stack/services/pom.xml
index 820b99f,ace7e79..c6dd815
--- a/stack/services/pom.xml
+++ b/stack/services/pom.xml
@@@ -248,6 -237,6 +248,16 @@@
      <dependency>
        <groupId>org.apache.jclouds</groupId>
        <artifactId>jclouds-core</artifactId>
++        <exclusions>
++            <exclusion>
++                <artifactId>guice-assistedinject</artifactId>
++                <groupId>com.google.inject.extensions</groupId>
++            </exclusion>
++            <exclusion>
++                <artifactId>guice</artifactId>
++                <groupId>com.google.inject</groupId>
++            </exclusion>
++        </exclusions>
      </dependency>
  
      <dependency>

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/265f84bf/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
----------------------------------------------------------------------
diff --cc stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
index 4fff7c7,4fff7c7..f8005d1
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
@@@ -82,10 -82,10 +82,17 @@@ public class S3ExportImpl implements S3
  
          try {
              BlobStore blobStore = context.getBlobStore();
++
++            // need this for JClouds 1.7.x:
++//            BlobBuilder blobBuilder =
++//                blobStore.blobBuilder( filename ).payload( ephemeral ).calculateMD5().contentType(
"application/json" );
++
++            // needed for JClouds 1.8.x:
              BlobBuilder blobBuilder = blobStore.blobBuilder( filename )
                  .payload( ephemeral )
                  .contentMD5(Files.hash( ephemeral, Hashing.md5() ))
                  .contentType("application/json");
++
              Blob blob = blobBuilder.build();
  
              final String uploadedFile = blobStore.putBlob(

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/265f84bf/stack/services/src/main/java/org/apache/usergrid/services/assets/data/S3BinaryStore.java
----------------------------------------------------------------------
diff --cc stack/services/src/main/java/org/apache/usergrid/services/assets/data/S3BinaryStore.java
index a805e42,90bd24f..d028852
--- a/stack/services/src/main/java/org/apache/usergrid/services/assets/data/S3BinaryStore.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/assets/data/S3BinaryStore.java
@@@ -17,19 -17,26 +17,18 @@@
  package org.apache.usergrid.services.assets.data;
  
  
 -import java.io.BufferedOutputStream;
 -import java.io.ByteArrayOutputStream;
 -import java.io.File;
 -import java.io.FileOutputStream;
 -import java.io.IOException;
 -import java.io.InputStream;
 -import java.io.OutputStream;
 -import java.security.MessageDigest;
 -import java.security.NoSuchAlgorithmException;
 -import java.util.Map;
 -import java.util.UUID;
 -import java.util.concurrent.ExecutorService;
 -import java.util.concurrent.Executors;
 -
 -import com.google.common.hash.HashCode;
 -import com.google.common.hash.HashFunction;
 +import com.google.common.collect.ImmutableSet;
- import com.google.common.hash.HashFunction;
  import com.google.common.hash.Hashing;
  import com.google.common.io.Files;
 +import com.google.inject.Module;
 +import org.apache.commons.codec.binary.Hex;
 +import org.apache.commons.io.FileUtils;
 +import org.apache.commons.io.IOUtils;
 +import org.apache.usergrid.persistence.Entity;
 +import org.apache.usergrid.persistence.EntityManager;
 +import org.apache.usergrid.persistence.EntityManagerFactory;
 +import org.apache.usergrid.utils.StringUtils;
  import org.jclouds.ContextBuilder;
 -import org.jclouds.blobstore.AsyncBlobStore;
  import org.jclouds.blobstore.BlobStore;
  import org.jclouds.blobstore.BlobStoreContext;
  import org.jclouds.blobstore.domain.Blob;
@@@ -41,16 -49,19 +40,15 @@@ import org.jclouds.netty.config.NettyPa
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  import org.springframework.beans.factory.annotation.Autowired;
 -import org.apache.usergrid.persistence.Entity;
 -import org.apache.usergrid.persistence.EntityManager;
 -import org.apache.usergrid.persistence.EntityManagerFactory;
 -
 -import org.apache.commons.codec.binary.Hex;
 -import org.apache.commons.io.FileUtils;
 -import org.apache.commons.io.IOUtils;
 -
 -import com.google.common.collect.ImmutableSet;
 -import com.google.common.util.concurrent.ListenableFuture;
 -import com.google.inject.Module;
  
 +import java.io.*;
 +import java.util.Map;
 +import java.util.Properties;
 +import java.util.UUID;
 +import java.util.concurrent.Callable;
 +import java.util.concurrent.ExecutorService;
 +import java.util.concurrent.Executors;
  
- 
  public class S3BinaryStore implements BinaryStore {
  
      private static final Iterable<? extends Module> MODULES = ImmutableSet
@@@ -109,19 -115,18 +107,25 @@@
          long written = IOUtils.copyLarge( inputStream, baos, 0, FIVE_MB );
          byte[] data = baos.toByteArray();
  
 -        final Map<String, Object> fileMetadata = AssetUtils.getFileMetadata( entity
);
 -        fileMetadata.put( AssetUtils.LAST_MODIFIED, System.currentTimeMillis() );
 +        if ( written < FIVE_MB ) { // total smaller than 5mb
  
 -        String mimeType = AssetMimeHandler.get().getMimeType( entity, data );
 +            final String uploadFileName = AssetUtils.buildAssetKey( appId, entity );
 +            final String mimeType = AssetMimeHandler.get().getMimeType( entity, data );
  
 -        if ( written < FIVE_MB ) { // total smaller than 5mb
 +            final Map<String, Object> fileMetadata = AssetUtils.getFileMetadata( entity
);
 +            fileMetadata.put( AssetUtils.LAST_MODIFIED, System.currentTimeMillis() );
  
              BlobStore blobStore = getContext().getBlobStore();
++
++            // need this for JClouds 1.7.x:
++//            BlobBuilder.PayloadBlobBuilder bb =  blobStore.blobBuilder( uploadFileName
)
++//                .payload( data ).calculateMD5().contentType( mimeType );
++
++            // need this for JClouds 1.8.x:
              BlobBuilder.PayloadBlobBuilder bb = blobStore.blobBuilder(uploadFileName)
 -                .payload(data)
 -                .contentMD5(Hashing.md5().newHasher().putBytes( data ).hash())
 -                .contentType(mimeType);
 +                .payload( data )
 +                .contentMD5( Hashing.md5().newHasher().putBytes( data ).hash() )
 +                .contentType( mimeType );
  
              fileMetadata.put( AssetUtils.CONTENT_LENGTH, written );
              if ( fileMetadata.get( AssetUtils.CONTENT_DISPOSITION ) != null ) {
@@@ -193,133 -224,5 +197,138 @@@
          BlobStore blobStore = getContext().getBlobStore();
          blobStore.removeBlob( bucketName, AssetUtils.buildAssetKey( appId, entity ) );
      }
 +
 +    class UploadWorker implements Callable<Void> {
 +
 +        private UUID appId;
 +        private Entity entity;
 +        private InputStream inputStream;
 +        private byte[] data;
 +        private long written;
 +
 +
 +        public UploadWorker( UUID appId, Entity entity, InputStream is, byte[] data, long
written ) {
 +            this.appId = appId;
 +            this.entity = entity;
 +            this.inputStream = is;
 +            this.data = data;
 +            this.written = written;
 +        }
 +
 +        @Override
 +        public Void call() {
 +
 +            LOG.debug( "Writing temp file for S3 upload" );
 +
 +            // determine max size file allowed, default to 50mb
 +            long maxSizeBytes = 50 * FileUtils.ONE_MB;
 +            String maxSizeMbString = properties.getProperty( "usergrid.binary.max-size-mb",
"50" );
 +            if (StringUtils.isNumeric( maxSizeMbString )) {
 +                maxSizeBytes = Long.parseLong( maxSizeMbString ) * FileUtils.ONE_MB;
 +            }
 +
 +            // always allow files up to 5mb
 +            if (maxSizeBytes < 5 * FileUtils.ONE_MB ) {
 +                maxSizeBytes = 5 * FileUtils.ONE_MB;
 +            }
 +
 +            // write temporary file, slightly larger than our size limit
 +            OutputStream os = null;
 +            File tempFile;
 +            try {
 +                tempFile = File.createTempFile( entity.getUuid().toString(), "tmp" );
 +                tempFile.deleteOnExit();
 +                os = new BufferedOutputStream( new FileOutputStream( tempFile.getAbsolutePath()
) );
 +                os.write( data );
 +                written += data.length;
 +                written += IOUtils.copyLarge( inputStream, os, 0, maxSizeBytes + 1 );
 +
 +                LOG.debug("Write temp file {} length {}", tempFile.getName(), written);
 +
 +            } catch ( IOException e ) {
 +                throw new RuntimeException( "Error creating temp file", e );
 +
 +            } finally {
 +                if ( os != null ) {
 +                    try {
 +                        os.flush();
 +                    } catch (IOException e) {
 +                        LOG.error( "Error flushing data to temporary upload file", e );
 +                    }
 +                    IOUtils.closeQuietly( os );
 +                }
 +            }
 +
 +            // if tempFile is too large, delete it, add error to entity file metadata and
abort
 +
 +            Map<String, Object> fileMetadata = AssetUtils.getFileMetadata( entity
);
 +
 +            if ( tempFile.length() > maxSizeBytes ) {
 +                LOG.debug("File too large. Temp file size (bytes) = {}, " +
 +                          "Max file size (bytes) = {} ", tempFile.length(), maxSizeBytes);
 +                try {
 +                    EntityManager em = emf.getEntityManager( appId );
 +                    fileMetadata.put( "error", "Asset size " + tempFile.length()
 +                                    + " is larger than max size of " + maxSizeBytes );
 +                    em.update( entity );
 +                    tempFile.delete();
 +
 +                } catch ( Exception e ) {
 +                    LOG.error( "Error updating entity with error message", e);
 +                }
 +                return null;
 +            }
 +
 +            String uploadFileName = AssetUtils.buildAssetKey( appId, entity );
 +            String mimeType = AssetMimeHandler.get().getMimeType( entity, data );
 +
 +            try {  // start the upload
 +
 +                LOG.debug( "S3 upload thread started" );
 +
 +                BlobStore blobStore = getContext().getBlobStore();
 +
++                // need this for JClouds 1.7.x:
++//                BlobBuilder.PayloadBlobBuilder bb =  blobStore.blobBuilder( uploadFileName
)
++//                    .payload( tempFile ).calculateMD5().contentType( mimeType );
++
++                // need this for JClouds 1.8.x:
 +                BlobBuilder.PayloadBlobBuilder bb = blobStore.blobBuilder( uploadFileName
)
 +                    .payload( tempFile )
 +                    .contentMD5( Files.hash( tempFile, Hashing.md5() ) )
 +                    .contentType( mimeType );
 +
 +                if ( fileMetadata.get( AssetUtils.CONTENT_DISPOSITION ) != null ) {
 +                    bb.contentDisposition( fileMetadata.get( AssetUtils.CONTENT_DISPOSITION
).toString() );
 +                }
 +                final Blob blob = bb.build();
 +
 +                String md5sum = Hex.encodeHexString( blob.getMetadata().getContentMetadata().getContentMD5()
);
 +                fileMetadata.put( AssetUtils.CHECKSUM, md5sum );
 +
 +                LOG.debug( "S3 upload starting" );
 +
 +                String eTag = blobStore.putBlob( bucketName, blob );
 +
 +                LOG.debug( "S3 upload complete eTag=" + eTag);
 +
 +                // update entity with eTag
 +                EntityManager em = emf.getEntityManager( appId );
 +                fileMetadata.put( AssetUtils.LAST_MODIFIED, System.currentTimeMillis() );
 +                fileMetadata.put( AssetUtils.CONTENT_LENGTH, written );
 +                fileMetadata.put( AssetUtils.E_TAG, eTag );
 +                em.update( entity );
 +            }
 +            catch ( Exception e ) {
 +                LOG.error( "error uploading", e );
 +            }
 +
 +            if ( tempFile != null && tempFile.exists() ) {
 +                tempFile.delete();
 +            }
 +
 +            return null;
 +        }
 +    }
  }
  

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/265f84bf/stack/services/src/test/java/org/apache/usergrid/management/importer/S3Upload.java
----------------------------------------------------------------------
diff --cc stack/services/src/test/java/org/apache/usergrid/management/importer/S3Upload.java
index c56ff66,c56ff66..e76c90a
--- a/stack/services/src/test/java/org/apache/usergrid/management/importer/S3Upload.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/importer/S3Upload.java
@@@ -87,13 -87,13 +87,20 @@@ public class S3Upload 
  
              String filename = fileNameIterator.next();
              File uploadFile = new File( filename );
--            
++
              try {
                  BlobStore blobStore = context.getBlobStore();
++
++                // need this for JClouds 1.7.x:
++//                BlobBuilder.PayloadBlobBuilder blobBuilder =  blobStore.blobBuilder( filename
)
++//                    .payload( uploadFile ).calculateMD5().contentType( "application/json"
);
++
++                // needed for JClouds 1.8.x:
                  BlobBuilder blobBuilder = blobStore.blobBuilder( filename )
                      .payload( uploadFile )
                      .contentMD5(Files.hash( uploadFile, Hashing.md5()))
                      .contentType( "application/json" );
++
                  Blob blob = blobBuilder.build();
  
                  final String uploadedFile = blobStore.putBlob( bucketName, blob, PutOptions.Builder.multipart()
);

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/265f84bf/stack/test-utils/src/main/java/org/apache/usergrid/setup/ConcurrentProcessSingleton.java
----------------------------------------------------------------------
diff --cc stack/test-utils/src/main/java/org/apache/usergrid/setup/ConcurrentProcessSingleton.java
index 7cb5073,7cb5073..3b08bc8
--- a/stack/test-utils/src/main/java/org/apache/usergrid/setup/ConcurrentProcessSingleton.java
+++ b/stack/test-utils/src/main/java/org/apache/usergrid/setup/ConcurrentProcessSingleton.java
@@@ -89,14 -89,14 +89,18 @@@ public class ConcurrentProcessSingleto
                  // maybe delete existing column families and indexes
                  if ( CLEAN_STORAGE ) {
                      logger.info("Destroying current database");
--                    schemaManager.destroy();
++                    try {
++                        schemaManager.destroy();
++                    } catch ( Exception e ) {
++                        logger.error("Exception on destroying current database, continuing",
e);
++                    }
                  }
  
                  // create our schema
                  logger.info("Creating database");
                  schemaManager.create();
  
--                logger.info("Populating database");
++                logger.info( "Populating database" );
                  schemaManager.populateBaseData();
  
                  // signal to other processes we've migrated, and they can proceed


Mime
View raw message