usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From snoopd...@apache.org
Subject [05/27] git commit: Add query tests, but doesn’t protect the users from themselves. Reduced number of max entities so people don’t crash the JVM with huge queries.
Date Tue, 01 Apr 2014 01:20:57 GMT
Add query tests, but doesn’t protect the users from themselves.
Reduced number of max entities so people don’t crash the JVM with huge queries.


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/1a5f8b47
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/1a5f8b47
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/1a5f8b47

Branch: refs/pull/92/head
Commit: 1a5f8b479e4dfbfc4036032d6814b931fa9e7aa0
Parents: 844fd5d
Author: grey <greyes@apigee.com>
Authored: Mon Mar 31 11:28:11 2014 -0700
Committer: grey <greyes@apigee.com>
Committed: Mon Mar 31 11:28:11 2014 -0700

----------------------------------------------------------------------
 .../management/export/ExportServiceImpl.java    |  30 ++++-
 .../cassandra/ManagementServiceIT.java          | 109 +++++++++++++++----
 2 files changed, 113 insertions(+), 26 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1a5f8b47/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
index 053a4e3..92e76bc 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
@@ -71,7 +71,7 @@ public class ExportServiceImpl implements ExportService {
     private ManagementService managementService;
 
     //Maximum amount of entities retrieved in a single go.
-    public static final int MAX_ENTITY_FETCH = 100000;
+    public static final int MAX_ENTITY_FETCH = 1000;
 
     //Amount of time that has passed before sending another heart beat in millis
     public static final int TIMESTAMP_DELTA = 5000;
@@ -314,7 +314,13 @@ public class ExportServiceImpl implements ExportService {
                 if ( ( config.get( "collectionName" ) == null ) || collectionName
                         .equals( config.get( "collectionName" ) ) ) {
                     //Query entity manager for the entities in a collection
-                    Query query = new Query();
+                    Query query;
+                    if(config.get( "query" ) == null) {
+                        query = new Query(  );
+                    }
+                    else {
+                        query = Query.fromQL( ( String ) config.get( "query" ) );
+                    }
                     query.setLimit( MAX_ENTITY_FETCH );
                     query.setResultsLevel( Results.Level.ALL_PROPERTIES );
                     Results entities = em.searchCollection( em.getApplicationRef(), collectionName,
query );
@@ -394,9 +400,19 @@ public class ExportServiceImpl implements ExportService {
             if ( ( config.get( "collectionName" ) == null ) || collectionName
                     .equals( config.get( "collectionName" ) ) ) {
                 //Query entity manager for the entities in a collection
-                Query query = new Query();
+                Query query;
+                if(config.get( "query" ) == null) {
+                    query = new Query(  );
+                }
+                else {
+                    query = Query.fromQL( ( String ) config.get( "query" ) );
+                }
+               // Query query = Query.fromQL( ( String ) config.get( "query" ) ); //new Query();
                 query.setLimit( MAX_ENTITY_FETCH );
                 query.setResultsLevel( Results.Level.ALL_PROPERTIES );
+                query.setCollection( collectionName );
+                //query.setQl( ( String ) config.get( "query" ) );
+
                 Results entities = em.searchCollection( em.getApplicationRef(), collectionName,
query );
 
                 //pages through the query and backs up all results.
@@ -467,7 +483,13 @@ public class ExportServiceImpl implements ExportService {
             //if the collection you are looping through doesn't match the name of the one
you want. Don't export it.
             if ( collectionName.equals( ( String ) config.get( "collectionName" ) ) ) {
                 //Query entity manager for the entities in a collection
-                Query query = new Query();
+                Query query;
+                if(config.get( "query" ) == null) {
+                    query = new Query(  );
+                }
+                else {
+                    query = Query.fromQL( ( String ) config.get( "query" ) );
+                }
                 query.setLimit( MAX_ENTITY_FETCH );
                 query.setResultsLevel( Results.Level.ALL_PROPERTIES );
                 Results entities = em.searchCollection( em.getApplicationRef(), collectionName,
query );

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1a5f8b47/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
index fa8dd0c..cb817f0 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
@@ -1112,6 +1112,8 @@ public class ManagementServiceIT {
         catch ( Exception e ) {
             //consumed because this checks to see if the file exists. If it doesn't, don't
do anything and carry on.
         }
+        f.deleteOnExit();
+
 
         UUID appId = setup.getEmf().createApplication( orgName, appName );
 
@@ -1161,27 +1163,25 @@ public class ManagementServiceIT {
             String entityName = ( String ) entityData.get( "name" );
             assertFalse( "junkRealName".equals( entityName ) );
         }
-        f.deleteOnExit();
     }
-//
+
     @Test
-    public void testExportOneAppOnApplicationEndpoint() throws Exception {
+    public void testExportOneAppOnApplicationEndpointWQuery() throws Exception {
 
         File f = null;
-        String orgName = "ed-organization";
-        String appName = "testAppNotExported";
 
         try {
-            f = new File( "exportOneApp.json" );
+            f = new File( "exportOneAppWQuery.json" );
         }
         catch ( Exception e ) {
             //consumed because this checks to see if the file exists. If it doesn't, don't
do anything and carry on.
         }
 
-        UUID appId = setup.getEmf().createApplication( orgName, appName );
+        f.deleteOnExit();
 
 
-        EntityManager em = setup.getEmf().getEntityManager( appId );
+
+        EntityManager em = setup.getEmf().getEntityManager( applicationId );
         //intialize user object to be posted
         Map<String, Object> userProperties = null;
         Entity[] entity;
@@ -1189,16 +1189,19 @@ public class ManagementServiceIT {
         //creates entities
         for ( int i = 0; i < 1; i++ ) {
             userProperties = new LinkedHashMap<String, Object>();
+            userProperties.put("name","me");
             userProperties.put( "username", "junkRealName");
-            userProperties.put( "email", "test" + i + "@anuff.com" );//String.format( "test%i@anuff.com",
i ) );
+            userProperties.put( "email", "burp" + i + "@anuff.com" );//String.format( "test%i@anuff.com",
i ) );
             entity[i] = em.create( "users", userProperties );
         }
 
         S3Export s3Export = new MockS3ExportImpl();
-        s3Export.setFilename( "exportOneApp.json" );
+        s3Export.setFilename( "exportOneAppWQuery.json" );
         ExportService exportService = setup.getExportService();
         HashMap<String, Object> payload = payloadBuilder();
 
+        payload.put( "query","select * where username = 'junkRealName'" );
+
         payload.put( "organizationId",organization.getUuid() );
         payload.put( "applicationId",applicationId);
 
@@ -1219,17 +1222,15 @@ public class ManagementServiceIT {
 
         org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader(
f ) );
 
-        assertEquals( 3 , a.size() );
+        assertEquals( 1 , a.size() );
         for ( int i = 0; i < a.size(); i++ ) {
             org.json.simple.JSONObject data = ( org.json.simple.JSONObject ) a.get( i );
             org.json.simple.JSONObject entityData = ( JSONObject ) data.get( "Metadata" );
             String entityName = ( String ) entityData.get( "name" );
             assertFalse( "junkRealName".equals( entityName ) );
         }
-
-        f.deleteOnExit();
-
     }
+
 //
     @Test
     public void testExportOneCollection() throws Exception {
@@ -1245,7 +1246,7 @@ public class ManagementServiceIT {
         }
 
         EntityManager em = setup.getEmf().getEntityManager( applicationId);
-        em.createApplicationCollection( "baconators" );
+        em.createApplicationCollection( "qt" );
         //intialize user object to be posted
         Map<String, Object> userProperties = null;
         Entity[] entity;
@@ -1255,7 +1256,7 @@ public class ManagementServiceIT {
             userProperties = new LinkedHashMap<String, Object>();
             userProperties.put( "username", "billybob" + i );
             userProperties.put( "email", "test" + i + "@anuff.com" );//String.format( "test%i@anuff.com",
i ) );
-            entity[i] = em.create( "baconators", userProperties );
+            entity[i] = em.create( "qts", userProperties );
         }
 
         S3Export s3Export = new MockS3ExportImpl();
@@ -1265,7 +1266,7 @@ public class ManagementServiceIT {
 
         payload.put( "organizationId",organization.getUuid() );
         payload.put( "applicationId",applicationId);
-        payload.put( "collectionName","baconators");
+        payload.put( "collectionName","qts");
 
         UUID exportUUID = exportService.schedule( payload );
         exportService.setS3Export( s3Export );
@@ -1289,6 +1290,70 @@ public class ManagementServiceIT {
 
     }
 
+    @Test
+    public void testExportOneCollectionWQuery() throws Exception {
+
+        File f = null;
+        int entitiesToCreate = 5;
+
+        try {
+            f = new File( "exportOneCollectionWQuery.json" );
+        }
+        catch ( Exception e ) {
+            //consumed because this checks to see if the file exists. If it doesn't, don't
do anything and carry on.
+        }
+        f.deleteOnExit();
+
+        EntityManager em = setup.getEmf().getEntityManager( applicationId);
+        em.createApplicationCollection( "baconators" );
+        //intialize user object to be posted
+        Map<String, Object> userProperties = null;
+        Entity[] entity;
+        entity = new Entity[entitiesToCreate];
+        //creates entities
+        for ( int i = 0; i < entitiesToCreate; i++ ) {
+            userProperties = new LinkedHashMap<String, Object>();
+            userProperties.put( "username", "billybob" + i );
+            userProperties.put( "email", "test" + i + "@anuff.com" );//String.format( "test%i@anuff.com",
i ) );
+            entity[i] = em.create( "baconators", userProperties );
+        }
+
+        S3Export s3Export = new MockS3ExportImpl();
+        s3Export.setFilename( "exportOneCollectionWQuery.json" );
+        ExportService exportService = setup.getExportService();
+        HashMap<String, Object> payload = payloadBuilder();
+        payload.put( "query","select * where username contains 'billybob0'" );
+
+        payload.put( "organizationId",organization.getUuid() );
+        payload.put( "applicationId",applicationId);
+        payload.put( "collectionName","baconators");
+
+        UUID exportUUID = exportService.schedule( payload );
+        exportService.setS3Export( s3Export );
+
+        JobData jobData = new JobData();
+        jobData.setProperty( "jobName", "exportJob" );
+        jobData.setProperty( "exportInfo", payload );
+        jobData.setProperty( "exportId", exportUUID );
+
+        JobExecution jobExecution = mock( JobExecution.class );
+        when( jobExecution.getJobData() ).thenReturn( jobData );
+
+        exportService.doExport( jobExecution );
+
+        JSONParser parser = new JSONParser();
+
+        org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader(
f ) );
+
+        //only one entity should match the query.
+        assertEquals( 1 , a.size() );
+
+    }
+
+
+
+
+
     //@Ignore("file created won't be deleted when running tests")
     @Test
     public void testExportOneOrganization() throws Exception {
@@ -1503,11 +1568,11 @@ public class ManagementServiceIT {
         //intialize user object to be posted
         Map<String, Object> userProperties = null;
         Entity[] entity;
-        entity = new Entity[100];
+        entity = new Entity[5];
         //creates entities
 
         ApplicationInfo appMade = null;
-        for(int i = 0; i < 100; i++) {
+        for(int i = 0; i < 5; i++) {
             appMade = setup.getMgmtSvc().createApplication( organization.getUuid(), "superapp"+i);
 
             EntityManager customMaker = setup.getEmf().getEntityManager( appMade.getId()
);
@@ -1515,9 +1580,9 @@ public class ManagementServiceIT {
             //intialize user object to be posted
             Map<String, Object> entityLevelProperties = null;
             Entity[] entNotCopied;
-            entNotCopied = new Entity[20];
+            entNotCopied = new Entity[5];
             //creates entities
-            for ( int index = 0; index < 20; index++ ) {
+            for ( int index = 0; index < 5; index++ ) {
                 entityLevelProperties = new LinkedHashMap<String, Object>();
                 entityLevelProperties.put( "username", "bobso" + index );
                 entityLevelProperties.put( "email", "derp" + index + "@anuff.com" );
@@ -1710,7 +1775,7 @@ public class ManagementServiceIT {
         OrganizationInfo orgMade = null;
         ApplicationInfo appMade = null;
         for(int i = 0; i < 100; i++) {
-            orgMade =setup.getMgmtSvc().createOrganization( "minorboss"+i,adminUser,true
);
+            orgMade =setup.getMgmtSvc().createOrganization( "largerboss"+i,adminUser,true
);
             appMade = setup.getMgmtSvc().createApplication( orgMade.getUuid(), "superapp"+i);
 
             EntityManager customMaker = setup.getEmf().getEntityManager( appMade.getId()
);


Mime
View raw message