usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From g...@apache.org
Subject [22/50] [abbrv] incubator-usergrid git commit: Ignore users with no orgs or creds, log total orgs exported and better logging.
Date Fri, 24 Jul 2015 15:21:50 GMT
Ignore users with no orgs or creds, log total orgs exported and better logging.


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/125ffe98
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/125ffe98
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/125ffe98

Branch: refs/heads/USERGRID-869
Commit: 125ffe9893c617a81b26a9aaaf1f13460fb01ca6
Parents: 16de78d
Author: Dave Johnson <snoopdave@apache.org>
Authored: Fri Jul 17 11:28:07 2015 -0400
Committer: Dave Johnson <snoopdave@apache.org>
Committed: Fri Jul 17 11:28:07 2015 -0400

----------------------------------------------------------------------
 .../org/apache/usergrid/tools/ExportAdmins.java | 117 ++++++++++++-------
 .../org/apache/usergrid/tools/ImportAdmins.java |   2 +-
 2 files changed, 78 insertions(+), 41 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/125ffe98/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
index ae9c16b..9dce862 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
@@ -61,15 +61,25 @@ import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEM
  *    cassandra.lock.keyspace=My_Usergrid_Locks
  */
 public class ExportAdmins extends ExportingToolBase {
-
+    
     static final Logger logger = LoggerFactory.getLogger( ExportAdmins.class );
+    
     public static final String ADMIN_USERS_PREFIX = "admin-users";
     public static final String ADMIN_USER_METADATA_PREFIX = "admin-user-metadata";
+   
+    // map admin user UUID to list of organizations to which user belongs
+    private Map<UUID, List<Org>> userToOrgsMap = new HashMap<UUID, List<Org>>(50000);
+
+    private Map<String, UUID> orgNameToUUID = new HashMap<String, UUID>(50000);
+    
+    private Set<UUID> orgsWritten = new HashSet<UUID>(50000);
+    
+    private Set<UUID> duplicateOrgs = new HashSet<UUID>();
+    
     private static final String READ_THREAD_COUNT = "readThreads";
-    private Map<String, List<Org>> orgMap = new HashMap<String, List<Org>>(80000);
     private int readThreadCount;
-    
-    AtomicInteger count = new AtomicInteger( 0 );
+
+    AtomicInteger userCount = new AtomicInteger( 0 );
    
     
     /**
@@ -173,7 +183,7 @@ public class ExportAdmins extends ExportingToolBase {
         while ( !done ) {
             writeThread.join( 10000, 0 );
             done = !writeThread.isAlive();
-            logger.info( "Wrote {} users", count.get() );
+            logger.info( "Wrote {} users", userCount.get() );
         }
     }
     
@@ -211,10 +221,11 @@ public class ExportAdmins extends ExportingToolBase {
             organizations = em.searchCollection( em.getApplicationRef(), "groups", query
);
             for ( Entity organization : organizations.getEntities() ) {
                 execService.submit( new OrgMapWorker( organization ) );
+                count++;
             }
-            count++;
+             
             if ( count % 1000 == 0 ) {
-                logger.info("Processed {} orgs for org map", count);
+                logger.info("Queued {} org map workers", count);
             }
             query.setCursor( organizations.getCursor() );
         }
@@ -222,8 +233,10 @@ public class ExportAdmins extends ExportingToolBase {
 
         execService.shutdown();
         while ( !execService.awaitTermination( 10, TimeUnit.SECONDS ) ) {
-            logger.info("Processed {} orgs for map", orgMap.size() );
+            logger.info( "Processed {} orgs for map", userToOrgsMap.size() );
         }
+        
+        logger.info("Org map complete, counted {} organizations", count);
     }
 
 
@@ -239,17 +252,33 @@ public class ExportAdmins extends ExportingToolBase {
             try {
                 final String orgName = orgEntity.getProperty( "path" ).toString();
                 final UUID orgId = orgEntity.getUuid();
+                
                 for (UserInfo user : managementService.getAdminUsersForOrganization( orgEntity.getUuid()
)) {
                     try {
                         Entity admin = managementService.getAdminUserEntityByUuid( user.getUuid()
);
-                        List<Org> orgs = orgMap.get( admin.getProperty( "username"
) );
-                        if (orgs == null) {
-                            orgs = new ArrayList<Org>();
-                            orgMap.put( admin.getProperty( "username" ).toString().toLowerCase(),
orgs );
+                        Org org = new Org( orgId, orgName );
+
+                        synchronized (userToOrgsMap) {
+                            List<Org> userOrgs = userToOrgsMap.get( admin.getUuid()
);
+                            if (userOrgs == null) {
+                                userOrgs = new ArrayList<Org>();
+                                userToOrgsMap.put( admin.getUuid(), userOrgs );
+                            }
+                            userOrgs.add( org );
                         }
-                        orgs.add( new Org( orgId, orgName ) );
 
-                        //logger.debug("Added org {} for user {}", orgName, admin.getProperty(
"username" ));
+                        synchronized (orgNameToUUID) {
+                            UUID existingOrgId = orgNameToUUID.get( orgName );
+                            ;
+                            if (existingOrgId != null && !orgId.equals( existingOrgId
)) {
+                                if ( !duplicateOrgs.contains( orgId )) {
+                                    logger.info( "Org {}:{} is a duplicate", orgId, orgName
);
+                                    duplicateOrgs.add(orgId);
+                                }
+                            } else {
+                                orgNameToUUID.put( orgName, orgId );
+                            }
+                        }
 
                     } catch (Exception e) {
                         logger.warn( "Cannot get orgs for userId {}", user.getUuid() );
@@ -301,10 +330,33 @@ public class ExportAdmins extends ExportingToolBase {
                     AdminUserWriteTask task = new AdminUserWriteTask();
                     task.adminUser = entity;
 
-                    addDictionariesToTask(  task, entity );
+                    addDictionariesToTask( task, entity );
                     addOrganizationsToTask( task );
 
-                    writeQueue.add( task );
+                    String actionTaken = "Processed";
+
+                    if (task.orgNamesByUuid.isEmpty()
+                            || task.dictionariesByName.isEmpty()
+                            || task.dictionariesByName.get( "credentials" ).isEmpty()) {
+                        actionTaken = "Ignored";
+                        
+                    } else {
+                        writeQueue.add( task );
+                    }
+
+                    Map<String, Object> creds = (Map<String, Object>) (task.dictionariesByName.isEmpty()
? 
+                                                0 : task.dictionariesByName.get( "credentials"
));
+                    
+                    logger.error( "{} admin user {}:{}:{} has organizations={} dictionaries={}
credentials={}",
+                            new Object[]{
+                                    actionTaken,
+                                    task.adminUser.getProperty( "username" ),
+                                    task.adminUser.getProperty( "email" ),
+                                    task.adminUser.getUuid(),
+                                    task.orgNamesByUuid.size(),
+                                    task.dictionariesByName.size(),
+                                    creds == null ? 0 : creds.size()
+                            } ); 
 
                 } catch ( Exception e ) {
                     logger.error("Error reading data for user " + uuid, e );
@@ -327,20 +379,8 @@ public class ExportAdmins extends ExportingToolBase {
 
             Map<Object, Object> credentialsDictionary = em.getDictionaryAsMap( entity,
"credentials" );
 
-            if ( credentialsDictionary != null && !credentialsDictionary.isEmpty()
) {
+            if ( credentialsDictionary != null ) {
                 task.dictionariesByName.put( "credentials", credentialsDictionary );
-
-                if (credentialsDictionary.get( "password" ) == null) {
-                    logger.error( "User {}:{} has no password in credential dictionary",
-                        new Object[]{task.adminUser.getName(), task.adminUser.getUuid()}
);
-                }
-                if (credentialsDictionary.get( "secret" ) == null) {
-                    logger.error( "User {}:{} has no secret in credential dictionary",
-                        new Object[]{task.adminUser.getName(), task.adminUser.getUuid()}
);
-                }
-            } else {
-                logger.error( "User {}:{} has no or empty credentials dictionary",
-                    new Object[]{task.adminUser.getName(), task.adminUser.getUuid()} );
             }
         }
 
@@ -348,22 +388,17 @@ public class ExportAdmins extends ExportingToolBase {
 
             task.orgNamesByUuid = managementService.getOrganizationsForAdminUser( task.adminUser.getUuid()
);
 
-            List<Org> orgs = orgMap.get( task.adminUser.getProperty( "username" ).toString().toLowerCase()
);
+            List<Org> orgs = userToOrgsMap.get( task.adminUser.getProperty( "username"
).toString().toLowerCase() );
             
             if ( orgs != null && task.orgNamesByUuid.size() < orgs.size() ) {
+                
+                // list of orgs from getOrganizationsForAdminUser() is less than expected,
use userToOrgsMap
                 BiMap<UUID, String> bimap = HashBiMap.create();
                 for (Org org : orgs) {
                     bimap.put( org.orgId, org.orgName );
                 }
                 task.orgNamesByUuid = bimap;
             }
-            
-            if ( task.orgNamesByUuid.isEmpty() ) {
-                logger.error("{}:{}:{} has no orgs", new Object[] {
-                        task.adminUser.getProperty("username"), 
-                        task.adminUser.getProperty("email"), 
-                        task.adminUser.getUuid() } );
-            }
         }
     }
 
@@ -425,7 +460,7 @@ public class ExportAdmins extends ExportingToolBase {
                         task.adminUser.getProperty("email"),
                         task.adminUser.getUuid() } );
 
-                    count.addAndGet( 1 );
+                    userCount.addAndGet( 1 );
 
                 } catch (InterruptedException e) {
                     throw new Exception("Interrupted", e);
@@ -438,7 +473,7 @@ public class ExportAdmins extends ExportingToolBase {
             usersFile.writeEndArray();
             usersFile.close();
 
-            logger.info( "Exported TOTAL {} admin users", count );
+            logger.info( "Exported TOTAL {} admin users and {} organizations", userCount.get(),
orgsWritten.size() );
         }
 
 
@@ -490,7 +525,9 @@ public class ExportAdmins extends ExportingToolBase {
 
                 jg.writeEndObject();
                 
-                logger.debug( "Exported organization {}:{}", uuid, orgs.get( uuid ) );
+                synchronized (orgsWritten) {
+                    orgsWritten.add( uuid );
+                }
             }
 
             jg.writeEndArray();

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/125ffe98/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java
index c6aada7..7e08a4c 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java
@@ -636,7 +636,7 @@ public class ImportAdmins extends ToolBase {
                     long duration = stopTime - startTime;
                     durationSum += duration;
 
-                    //logger.debug( "Audited {}th admin", count );
+                    //logger.debug( "Audited {}th admin", userCount );
                     
                     if ( count % 100 == 0 ) {
                         logger.info( "Audited {}. Average Audit Rate: {}(ms)", count, durationSum
/ count );


Mime
View raw message