usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mru...@apache.org
Subject [1/7] usergrid git commit: Duplicate org repair tools, and implementation for managementService.removeOrganizationApplication().
Date Thu, 17 Dec 2015 22:14:20 GMT
Repository: usergrid
Updated Branches:
  refs/heads/1.x ccf4e4231 -> 69eeb9187


Duplicate org repair tools, and implementation for managementService.removeOrganizationApplication().


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/674532f4
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/674532f4
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/674532f4

Branch: refs/heads/1.x
Commit: 674532f4d8011afa5b44178467af2fefa3319dea
Parents: 98658fd
Author: Dave Johnson <snoopdave@apache.org>
Authored: Fri Dec 11 12:21:19 2015 -0500
Committer: Dave Johnson <snoopdave@apache.org>
Committed: Fri Dec 11 12:21:19 2015 -0500

----------------------------------------------------------------------
 .../usergrid/management/ManagementService.java  |   4 +-
 .../cassandra/ManagementServiceImpl.java        |  28 +-
 .../usergrid/tools/DuplicateOrgInterface.java   | 110 +++++
 .../usergrid/tools/DuplicateOrgRepair.java      | 494 +++++++++++++++++++
 .../usergrid/tools/DuplicateOrgRepairTest.java  | 400 +++++++++++++++
 5 files changed, 1032 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/674532f4/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
b/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
index b5537ec..4b0cbfd 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
@@ -68,7 +68,9 @@ public interface ManagementService {
 
     public UserInfo createAdminFromPrexistingPassword( User user, CredentialsInfo ci ) throws
Exception;
 
-    public ApplicationInfo createApplication( UUID organizationId, String applicationName
) throws Exception;
+    void removeAdminUserFromOrganization(UUID userId, UUID organizationId, boolean force)
throws Exception;
+
+    public ApplicationInfo createApplication(UUID organizationId, String applicationName
) throws Exception;
 
     public ApplicationInfo createApplication( UUID organizationId, String applicationName,
                                               Map<String, Object> properties ) throws
Exception;

http://git-wip-us.apache.org/repos/asf/usergrid/blob/674532f4/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
index cb19733..3cdc806 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
@@ -28,6 +28,7 @@ import java.util.Properties;
 import java.util.Set;
 import java.util.UUID;
 
+import org.apache.usergrid.persistence.cassandra.ConnectionRefImpl;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -1594,6 +1595,12 @@ public class ManagementServiceImpl implements ManagementService {
 
     @Override
     public void removeAdminUserFromOrganization( UUID userId, UUID organizationId ) throws
Exception {
+        removeAdminUserFromOrganization( userId, organizationId, false );     
+    }
+
+    
+    @Override
+    public void removeAdminUserFromOrganization( UUID userId, UUID organizationId, boolean
force ) throws Exception {
 
         if ( ( userId == null ) || ( organizationId == null ) ) {
             return;
@@ -1602,8 +1609,10 @@ public class ManagementServiceImpl implements ManagementService {
         EntityManager em = emf.getEntityManager( MANAGEMENT_APPLICATION_ID );
 
         try {
-            if ( em.getCollection( new SimpleEntityRef( Group.ENTITY_TYPE, organizationId
), "users", null, 2,
-                    Level.IDS, false ).size() <= 1 ) {
+            int size = em.getCollection( new SimpleEntityRef( Group.ENTITY_TYPE, organizationId
), 
+                    "users", null, 2, Level.IDS, false ).size();
+            
+            if ( !force && size <= 1 ) { 
                 throw new Exception();
             }
         }
@@ -1751,10 +1760,23 @@ public class ManagementServiceImpl implements ManagementService {
     }
 
 
+    /**
+     * Remove application from an organization. 
+     */
     @Override
     public void removeOrganizationApplication( UUID organizationId, UUID applicationId )
throws Exception {
-        // TODO Auto-generated method stub
+        if ( ( organizationId == null ) || ( applicationId == null ) ) {
+            return;
+        }
 
+        EntityManager em = emf.getEntityManager( MANAGEMENT_APPLICATION_ID );
+        em.deleteConnection( new ConnectionRefImpl(  
+            "group",          // String connectingEntityType 
+            organizationId,   // UUID connectingEntityId 
+            "owns",           // String connectionType
+            APPLICATION_INFO, // String connectedEntityType
+            applicationId     // UUID connectedEntityId
+        ) );
     }
 
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/674532f4/stack/tools/src/main/java/org/apache/usergrid/tools/DuplicateOrgInterface.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/DuplicateOrgInterface.java
b/stack/tools/src/main/java/org/apache/usergrid/tools/DuplicateOrgInterface.java
new file mode 100644
index 0000000..596d55e
--- /dev/null
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/DuplicateOrgInterface.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.tools;
+
+import rx.Observable;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+
+/**
+ * Abstraction to make duplicate org repair testable.
+ */
+interface DuplicateOrgInterface {
+    
+    Observable<Org> getOrgs() throws Exception;
+    
+    Observable<OrgUser> getUsers() throws Exception;
+
+    Set<Org> getUsersOrgs(OrgUser user) throws Exception;
+
+    Set<OrgUser> getOrgUsers(Org org ) throws Exception;
+    
+    void removeOrg(Org keeper, Org duplicate) throws Exception;
+
+    void removeUserFromOrg( OrgUser user, Org org ) throws Exception;
+    
+    void addUserToOrg( OrgUser user, Org org ) throws Exception;
+
+    Set<UUID> getOrgApps(Org org) throws Exception;
+    
+    void removeAppFromOrg( UUID appId, Org org ) throws Exception;
+    
+    void addAppToOrg( UUID appId, Org org ) throws Exception;
+  
+    void logDuplicates(Map<String, Set<Org>> duplicatesByName);
+    
+    class Org implements Comparable<Org> {
+        private UUID id;
+        private String name;
+        private long created;
+        public Object sourceValue;
+        
+        public Org( UUID id, String name, long created ) {
+            this.id = id;
+            this.name = name;
+            this.created = created;
+        }
+        
+        @Override
+        public boolean equals( Object o ) {
+            if ( o instanceof Org ) {
+                Org other = (Org)o;
+                return getId().equals( other.getId() );
+            }
+            return false;
+        }
+
+        @Override
+        public int compareTo(Org o) {
+            return Long.compare( this.created, o.created );
+        }
+
+        public UUID getId() {
+            return id;
+        }
+
+        public String getName() {
+            return name;
+        }
+
+        public long getCreated() {
+            return created;
+        }
+    }
+    
+    class OrgUser {
+        private UUID id;
+        private String name;
+        public Object sourceValue;
+        
+        public OrgUser( UUID id, String name ) {
+            this.id = id;
+            this.name = name;
+        }
+
+        public UUID getId() {
+            return id;
+        }
+
+        public String getName() {
+            return name;
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/usergrid/blob/674532f4/stack/tools/src/main/java/org/apache/usergrid/tools/DuplicateOrgRepair.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/DuplicateOrgRepair.java b/stack/tools/src/main/java/org/apache/usergrid/tools/DuplicateOrgRepair.java
new file mode 100644
index 0000000..bc4acb5
--- /dev/null
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/DuplicateOrgRepair.java
@@ -0,0 +1,494 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.usergrid.tools;
+
+import com.google.common.collect.BiMap;
+import com.google.common.collect.Sets;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.usergrid.management.OrganizationInfo;
+import org.apache.usergrid.management.UserInfo;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.cassandra.CassandraService;
+import org.apache.usergrid.persistence.entities.Group;
+import org.apache.usergrid.utils.StringUtils;
+import rx.Observable;
+import rx.Scheduler;
+import rx.Subscriber;
+import rx.functions.Action1;
+import rx.schedulers.Schedulers;
+
+import java.util.*;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import static org.apache.usergrid.tools.DuplicateOrgInterface.Org;
+import static org.apache.usergrid.tools.DuplicateOrgInterface.OrgUser;
+
+/**
+ * Find duplicate orgs, delete all but oldest of each and assign users to it.
+ */
+public class DuplicateOrgRepair extends ToolBase {
+
+    DuplicateOrgInterface   manager = new Manager();
+    
+    Map<String, Set<Org>>   orgsByName = new HashMap<String, Set<Org>>();
+    
+    Map<UUID, Org>          orgsById = new HashMap<UUID, Org>();
+    
+    Map<OrgUser, Set<Org>>  orgsByUser = new HashMap<OrgUser, Set<Org>>();
+    
+    Map<String, Set<Org>>   duplicatesByName = new HashMap<String, Set<Org>>();
+    
+    static final String     THREADS_ARG_NAME = "threads"; 
+    
+    static int              threadCount = 5;
+    
+    static final String     DRYRUN_ARG_NAME = "dryrun";
+    
+    static boolean          dryRun = false;
+
+    
+    @Override
+    @SuppressWarnings("static-access")
+    public Options createOptions() {
+
+        Options options = super.createOptions();
+
+        Option dryRunOption = OptionBuilder.hasArg()
+            .withType(Boolean.TRUE)
+            .withDescription( "-" + DRYRUN_ARG_NAME + "true to print what tool would do and
do not alter data.")
+            .create( DRYRUN_ARG_NAME );
+        options.addOption( dryRunOption );
+
+        Option writeThreadsOption = OptionBuilder.hasArg()
+            .withType(0)
+            .withDescription( "Write Threads -" + THREADS_ARG_NAME )
+            .create(THREADS_ARG_NAME);
+        options.addOption( writeThreadsOption );
+
+        return options;
+    }
+
+    @Override
+    public void runTool(CommandLine line) throws Exception {
+
+        logger.info( "DuplicateOrgRepair tool starting up..." );
+
+        startSpring();
+        setVerbose( line );
+
+        dryRun = Boolean.parseBoolean( line.getOptionValue( DRYRUN_ARG_NAME ));
+
+        if (StringUtils.isNotEmpty( line.getOptionValue( THREADS_ARG_NAME ) )) {
+            try {
+                threadCount = Integer.parseInt( line.getOptionValue( THREADS_ARG_NAME ) );
+            } catch (NumberFormatException nfe) {
+                logger.error( "-" + THREADS_ARG_NAME + " must be specified as an integer.
Aborting..." );
+                return;
+            }
+        }
+
+        buildOrgMaps();
+
+        augmentUserOrgsMap();
+        
+        mergeDuplicateOrgs();
+
+        removeDuplicateOrgs();
+
+        logger.info( "DuplicateOrgRepair work is done." );
+    }
+
+    
+    /** 
+     * build map of orgs by name, orgs by id, orgs by user and duplicate orgs by name 
+     */
+    private void buildOrgMaps() throws Exception {
+        
+        manager.getOrgs().doOnNext( new Action1<Org>() {
+            @Override
+            public void call(Org org) {
+               
+                // orgs by name and duplicate orgs by name maps
+                
+                Set<Org> orgs = orgsByName.get( org.getName() );
+                if (orgs == null) {
+                    orgs = new HashSet<Org>();
+                    orgsByName.put( org.getName(), orgs );
+                } else {
+                    duplicatesByName.put( org.getName(), orgs );
+                }
+                orgs.add( org );
+                
+                orgsById.put( org.getId(), org );
+
+                // orgs by user map, created via org -> user connections
+                
+                try {
+                    Set<OrgUser> orgUsers = manager.getOrgUsers( org );
+                    for ( OrgUser user : orgUsers ) {
+                        Set<Org> usersOrgs = orgsByUser.get( user );
+                        if (usersOrgs == null) {
+                            usersOrgs = new HashSet<Org>();
+                            orgsByUser.put( user, usersOrgs );
+                        }
+                        usersOrgs.add( org );
+                    }
+                } catch (Exception e) {
+                    logger.error("Error getting users for org {}:{}", org.getName(), org.getId());
+                }
+
+            }
+        } ).toBlocking().lastOrDefault( null );
+    }
+
+    
+    /**
+     * augment user orgs map via user -> org connections
+     */
+    private void augmentUserOrgsMap() throws Exception {
+        
+        ExecutorService writeThreadPoolExecutor = Executors.newFixedThreadPool( threadCount
);
+        Scheduler scheduler = Schedulers.from( writeThreadPoolExecutor );
+
+        manager.getUsers().doOnNext( new Action1<OrgUser>() {
+            @Override
+            public void call(OrgUser user) {
+                try {
+                    Set<Org> connectedToOrgs = manager.getUsersOrgs(user);
+                    Set<Org> usersOrgs = orgsByUser.get(user);
+                    for ( Org org : connectedToOrgs ) {
+                        if (!usersOrgs.contains(org)) {
+                            usersOrgs.add(org);
+                        }
+                    }
+
+                } catch (Exception e) {
+                    logger.error("Error getting orgs for user {}:{}", user.getName(), user.getId());
+                }
+            }
+        } ).subscribeOn( scheduler ).toBlocking().lastOrDefault( null );
+    }
+
+
+    /**
+     * For each duplicate name, pick best org and merge apps and users into it 
+     */
+    private void mergeDuplicateOrgs() throws Exception {
+        
+        for ( String dupName : duplicatesByName.keySet() ) {
+            Set<Org> duplicateOrgs = duplicatesByName.get(dupName);
+            Org bestOrg = selectBest( duplicateOrgs );
+            
+            for ( Org org : duplicateOrgs ) {
+                
+                if ( !org.equals( bestOrg )) {
+                    
+                    Set<OrgUser> orgUsers = new HashSet<OrgUser>( manager.getOrgUsers(
org ));
+                    
+                    for ( OrgUser user : orgUsers ) {
+                        if (dryRun) {
+                            Object[] args = new Object[] { 
+                                    user.getName(), user.getId(), bestOrg.getName(), bestOrg.getId()};
+                            logger.info( "Would add user {}:{} to org {}:{}", args);
+                            args = new Object[] { 
+                                    user.getName(), user.getId(), org.getName(), org.getId()};
+                            logger.info( "Would remove user {}:{}  org {}:{}", args);
+                            continue;
+                        } 
+                        manager.addUserToOrg( user, bestOrg );
+                        manager.removeUserFromOrg( user, org );
+                    }
+                    
+                    Set<UUID> orgApps = new HashSet<UUID>( manager.getOrgApps(
org ));
+                            
+                    for ( UUID appId : orgApps ) {
+                        if (dryRun) {
+                            Object[] args = new Object[] { 
+                                    appId, bestOrg.getName(), bestOrg.getId()};
+                            logger.info( "Would add app {} to org {}:{}", args);
+                            args = new Object[] { 
+                                    appId, org.getName(), org.getId()};
+                            logger.info( "Would remove app {} org {}:{}", args);
+                            continue;
+                        } 
+                        manager.addAppToOrg( appId, bestOrg );
+                        manager.removeAppFromOrg( appId, org );
+                    }
+                }
+            }
+        }
+    }
+
+
+    /**
+     * remove/rename duplicate orgs so they no longer impact operation of system 
+     */
+    private void removeDuplicateOrgs() throws Exception {
+        for ( String dupName : duplicatesByName.keySet() ) {
+            Set<Org> orgs = duplicatesByName.get( dupName );
+            Org best = selectBest( orgs );
+            for ( Org candidate : orgs ) {
+                if ( !candidate.equals(best) ) {
+                    if ( dryRun ) {
+                        logger.info("Would rename/remove org {}:{}", 
+                           new Object[] { candidate.getName(), candidate.getId() });
+                    } else {
+                        manager.removeOrg( best, candidate );
+                    }
+                }
+            }
+        }
+    }
+
+
+    /**
+     * select best org from a set of duplicates by picking the oldest org
+     */
+    public Org selectBest(Set<Org> orgs) throws Exception {
+        Org oldest = null;
+        for ( Org org :orgs ) {
+            if ( oldest == null || org.compareTo( oldest ) < 0 ) {
+                oldest = org;
+            }
+        }
+        return oldest;
+    }
+    
+
+    class Manager implements DuplicateOrgInterface {
+
+        private boolean dryRun = true;
+
+        @Override
+        public Observable<Org> getOrgs() throws Exception {
+
+            return Observable.create( new Observable.OnSubscribe<Org>() {
+
+                @Override
+                public void call(Subscriber<? super Org> subscriber) {
+                    subscriber.onStart();
+                    try {
+                        Query query = new Query();
+                        query.setLimit( MAX_ENTITY_FETCH );
+                        query.setResultsLevel( Results.Level.ALL_PROPERTIES );
+                        EntityManager em = emf.getEntityManager( CassandraService.MANAGEMENT_APPLICATION_ID
);
+                        Results results = em.searchCollection( em.getApplicationRef(), "groups",
query );
+
+                        while (results.size() > 0) {
+                            for (Entity orgEntity : results.getList()) {
+                                
+                                Org org = new Org(
+                                    orgEntity.getUuid(), 
+                                    orgEntity.getProperty( "path" )+"", 
+                                    orgEntity.getCreated() );
+                                org.sourceValue = orgEntity;
+                                
+                                subscriber.onNext( org );
+
+                                // logger.info( "org: {}, \"{}\", {}", new Object[]{
+                                //     orgEntity.getProperty( "path" ),
+                                //     orgEntity.getUuid(),
+                                //     orgEntity.getCreated()} );
+                            }
+                            if (results.getCursor() == null) {
+                                break;
+                            }
+                            query.setCursor( results.getCursor() );
+                            results = em.searchCollection( em.getApplicationRef(), "groups",
query );
+                        }
+
+                    } catch (Exception e) {
+                        subscriber.onError( e );
+                    }
+                    subscriber.onCompleted();
+                }
+            } );
+        }
+
+        @Override
+        public Observable<OrgUser> getUsers() throws Exception {
+
+            return Observable.create( new Observable.OnSubscribe<OrgUser>() {
+
+                @Override
+                public void call(Subscriber<? super OrgUser> subscriber) {
+                    subscriber.onStart();
+                    try {
+                        Query query = new Query();
+                        query.setLimit( MAX_ENTITY_FETCH );
+                        query.setResultsLevel( Results.Level.ALL_PROPERTIES );
+                        EntityManager em = emf.getEntityManager( CassandraService.MANAGEMENT_APPLICATION_ID
);
+                        Results results = em.searchCollection( em.getApplicationRef(), "users",
query );
+
+                        while (results.size() > 0) {
+                            for (Entity entity : results.getList()) {
+
+                                OrgUser orgUser = new OrgUser( 
+                                    entity.getUuid(), 
+                                    entity.getProperty( "username" ) + "" );
+                                orgUser.sourceValue = entity;
+
+                                subscriber.onNext( orgUser );
+                                
+                                // logger.info( "org: {}, \"{}\", {}", new Object[]{
+                                //     entity.getProperty( "path" ),
+                                //     entity.getUuid(),
+                                //     entity.getCreated()} );
+                            }
+                            if (results.getCursor() == null) {
+                                break;
+                            }
+                            query.setCursor( results.getCursor() );
+                            results = em.searchCollection( em.getApplicationRef(), "users",
query );
+                        }
+
+                    } catch (Exception e) {
+                        subscriber.onError( e );
+                    }
+                    subscriber.onCompleted();
+                }
+            } );
+        }
+
+        @Override
+        public Set<Org> getUsersOrgs(OrgUser user) throws Exception {
+           
+            Set<Org> ret = new HashSet<Org>();
+            
+            Map<String, Object> orgData = managementService.getAdminUserOrganizationData(
user.getId() );
+           
+            Map<String, Object> orgs = (Map<String, Object>)orgData.get("organizations");
+            for ( String orgName : orgs.keySet() ) {
+
+                Map<String, Object> orgMap = (Map<String, Object>)orgs.get( orgName
);
+                Group group = managementService.getOrganizationProps( 
+                        UUID.fromString( orgMap.get( "uuid" ).toString() ) );
+
+                Org org = new Org(
+                    group.getUuid(),
+                    group.getPath(), 
+                    group.getCreated()
+                );
+                ret.add(org);   
+            }
+            
+            return ret;
+        }
+
+        
+        @Override
+        public void removeOrg(Org keeper, Org duplicate) throws Exception {
+
+            // we don't have a remove org API so rename org so that it is no longer a duplicate
+            
+            OrganizationInfo orgInfo = managementService.getOrganizationByUuid( duplicate.getId()
);
+            orgInfo.setName( "dup_" + keeper.getId() + "_" + RandomStringUtils.randomAlphanumeric(10)
);
+            managementService.updateOrganization( orgInfo );
+        }
+        
+
+        @Override
+        public Set<OrgUser> getOrgUsers(Org org) throws Exception {
+            
+            Set<OrgUser> ret = new HashSet<OrgUser>();
+            
+            List<UserInfo> userInfos = managementService.getAdminUsersForOrganization(
org.getId() );
+            
+            for ( UserInfo userInfo : userInfos ) {
+                OrgUser orgUser = new OrgUser( userInfo.getUuid(), userInfo.getUsername()
);
+                ret.add(orgUser);
+            }
+            
+            return ret;
+        }
+
+        
+        @Override
+        public void removeUserFromOrg(OrgUser user, Org org) throws Exception {
+            // forcefully remove admin user from org
+            managementService.removeAdminUserFromOrganization( user.getId(), org.getId(),
true );
+        }
+
+        
+        @Override
+        public void addUserToOrg(OrgUser user, Org org) throws Exception {
+            UserInfo userInfo = managementService.getAdminUserByUsername( user.getName()
);
+            OrganizationInfo orgInfo = managementService.getOrganizationByUuid( org.getId()
);
+            managementService.addAdminUserToOrganization( userInfo, orgInfo, false );
+        }
+
+        
+        @Override
+        public Set<UUID> getOrgApps(Org org) throws Exception {
+            BiMap<UUID, String> apps = managementService.getApplicationsForOrganization(
org.getId() );
+            return apps.keySet();
+        }
+
+        
+        @Override
+        public void removeAppFromOrg(UUID appId, Org org) throws Exception {
+            managementService.removeOrganizationApplication( org.getId(), appId ); 
+        }
+
+        
+        @Override
+        public void addAppToOrg(UUID appId, Org org) throws Exception {
+            managementService.addApplicationToOrganization( org.getId(), appId );
+        }
+
+        
+        @Override
+        public void logDuplicates(Map<String, Set<Org>> duplicatesByName) {
+
+            for ( String orgName : duplicatesByName.keySet() ) {
+                Set<Org> orgs = duplicatesByName.get(orgName);
+                for ( Org org : orgs ) {
+                    Entity orgEntity = (Entity)org.sourceValue;
+
+                    StringBuilder sb = new StringBuilder();
+                    sb.append(orgEntity.toString()).append(", ");
+                    
+                    try {
+                        BiMap<UUID, String> apps = 
+                            managementService.getApplicationsForOrganization( orgEntity.getUuid()
);
+                        String sep = "";
+                        for ( UUID uuid : apps.keySet() ) {
+                            String appName = apps.get(uuid);
+                            sb.append(appName).append(":").append(uuid).append(sep);
+                            sep = ", ";
+                        }
+                        
+                    } catch (Exception e) {
+                        logger.error("Error getting applications for org {}:{}", org.getName(),
org.getId() );
+                    }
+                    
+                    logger.info(sb.toString());
+                }
+            }
+        }
+        
+    }
+}

http://git-wip-us.apache.org/repos/asf/usergrid/blob/674532f4/stack/tools/src/test/java/org/apache/usergrid/tools/DuplicateOrgRepairTest.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/test/java/org/apache/usergrid/tools/DuplicateOrgRepairTest.java
b/stack/tools/src/test/java/org/apache/usergrid/tools/DuplicateOrgRepairTest.java
new file mode 100644
index 0000000..ac95748
--- /dev/null
+++ b/stack/tools/src/test/java/org/apache/usergrid/tools/DuplicateOrgRepairTest.java
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.tools;
+
+
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.usergrid.ServiceITSetup;
+import org.apache.usergrid.ServiceITSetupImpl;
+import org.apache.usergrid.ServiceITSuite;
+import org.apache.usergrid.management.ApplicationInfo;
+import org.apache.usergrid.management.OrganizationOwnerInfo;
+import org.junit.ClassRule;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import rx.Observable;
+import rx.functions.Action1;
+
+import java.util.*;
+
+import static junit.framework.Assert.assertTrue;
+import static org.apache.usergrid.tools.DuplicateOrgInterface.Org;
+import static org.apache.usergrid.tools.DuplicateOrgInterface.OrgUser;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+
+/**
+ * Test duplicate org repair.
+ */
+public class DuplicateOrgRepairTest {
+    static final Logger logger = LoggerFactory.getLogger( DuplicateOrgRepairTest.class );
+    
+    @ClassRule
+    public static ServiceITSetup setup = new ServiceITSetupImpl( ServiceITSuite.cassandraResource
);
+
+    
+    /**
+     * Test tool logic with mock manager that returns duplicates
+     */
+    @org.junit.Test
+    public void testMockWithDups() throws Exception {
+
+        int numOrgs = 10; // create 10 orgs and a dup for each
+
+        final DuplicateOrgRepair dor = new DuplicateOrgRepair();
+        dor.manager = new Manager( numOrgs );
+
+        assertEquals( "must start with dups", 
+                2 * numOrgs, (long)dor.manager.getOrgs().count().toBlocking().single());
+
+        dor.startTool( new String[] {}, false ); // false means do not call System.exit()
+
+        assertEquals( "must remove dups", 
+                numOrgs, (long)dor.manager.getOrgs().count().toBlocking().single());
+       
+        dor.manager.getOrgs().doOnNext( new Action1<Org>() {
+            @Override
+            public void call(Org org) {
+                try {
+                    assertEquals("remaining orgs should have right number of users",
+                            3, dor.manager.getOrgUsers(org).size());
+
+                    assertEquals("remaining orgs should have right number of apps", 
+                            3, dor.manager.getOrgApps(org).size());
+                    
+                } catch (Exception e) {
+                    logger.error("Error counting apps or users: " + e.getMessage(), e);
+                    fail("Error counting apps or users");
+                }
+            }
+        }).toBlocking().lastOrDefault( null );
+    }
+    
+    
+    @org.junit.Test
+    public void testMockWithDupsDryRun() throws Exception {
+
+        int numOrgs = 10; // create 10 orgs and a dup for each
+
+        DuplicateOrgRepair dor = new DuplicateOrgRepair();
+        dor.manager = new Manager( numOrgs );
+
+        assertEquals( "must start with dups", 
+                2 * numOrgs, (long)dor.manager.getOrgs().count().toBlocking().single());
+
+        dor.startTool( new String[] { "-dryrun", "true" }, false ); // false means do not
call System.exit()
+
+        assertEquals( "must detect right number of dups", 
+                numOrgs, dor.duplicatesByName.keySet().size() );
+        
+        assertEquals( "dryrun must not remove dups", 
+                2 * numOrgs, (long)dor.manager.getOrgs().count().toBlocking().single());
+    }
+
+
+    /**
+     * Smoke test: does "real" manager run without throwing exceptions?
+     */
+    @org.junit.Test
+    public void testManagerNoDups() throws Exception {
+
+        // create two orgs each with owning user
+
+        final String random1 = RandomStringUtils.randomAlphanumeric( 10 );
+        final OrganizationOwnerInfo orgOwnerInfo1 = setup.getMgmtSvc().createOwnerAndOrganization(
+                "org_" + random1, "user_" + random1, "user_" + random1,
+                "user_" + random1 + "@example.com", "password" );
+
+        final String random2 = RandomStringUtils.randomAlphanumeric( 10 );
+        final OrganizationOwnerInfo orgOwnerInfo2 = setup.getMgmtSvc().createOwnerAndOrganization(
+                "org_" + random2, "user_" + random2, "user_" + random2,
+                "user_" + random2 + "@example.com", "password" );
+
+        // Add user1 to org2
+
+        setup.getMgmtSvc().addAdminUserToOrganization(
+                orgOwnerInfo1.getOwner(), orgOwnerInfo2.getOrganization(), false );
+        
+        DuplicateOrgRepair dor = new DuplicateOrgRepair();
+        
+        dor.startTool( new String[] {}, false );  // false means do not call System.exit()
+        
+        dor.startTool( new String[] { "dryrun", "true" }, false ); // false means do not
call System.exit()
+
+        assertTrue(true); // we're happy if we get to this point
+    }
+
+    
+    @org.junit.Test
+    public void testManagerAddUserToOrg() throws Exception {
+
+        // create two orgs each with owning user
+
+        final String random1 = RandomStringUtils.randomAlphanumeric( 10 );
+        final OrganizationOwnerInfo orgOwnerInfo1 = setup.getMgmtSvc().createOwnerAndOrganization(
+                "org_" + random1, "user_" + random1, "user_" + random1,
+                "user_" + random1 + "@example.com", "password" );
+
+        final String random2 = RandomStringUtils.randomAlphanumeric( 10 );
+        final OrganizationOwnerInfo orgOwnerInfo2 = setup.getMgmtSvc().createOwnerAndOrganization(
+                "org_" + random2, "user_" + random2, "user_" + random2,
+                "user_" + random2 + "@example.com", "password" );
+
+        DuplicateOrgRepair dor = new DuplicateOrgRepair();
+
+        // start the tool so thaht Spring, Cassandra, etc/ gets initialized
+        dor.startTool( new String[] { "-dryrun", "true" }, false ); // false means do not
call System.exit()
+        
+        Org org1 = new Org(
+            orgOwnerInfo1.getOrganization().getUuid(), orgOwnerInfo1.getOrganization().getName(),
0L);
+
+        Org org2 = new Org(
+            orgOwnerInfo2.getOrganization().getUuid(), orgOwnerInfo2.getOrganization().getName(),
0L);
+
+        OrgUser user1 = new OrgUser(
+                orgOwnerInfo1.getOwner().getUuid(), orgOwnerInfo1.getOwner().getUsername());
+
+        OrgUser user2 = new OrgUser(
+                orgOwnerInfo2.getOwner().getUuid(), orgOwnerInfo2.getOwner().getUsername());
+
+        assertEquals( 1, dor.manager.getUsersOrgs( user1 ).size());
+        assertEquals( 1, dor.manager.getOrgUsers( org1 ).size());
+        assertEquals( 1, dor.manager.getOrgUsers( org2 ).size());
+
+        dor.manager.addUserToOrg( user1, org2 );
+
+        assertEquals( 2, dor.manager.getUsersOrgs( user1 ).size());
+        assertEquals( 1, dor.manager.getOrgUsers( org1 ).size() );
+        assertEquals( 2, dor.manager.getOrgUsers( org2 ).size() );
+
+        dor.manager.removeUserFromOrg( user1, org2 );
+        
+        assertEquals( 1, dor.manager.getUsersOrgs( user1 ).size());
+        assertEquals( 1, dor.manager.getOrgUsers( org1 ).size());
+        assertEquals( 1, dor.manager.getOrgUsers( org2 ).size() );
+    }
+
+
+
+    @org.junit.Test
+    public void testManagerAddAppToOrg() throws Exception {
+
+        // create two orgs each with owning user
+
+        final String random1 = RandomStringUtils.randomAlphanumeric( 10 );
+        final OrganizationOwnerInfo orgOwnerInfo1 = setup.getMgmtSvc().createOwnerAndOrganization(
+            "org_" + random1, "user_" + random1, "user_" + random1,
+            "user_" + random1 + "@example.com", "password" );
+
+        final String random2 = RandomStringUtils.randomAlphanumeric( 10 );
+        final OrganizationOwnerInfo orgOwnerInfo2 = setup.getMgmtSvc().createOwnerAndOrganization(
+            "org_" + random2, "user_" + random2, "user_" + random2,
+            "user_" + random2 + "@example.com", "password" );
+
+        ApplicationInfo app11 = setup.getMgmtSvc().createApplication(
+                orgOwnerInfo1.getOrganization().getUuid(), "app_" + RandomStringUtils.randomAlphanumeric(
10 ) );
+
+        ApplicationInfo app12= setup.getMgmtSvc().createApplication(
+            orgOwnerInfo1.getOrganization().getUuid(), "app_" + RandomStringUtils.randomAlphanumeric(
10 ));
+
+        ApplicationInfo app21 = setup.getMgmtSvc().createApplication(
+            orgOwnerInfo2.getOrganization().getUuid(), "app_" + RandomStringUtils.randomAlphanumeric(
10 ));
+
+        DuplicateOrgRepair dor = new DuplicateOrgRepair();
+
+        // start the tool so that Spring, Cassandra, etc/ gets initialized
+        dor.startTool( new String[] { "-dryrun", "true" }, false ); // false means do not
call System.exit()
+
+        Org org1 = new Org(
+            orgOwnerInfo1.getOrganization().getUuid(), orgOwnerInfo1.getOrganization().getName(),
0L);
+
+        Org org2 = new Org(
+            orgOwnerInfo2.getOrganization().getUuid(), orgOwnerInfo2.getOrganization().getName(),
0L);
+
+        assertEquals( 2, dor.manager.getOrgApps( org1 ).size() );
+        assertEquals( 1, dor.manager.getOrgApps( org2 ).size() );
+
+        dor.manager.removeAppFromOrg( app12.getId(), org1 );
+
+        assertEquals( 1, dor.manager.getOrgApps( org1 ).size() );
+        assertEquals( 1, dor.manager.getOrgApps( org2 ).size() );
+
+        dor.manager.addAppToOrg( app12.getId(), org2 );
+        
+        assertEquals( 1, dor.manager.getOrgApps( org1 ).size() );
+        assertEquals( 2, dor.manager.getOrgApps( org2 ).size() );
+    }
+    
+    
+    /**
+     * Mock manager implementation for testing.
+     */
+    class Manager implements DuplicateOrgInterface {
+
+        Set<Org> orgs;
+
+        Set<OrgUser> orgUsers;
+
+        Map<Org, Set<OrgUser>> usersByOrg = new HashMap<Org, Set<OrgUser>>();
+
+        Map<OrgUser, Set<Org>> orgsByUser = new HashMap<OrgUser, Set<Org>>();
+
+        Map<Org, Set<UUID>> appsByOrg = new HashMap<Org, Set<UUID>>();
+
+
+        /**
+         * Populate manager with orgs and users.
+         * Will create a number of orgs and a duplicate for each.
+         *
+         * @param numOrgs One half of the number of orgs to create.
+         */
+        public Manager(int numOrgs) {
+
+            for (int i = 0; i < numOrgs; i++) {
+
+                // each org name is duplicated once another org created 20 ms apart
+
+                Org org1 = new Org( UUID.randomUUID(), "org_" + i, System.currentTimeMillis()
);
+                try {
+                    Thread.sleep( 100 );
+                } catch (InterruptedException intentionallyIgnored) {
+                }
+                Org org2 = new Org( UUID.randomUUID(), "org_" + i, System.currentTimeMillis()
);
+
+                OrgUser usera = new OrgUser( UUID.randomUUID(), "user_" + i + "_a" );
+                OrgUser userb = new OrgUser( UUID.randomUUID(), "user_" + i + "_b" );
+                OrgUser userc = new OrgUser( UUID.randomUUID(), "user_" + i + "_c" );
+
+                // add users to orgs 
+
+                Set<OrgUser> org1Users = new HashSet<OrgUser>();
+                org1Users.add( usera );
+                org1Users.add( userb );
+                usersByOrg.put( org1, org1Users );
+
+                Set<OrgUser> org2Users = new HashSet<OrgUser>();
+                org2Users.add( userc );
+                usersByOrg.put( org2, org2Users );
+
+                // add orgs to users 
+
+                Set<Org> useraOrgs = new HashSet<Org>();
+                useraOrgs.add( org1 );
+                orgsByUser.put( usera, useraOrgs );
+
+                Set<Org> userbOrgs = new HashSet<Org>();
+                userbOrgs.add( org1 );
+                orgsByUser.put( userb, userbOrgs );
+
+                Set<Org> usercOrgs = new HashSet<Org>();
+                usercOrgs.add( org2 );
+                orgsByUser.put( userc, usercOrgs );
+
+                // add some apps to the orgs
+
+                Set<UUID> org1apps = new HashSet<UUID>();
+                org1apps.add( UUID.randomUUID() );
+                org1apps.add( UUID.randomUUID() );
+                appsByOrg.put( org1, org1apps );
+
+                Set<UUID> org2apps = new HashSet<UUID>();
+                org2apps.add( UUID.randomUUID() );
+                appsByOrg.put( org2, org2apps );
+            }
+        }
+
+
+        @Override
+        public Observable<Org> getOrgs() throws Exception {
+            return Observable.from( usersByOrg.keySet() );
+        }
+
+        @Override
+        public Observable<OrgUser> getUsers() throws Exception {
+            return Observable.from( orgsByUser.keySet() );
+        }
+
+        @Override
+        public Set<Org> getUsersOrgs(OrgUser user) {
+            return orgsByUser.get( user );
+        }
+
+        @Override
+        public void removeOrg(Org keeper, Org duplicate) throws Exception {
+            Set<OrgUser> users = usersByOrg.get( duplicate );
+            for (OrgUser user : users) {
+                Set<Org> userOrgs = orgsByUser.get( user );
+                userOrgs.remove( duplicate );
+            }
+            usersByOrg.remove( duplicate );
+        }
+
+        @Override
+        public Set<OrgUser> getOrgUsers(Org org) throws Exception {
+            return usersByOrg.get( org );
+        }
+
+        @Override
+        public void removeUserFromOrg(OrgUser user, Org org) throws Exception {
+            
+            Set<OrgUser> orgUsers = usersByOrg.get( org );
+            orgUsers.remove( user );
+            
+            Set<Org> usersOrgs = orgsByUser.get( user );
+            usersOrgs.remove( org );
+        }
+
+        @Override
+        public void addUserToOrg(OrgUser user, Org org) throws Exception {
+            
+            Set<Org> usersOrgs = orgsByUser.get( user );
+            usersOrgs.add( org );
+            
+            Set<OrgUser> orgsUsers = usersByOrg.get( org );
+            orgsUsers.add( user );
+        }
+
+        @Override
+        public Set<UUID> getOrgApps(Org org) {
+            return appsByOrg.get( org );
+        }
+
+        @Override
+        public void removeAppFromOrg(UUID appId, Org org) throws Exception {
+            Set<UUID> apps = appsByOrg.get( org );
+            apps.remove( appId );
+        }
+
+        @Override
+        public void addAppToOrg(UUID appId, Org org) throws Exception {
+            Set<UUID> apps = appsByOrg.get( org );
+            apps.add(appId); 
+        }
+
+        @Override
+        public void logDuplicates(Map<String, Set<Org>> duplicatesByName) {
+
+            for (String orgName : duplicatesByName.keySet()) {
+                Set<Org> orgs = duplicatesByName.get( orgName );
+                for (Org org : orgs) {
+                    logger.info( "Duplicate org {}:{}", orgName, org.getId() );
+                }
+            }
+        }
+    }
+}


Mime
View raw message