directory-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kayyag...@apache.org
Subject svn commit: r1622226 - /directory/apacheds/trunk/bulkloader/src/main/java/org/apache/directory/mavibot/btree/LdifDataLoader.java
Date Wed, 03 Sep 2014 11:24:46 GMT
Author: kayyagari
Date: Wed Sep  3 11:24:46 2014
New Revision: 1622226

URL: http://svn.apache.org/r1622226
Log:
LDIF loader

Added:
    directory/apacheds/trunk/bulkloader/src/main/java/org/apache/directory/mavibot/btree/LdifDataLoader.java

Added: directory/apacheds/trunk/bulkloader/src/main/java/org/apache/directory/mavibot/btree/LdifDataLoader.java
URL: http://svn.apache.org/viewvc/directory/apacheds/trunk/bulkloader/src/main/java/org/apache/directory/mavibot/btree/LdifDataLoader.java?rev=1622226&view=auto
==============================================================================
--- directory/apacheds/trunk/bulkloader/src/main/java/org/apache/directory/mavibot/btree/LdifDataLoader.java
(added)
+++ directory/apacheds/trunk/bulkloader/src/main/java/org/apache/directory/mavibot/btree/LdifDataLoader.java
Wed Sep  3 11:24:46 2014
@@ -0,0 +1,192 @@
+/*
+ *   Licensed to the Apache Software Foundation (ASF) under one
+ *   or more contributor license agreements.  See the NOTICE file
+ *   distributed with this work for additional information
+ *   regarding copyright ownership.  The ASF licenses this file
+ *   to you under the Apache License, Version 2.0 (the
+ *   "License"); you may not use this file except in compliance
+ *   with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing,
+ *   software distributed under the License is distributed on an
+ *   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ *   KIND, either express or implied.  See the License for the
+ *   specific language governing permissions and limitations
+ *   under the License.
+ *
+ */
+package org.apache.directory.mavibot.btree;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.directory.api.ldap.model.entry.Entry;
+import org.apache.directory.api.ldap.model.schema.SchemaManager;
+import org.apache.directory.api.ldap.model.schema.registries.Schema;
+import org.apache.directory.api.ldap.model.schema.registries.SchemaLoader;
+import org.apache.directory.api.ldap.schemaloader.LdifSchemaLoader;
+import org.apache.directory.api.ldap.schemamanager.impl.DefaultSchemaManager;
+import org.apache.directory.mavibot.btree.memory.BulkDataSorter;
+import org.apache.directory.mavibot.btree.serializer.ElementSerializer;
+import org.apache.directory.mavibot.btree.serializer.StringSerializer;
+import org.apache.directory.server.core.partition.impl.btree.mavibot.LdifTupleComparator;
+import org.apache.directory.server.core.partition.impl.btree.mavibot.LdifTupleReaderWriter;
+import org.apache.directory.server.core.partition.impl.btree.mavibot.MavibotEntrySerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * TODO LdifDataLoader.
+ *
+ * @author <a href="mailto:dev@directory.apache.org">Apache Directory Project</a>
+ */
+public class LdifDataLoader
+{
+    private SchemaManager schemaManager;
+    
+    private ElementSerializer<String> keySerializer = StringSerializer.INSTANCE;
+    
+    private ElementSerializer<Entry> valSerializer = new MavibotEntrySerializer();
+
+    private static final Logger LOG = LoggerFactory.getLogger( LdifDataLoader.class );
+    
+    public LdifDataLoader()
+    {
+    }
+    
+    public boolean load( String schemaPartitionDir, String ldifFile, String targetPartitionDir
)
+    {
+        initSchema( schemaPartitionDir );
+        
+        if( schemaManager == null )
+        {
+            LOG.warn( "Failed to load the schema, aborting data load" );
+            return false;
+        }
+        
+        return load( ldifFile, targetPartitionDir );
+    }
+
+    
+    public boolean load( String ldifFile, String targetPartitionDir )
+    {
+
+        if( schemaManager == null )
+        {
+            LOG.warn( "No SchemaManager instance was found, aborting data load" );
+            return false;
+        }
+
+        File dataFile = new File( ldifFile );
+        
+        if( !dataFile.exists() )
+        {
+            LOG.warn( "File {} does not exist", ldifFile );
+            return false;
+        }
+        else
+        {
+            if( !dataFile.canRead() )
+            {
+                LOG.warn( "File {} cannot be read by the current user", ldifFile );
+                return false;
+            }
+        }
+        
+        RecordManager rm = new RecordManager( targetPartitionDir );
+        Set<String> existing = rm.getManagedTrees();
+        if( existing.size() > 2 )
+        {
+            LOG.warn( "Looks like the given partition directory {} already contains data
of a mavibot partiton, please delete this data file and rerun this tool", ldifFile );
+            return false;
+        }
+        
+        //MavibotPartitionBuilder<String, Entry> builder = new MavibotPartitionBuilder<String,
Entry>( rm, "master", BTree.DEFAULT_PAGE_SIZE, keySerializer, valSerializer );
+        
+        LdifTupleReaderWriter readerWriter = new LdifTupleReaderWriter( ldifFile, schemaManager
);
+        LdifTupleComparator tupleComparator = new LdifTupleComparator();
+        
+        BulkDataSorter sorter = new BulkDataSorter( readerWriter, tupleComparator, 10 );
+        
+        try
+        {
+            sorter.sort( dataFile );
+            //builder.build( sorter.getMergeSortedTuples() );
+            return true;
+        }
+        catch( Exception e )
+        {
+            LOG.warn( "Errors occurred while loading data from the data file {}", ldifFile,
e );
+        }
+        finally
+        {
+            try
+            {
+                rm.close();
+            }
+            catch( IOException e )
+            {
+                LOG.warn( "Failed to close the recordmanager", e );
+            }
+        }
+        
+        return false;
+    }
+    
+    
+    private void initSchema( String dir )
+    {
+        if( schemaManager != null )
+        {
+            return;
+        }
+        
+        try
+        {
+            File schemaRepository = new File( dir );
+            
+            if( !schemaRepository.exists() )
+            {
+                LOG.warn( "The given schema location {} does not exist", dir );
+            }
+            
+            SchemaLoader loader = new LdifSchemaLoader( schemaRepository );
+            schemaManager = new DefaultSchemaManager( loader );
+            
+            LOG.debug( "Loading all enabled schemas" );
+            schemaManager.loadAllEnabled();
+            
+            LOG.debug( "Loading all disabled schemas" );
+            List<Schema> lstDisabled = schemaManager.getDisabled();
+            for( Schema s : lstDisabled )
+            {
+                schemaManager.loadDisabled( s );
+            }
+            
+            LOG.debug( "Successfully loaded schemas" );
+        }
+        catch( Exception e )
+        {
+            schemaManager = null;
+            LOG.warn( e.getMessage(), e );
+        }
+    }
+    
+    
+    /**
+     * @param schemaManager the schemaManager to set
+     */
+    public void setSchemaManager( SchemaManager schemaManager )
+    {
+        this.schemaManager = schemaManager;
+    }
+
+    public static void main( String[] args )
+    {
+        
+    }
+}



Mime
View raw message