usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From toddn...@apache.org
Subject [06/33] incubator-usergrid git commit: Updated mappings
Date Mon, 13 Apr 2015 19:18:13 GMT
Updated mappings


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/fb81d64b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/fb81d64b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/fb81d64b

Branch: refs/heads/two-dot-o-dev
Commit: fb81d64bf3b3aa73de303e705b477761db94f52b
Parents: f1f87b0
Author: Todd Nine <tnine@apigee.com>
Authored: Thu Apr 2 16:24:57 2015 -0600
Committer: Todd Nine <tnine@apigee.com>
Committed: Thu Apr 2 16:24:57 2015 -0600

----------------------------------------------------------------------
 stack/corepersistence/pom.xml                   |  26 ++--
 .../usergrid/persistence/index/SearchEdge.java  |  16 +++
 .../usergrid/persistence/index/SearchTypes.java |   2 +-
 .../persistence/index/impl/DeIndexRequest.java  |   2 +-
 .../index/impl/EsEntityIndexImpl.java           |  86 +++++++-----
 .../persistence/index/impl/IndexEdgeImpl.java   |   4 +-
 .../persistence/index/impl/IndexRequest.java    |   2 +-
 .../persistence/index/impl/IndexingUtils.java   | 137 ++-----------------
 .../persistence/index/impl/SearchEdgeImpl.java  |  11 +-
 .../persistence/index/usergrid-mappings.json    |  63 +++++++++
 .../index/impl/BufferQueueSQSImplTest.java      |   9 +-
 .../persistence/index/impl/EntityIndexTest.java |  19 ++-
 .../index/impl/IndexLoadTestsIT.java            |   7 +-
 .../persistence/query/tree/GrammarTreeTest.java |   4 +-
 14 files changed, 193 insertions(+), 195 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fb81d64b/stack/corepersistence/pom.xml
----------------------------------------------------------------------
diff --git a/stack/corepersistence/pom.xml b/stack/corepersistence/pom.xml
index b8227f7..f26f1cc 100644
--- a/stack/corepersistence/pom.xml
+++ b/stack/corepersistence/pom.xml
@@ -132,15 +132,23 @@ limitations under the License.
             </plugin>
         </plugins>
 
-        <resources>
-            <resource>
-                <directory>src/main/resources</directory>
-                <filtering>true</filtering>
-                <includes>
-                    <include>**/*.properties</include>
-                </includes>
-            </resource>
-        </resources>
+      <resources>
+        <resource>
+          <directory>src/main/resources</directory>
+          <filtering>true</filtering>
+          <includes>
+            <include>**/*.properties</include>
+          </includes>
+        </resource>
+        <!-- include everything else without filtering -->
+        <resource>
+          <directory>src/main/resources</directory>
+          <filtering>false</filtering>
+          <includes>
+            <include>**</include>
+          </includes>
+        </resource>
+      </resources>
 
         <testResources>
             <testResource>

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fb81d64b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchEdge.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchEdge.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchEdge.java
index b0b0d66..31d4dac 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchEdge.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchEdge.java
@@ -39,7 +39,23 @@ public interface SearchEdge {
      */
     String getEdgeName();
 
+    /**
+     * Get the edge type for this search edge.
+     *
+     * If the nodeId was the source in the edge, set this to source.  If it was the target,
set this to target
+     * @return
+     */
+    NodeType getNodeType();
 
 
+    /**
+     * If the nodeId was the source in the edge, set this to source.  If it was the target,
set this to target
+     *
+     */
+    enum NodeType {
+        SOURCE,
+        TARGET
+    }
+
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fb81d64b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
index 0bfef2c..6ba0756 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
@@ -48,7 +48,7 @@ public class SearchTypes {
     public String[] getTypeNames( ApplicationScope applicationScope ) {
         String[] typeNames = new String[types.length];
         for ( int i = 0; i < types.length; i++ ) {
-            typeNames[i++] = IndexingUtils.getType( applicationScope, types[i] );
+            typeNames[i] = IndexingUtils.getType( applicationScope, types[i] );
         }
         return typeNames;
     }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fb81d64b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexRequest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexRequest.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexRequest.java
index 24ea188..591e060 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexRequest.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexRequest.java
@@ -57,7 +57,7 @@ public class DeIndexRequest implements BatchRequest {
         String context = createContextName(applicationScope, searchEdge );
         this.indexes = indexes;
         this.entityTypes = SearchType.fromId(id).getTypeNames(applicationScope);
-        this.documentId =  createIndexDocId(id, version,context);
+        this.documentId =  createIndexDocId(applicationScope, id, version,context);
     }
 
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fb81d64b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 7a43f98..09bc4ff 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -20,7 +20,9 @@ package org.apache.usergrid.persistence.index.impl;
 
 import com.codahale.metrics.Meter;
 import com.codahale.metrics.Timer;
+import com.google.common.base.Charsets;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.io.Resources;
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
 import org.apache.commons.lang.StringUtils;
@@ -44,8 +46,6 @@ import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
 import org.elasticsearch.client.AdminClient;
 import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.indices.IndexAlreadyExistsException;
@@ -54,6 +54,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
+import java.net.URL;
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.List;
@@ -68,8 +69,6 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData
{
 
     private static final Logger logger = LoggerFactory.getLogger( EsEntityIndexImpl.class
);
 
-    public static final String DEFAULT_TYPE = "_default_";
-
     private final IndexAlias alias;
     private final IndexBufferProducer indexBatchBufferProducer;
     private final IndexFig indexFig;
@@ -86,10 +85,10 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData
{
     //number of milliseconds to try again before sleeping
     private static final int WAIT_TIME = 250;
 
-    private static final String VERIFY_TYPE = "verification";
+    private static final String VERIFY_TYPE = "entity";
 
     private static final ImmutableMap<String, Object> DEFAULT_PAYLOAD =
-            ImmutableMap.<String, Object>builder().put( "field", "test" ).put(IndexingUtils.ENTITYID_ID_FIELDNAME,
UUIDGenerator.newTimeUUID().toString()).build();
+            ImmutableMap.<String, Object>builder().put(IndexingUtils.ENTITYID_ID_FIELDNAME,
UUIDGenerator.newTimeUUID().toString()).build();
 
     private static final MatchAllQueryBuilder MATCH_ALL_QUERY_BUILDER = QueryBuilders.matchAllQuery();
     private final FailureMonitorImpl.IndexIdentifier indexIdentifier;
@@ -103,8 +102,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData
{
 
 
     @Inject
-    public EsEntityIndexImpl(
-                              final IndexBufferProducer indexBatchBufferProducer, final EsProvider
provider,
+    public EsEntityIndexImpl(final IndexBufferProducer indexBatchBufferProducer, final EsProvider
provider,
                               final IndexCache indexCache, final MetricsFactory metricsFactory,
                               final IndexFig indexFig, final FailureMonitorImpl.IndexIdentifier
indexIdentifier ) {
         this.indexBatchBufferProducer = indexBatchBufferProducer;
@@ -123,6 +121,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData
{
         this.refreshTimer = metricsFactory
             .getTimer(EsEntityIndexImpl.class, "refresh.timer");
         this.refreshIndexMeter = metricsFactory.getMeter(EsEntityIndexImpl.class,"refresh.meter");
+
         if(shouldInitialize()){
             initialize();
         }
@@ -133,16 +132,18 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData
{
     public void initialize() {
         final int numberOfShards = indexFig.getNumberOfShards();
         final int numberOfReplicas = indexFig.getNumberOfReplicas();
+
         aliasCache.invalidate(alias);
+
         if (shouldInitialize()) {
-            addIndex(null, numberOfShards, numberOfReplicas, indexFig.getWriteConsistencyLevel());
+            addIndex( null, numberOfShards, numberOfReplicas, indexFig.getWriteConsistencyLevel()
);
         }
     }
 
     @Override
     public boolean shouldInitialize() {
-        String[] reads = getIndexes(AliasedEntityIndex.AliasType.Read);
-        String[] writes = getIndexes(AliasedEntityIndex.AliasType.Write);
+        String[] reads = getIndexes( AliasedEntityIndex.AliasType.Read );
+        String[] writes = getIndexes( AliasedEntityIndex.AliasType.Write );
         return reads.length==0  || writes.length==0;
     }
 
@@ -158,8 +159,12 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData
{
                 Settings settings = ImmutableSettings.settingsBuilder()
                         .put("index.number_of_shards", numberOfShards)
                         .put("index.number_of_replicas", numberOfReplicas)
-                        .put("action.write_consistency", writeConsistency )
-                    .build();
+
+                        //dont' allow unmapped queries, and don't allow dynamic mapping
+                        .put("index.query.parse.allow_unmapped_fields", false )
+                        .put("index.mapper.dynamic", false)
+                        .put( "action.write_consistency", writeConsistency )
+                        .build();
 
                 //Added For Graphite Metrics
                 Timer.Context timeNewIndexCreation = addTimer.time();
@@ -226,8 +231,8 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData
{
             //add write alias
             aliasesRequestBuilder.addAlias(indexName, alias.getWriteAlias());
             isAck = aliasesRequestBuilder.execute().actionGet().isAcknowledged();
-            logger.info("Created new read and write aliases ACK=[{}]", isAck);
-            aliasCache.invalidate(alias);
+            logger.info( "Created new read and write aliases ACK=[{}]", isAck );
+            aliasCache.invalidate( alias );
 
         } catch (Exception e) {
             logger.warn("Failed to create alias ", e);
@@ -264,28 +269,23 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData
{
 
         logger.debug("Testing new index name: read {} write {}", alias.getReadAlias(), alias.getWriteAlias());
 
-        final RetryOperation retryOperation = new RetryOperation() {
-            @Override
-            public boolean doOp() {
-                final String tempId = UUIDGenerator.newTimeUUID().toString();
+        final RetryOperation retryOperation = () -> {
+            final String tempId = UUIDGenerator.newTimeUUID().toString();
 
-                esProvider.getClient().prepareIndex( alias.getWriteAlias(), VERIFY_TYPE,
tempId )
-                     .setSource(DEFAULT_PAYLOAD).get();
+            esProvider.getClient().prepareIndex( alias.getWriteAlias(), VERIFY_TYPE, tempId
)
+                 .setSource(DEFAULT_PAYLOAD).get();
 
-                logger.info( "Successfully created new document with docId {} "
-                     + "in index read {} write {} and type {}",
-                        tempId, alias.getReadAlias(), alias.getWriteAlias(), VERIFY_TYPE
);
+            logger.info( "Successfully created new document with docId {} "
+                 + "in index read {} write {} and type {}",
+                    tempId, alias.getReadAlias(), alias.getWriteAlias(), VERIFY_TYPE );
 
-                // delete all types, this way if we miss one it will get cleaned up
-                esProvider.getClient().prepareDeleteByQuery( alias.getWriteAlias())
-                        .setTypes(VERIFY_TYPE)
-                        .setQuery(MATCH_ALL_QUERY_BUILDER).get();
+            // delete all types, this way if we miss one it will get cleaned up
+            esProvider.getClient().prepareDelete( alias.getWriteAlias(), VERIFY_TYPE, tempId).get();
 
-                logger.info( "Successfully deleted all documents in index {} read {} write
{} and type {}",
-                        alias.getReadAlias(), alias.getWriteAlias(), VERIFY_TYPE );
+            logger.info( "Successfully deleted  documents in index {} read {} write {} and
type {} with id {}",
+                    alias.getReadAlias(), alias.getWriteAlias(), VERIFY_TYPE, tempId );
 
-                return true;
-            }
+            return true;
         };
 
         doInRetry( retryOperation );
@@ -298,14 +298,11 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData
{
      */
     private void createMappings(final String indexName) throws IOException {
 
-        XContentBuilder xcb = IndexingUtils.createDoubleStringIndexMapping( XContentFactory.jsonBuilder(),
-            DEFAULT_TYPE );
-
 
         //Added For Graphite Metrics
         Timer.Context timePutIndex = mappingTimer.time();
-        PutMappingResponse  pitr = esProvider.getClient().admin().indices().preparePutMapping(
indexName ).setType(
-            DEFAULT_TYPE ).setSource( xcb ).execute().actionGet();
+        PutMappingResponse  pitr = esProvider.getClient().admin().indices().preparePutMapping(
indexName ).setType( "entity" ).setSource(
+                getMappingsContent() ).execute().actionGet();
         timePutIndex.stop();
         if ( !pitr.isAcknowledged() ) {
             throw new IndexException( "Unable to create default mappings" );
@@ -313,6 +310,21 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData
{
     }
 
 
+    /**
+     * Get the content from our mappings file
+     * @return
+     */
+    private String getMappingsContent(){
+        URL url = Resources.getResource( "org/apache/usergrid/persistence/index/usergrid-mappings.json"
);
+        try {
+            return Resources.toString(url, Charsets.UTF_8);
+        }
+        catch ( IOException e ) {
+            throw new RuntimeException( "Unable to read mappings file", e );
+        }
+    }
+
+
 
 
     public void refresh() {

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fb81d64b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexEdgeImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexEdgeImpl.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexEdgeImpl.java
index 6744a3f..8b66c85 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexEdgeImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexEdgeImpl.java
@@ -36,8 +36,8 @@ public class IndexEdgeImpl extends SearchEdgeImpl implements IndexEdge {
     private final long timestamp;
 
 
-    public IndexEdgeImpl( final Id nodeId, final String name, final long timestamp ) {
-        super( nodeId, name );
+    public IndexEdgeImpl( final Id nodeId, final String name, final NodeType nodeType, final
long timestamp ) {
+        super( nodeId, name, nodeType );
         this.timestamp = timestamp;
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fb81d64b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
index bf2409a..37ed590 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
@@ -54,7 +54,7 @@ public class IndexRequest implements BatchRequest {
     }
 
     public IndexRequest( final String writeAlias, final ApplicationScope applicationScope,
String context , Entity entity) {
-        this(writeAlias, applicationScope, SearchType.fromId(entity.getId()),IndexingUtils.createIndexDocId(entity,context),
EntityToMapConverter.convert(applicationScope,entity, context));
+        this(writeAlias, applicationScope, SearchType.fromId(entity.getId()),IndexingUtils.createIndexDocId(applicationScope,
entity,context), EntityToMapConverter.convert(applicationScope,entity, context));
     }
 
     public IndexRequest( final String writeAlias, final ApplicationScope applicationScope,SearchType
searchType, String documentId,  Map<String, Object> data) {

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fb81d64b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index e066ca2..6fd1f33 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@ -105,8 +105,8 @@ public class IndexingUtils {
      * @param entity
      * @return
      */
-    public static String createIndexDocId(final Entity entity, final String context) {
-        return createIndexDocId(entity.getId(), entity.getVersion(), context);
+    public static String createIndexDocId(final ApplicationScope applicationScope, final
Entity entity, final String context) {
+        return createIndexDocId(applicationScope, entity.getId(), entity.getVersion(), context);
     }
 
 
@@ -117,132 +117,19 @@ public class IndexingUtils {
      * @para context The context it's indexed in
      * @return
      */
-    public static String createIndexDocId( final Id entityId, final UUID version, final String
context) {
-
-        throw new NotImplementedException("Fix me to use app scope too");
-//        StringBuilder sb = new StringBuilder();
-//        idString(sb, entityId);
-//        sb.append( SEPARATOR );
-//        sb.append( version.toString() ).append( SEPARATOR );
-//        sb.append( context);
-//        return sb.toString();
+    public static String createIndexDocId( final ApplicationScope applicationScope, final
Id entityId, final UUID version, final String context) {
+
+        StringBuilder sb = new StringBuilder();
+        idString( applicationScope.getApplication() );
+        sb.append( SEPARATOR );
+        idString(sb, entityId);
+        sb.append( SEPARATOR );
+        sb.append( version.toString() ).append( SEPARATOR );
+        sb.append( context);
+        return sb.toString();
     }
 
 
-    /**
-     * Build mappings for data to be indexed. Setup String fields as not_analyzed and analyzed,
-     * where the analyzed field is named {name}_ug_analyzed
-     *
-     * @param builder Add JSON object to this builder.
-     * @param type ElasticSearch type of entity.
-     *
-     * @return Content builder with JSON for mapping.
-     *
-     * @throws java.io.IOException On JSON generation error.
-     */
-    public static XContentBuilder createDoubleStringIndexMapping(
-            XContentBuilder builder, String type ) throws IOException {
-
-        builder = builder
-
-            .startObject()
-
-                    /**  add routing  "_routing":{ "required":true,  "path":"ug_entityId"
**/
-                    .startObject("_routing")
-                        .field("required", true)
-                        .field("path", ENTITYID_ID_FIELDNAME)
-                    .endObject()
-                    .startArray("dynamic_templates")
-                        // we need most specific mappings first since it's a stop on match
algorithm
-                            .startObject()
-                                .startObject("application_id_template")
-                                    .field("match", APPLICATION_ID_FIELDNAME)
-                                    .field("match_mapping_type", "string")
-                                    .startObject("mapping").field("type", "string").field("index",
"not_analyzed").field(DOC_VALUES_KEY, true).endObject()
-                                .endObject()
-                            .endObject()
-                            .startObject()
-                                .startObject("entity_id_template")
-                                    .field("match", IndexingUtils.ENTITYID_ID_FIELDNAME)
-                                    .field("match_mapping_type", "string")
-                                    .startObject("mapping").field("type", "string").field("index",
"not_analyzed").field(DOC_VALUES_KEY, true).endObject()
-                                .endObject()
-                            .endObject()
-
-                            .startObject()
-                                .startObject("entity_context_template")
-                                    .field("match", IndexingUtils.ENTITY_CONTEXT_FIELDNAME)
-                                    .field("match_mapping_type", "string")
-                                        .startObject("mapping")
-                                            .field("type", "string").field("index", "not_analyzed").field(DOC_VALUES_KEY,
true)
-                                        .endObject()
-                                .endObject()
-                            .endObject()
-
-                            .startObject()
-                                .startObject("entity_version_template")
-                                    .field("match", IndexingUtils.ENTITY_VERSION_FIELDNAME)
-                                    .field("match_mapping_type", "string")
-                                    .startObject("mapping").field("type", "long").field("index",
"not_analyzed").field(DOC_VALUES_KEY, true)
-                                    .endObject()
-                                .endObject()
-                            .endObject()
-
-                            // any string with field name that starts with sa_ gets analyzed
-                            .startObject()
-                                .startObject("template_string_analyzed")
-                                    .field("match", ANALYZED_STRING_PREFIX + "*")
-                                    .field("match_mapping_type", "string")
-                                    .startObject("mapping")
-                                        .field("type", "string")
-                                        .field("index", "analyzed")
-                                    .endObject()
-                                .endObject()
-                            .endObject()
-
-                        // all other strings are not analyzed
-                        .startObject()
-                            .startObject("template_string_not_analyzed")
-                                //todo, should be string prefix, remove 2 field mapping
-                                .field("match", "*")
-                                .field("match_mapping_type", "string")
-                                .startObject("mapping")
-                                    .field("type", "string")
-                                    .field("index", "not_analyzed")
-                                .endObject()
-                            .endObject()
-                        .endObject()
-
-                        // fields names starting with go_ get geo-indexed
-                        .startObject()
-                            .startObject("template_geo")
-                                .field("match", GEO_PREFIX + "location")
-                                    .startObject("mapping").field("type", "geo_point")
-                                    .endObject()
-                            .endObject()
-                        .endObject()
-                            // all other strings are not analyzed
-                        .startObject()
-                            .startObject("template__long")
-                                .field("match", LONG_PREFIX + "*")
-                                .field("match_mapping_type", "long")
-                                .startObject("mapping").field("type", "long").field("index",
"not_analyzed").field(DOC_VALUES_KEY, true).endObject()
-                            .endObject()
-                        .endObject()
-
-                        .startObject()
-                            .startObject("template__double")
-                                .field("match", DOUBLE_PREFIX + "*")
-                                .field("match_mapping_type", "double")
-                                .startObject("mapping").field("type", "double").field("index",
"not_analyzed").field(DOC_VALUES_KEY, true).endObject()
-                            .endObject()
-                        .endObject()
-
-                    .endArray()
-            .endObject();
-
-        return builder;
-    }
 
 
     public static String getType(ApplicationScope applicationScope, Id entityId) {

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fb81d64b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchEdgeImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchEdgeImpl.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchEdgeImpl.java
index 4a5c376..1813017 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchEdgeImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchEdgeImpl.java
@@ -19,7 +19,6 @@
 package org.apache.usergrid.persistence.index.impl;
 
 import org.apache.usergrid.persistence.index.SearchEdge;
-import org.apache.usergrid.persistence.index.utils.IndexValidationUtils;
 import org.apache.usergrid.persistence.model.entity.Id;
 
 
@@ -30,11 +29,13 @@ import org.apache.usergrid.persistence.model.entity.Id;
 public class SearchEdgeImpl implements SearchEdge {
     private final Id nodeId;
     private final String name;
+    private final NodeType nodeType;
 
 
-    public SearchEdgeImpl( final Id nodeId, final String name ) {
+    public SearchEdgeImpl( final Id nodeId, final String name, final NodeType nodeType )
{
         this.nodeId = nodeId;
         this.name = name;
+        this.nodeType = nodeType;
     }
 
 
@@ -48,4 +49,10 @@ public class SearchEdgeImpl implements SearchEdge {
     public String getEdgeName() {
         return name;
     }
+
+
+    @Override
+    public NodeType getNodeType() {
+        return nodeType;
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fb81d64b/stack/corepersistence/queryindex/src/main/resources/org/apache/usergrid/persistence/index/usergrid-mappings.json
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/resources/org/apache/usergrid/persistence/index/usergrid-mappings.json
b/stack/corepersistence/queryindex/src/main/resources/org/apache/usergrid/persistence/index/usergrid-mappings.json
new file mode 100644
index 0000000..587eab3
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/resources/org/apache/usergrid/persistence/index/usergrid-mappings.json
@@ -0,0 +1,63 @@
+{
+  "entity": {
+    "properties": {
+      "entityId": {
+        "type": "string",
+        "index": "not_analyzed"
+      },
+      "entityVersion": {
+        "type": "string",
+        "index": "not_analyzed"
+      },
+      "entityType": {
+        "type": "string",
+        "index": "not_analyzed"
+      },
+      "applicationId": {
+        "type": "string",
+        "index": "not_analyzed"
+      },
+      "edgeNodeId": {
+        "type": "string",
+        "index": "not_analyzed"
+      },
+      "edgeName": {
+        "type": "string",
+        "index": "not_analyzed"
+      },
+      "edgeType": {
+        "type": "string",
+        "index": "not_analyzed"
+      },
+      "edgeTimestamp": {"type": "long"},
+      "edgeSearch": {
+        "type": "string",
+        "index": "not_analyzed"
+      },
+      "fields": {
+        "type": "nested",
+        "properties": {
+          "name": {
+            "type": "string",
+            "index": "not_analyzed"
+          },
+          "boolean": {"type": "boolean"},
+          "short": {"type": "short"},
+          "int": {"type": "integer"},
+          "long": {"type": "long"},
+          "float": {"type": "float"},
+          "double": {"type": "double"},
+          "string": {
+            "type": "string",
+            "fields": {
+              "exact": {
+                "type": "string",
+                "index": "not_analyzed"
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fb81d64b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java
b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java
index f92c57a..12be269 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java
@@ -25,6 +25,7 @@ import java.util.concurrent.TimeUnit;
 
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
 import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.index.SearchEdge;
 import org.apache.usergrid.persistence.index.SearchType;
 import org.apache.usergrid.persistence.model.entity.SimpleId;
 import org.junit.Before;
@@ -104,9 +105,13 @@ public class BufferQueueSQSImplTest {
 
 
         //de-index request
-        final DeIndexRequest deIndexRequest1 = new DeIndexRequest( new String[]{"index1.1,
index1.2"}, applicationScope, new SearchEdgeImpl(new SimpleId("testId3"),"name3"),  new SimpleId("id3"),
UUID.randomUUID() );
+        final DeIndexRequest deIndexRequest1 = new DeIndexRequest( new String[]{"index1.1,
index1.2"}, applicationScope, new SearchEdgeImpl(new SimpleId("testId3"),"name3",
 
-        final DeIndexRequest deIndexRequest2 = new DeIndexRequest( new String[]{"index2.1",
"index2.1"}, applicationScope,  new SearchEdgeImpl(new SimpleId("testId4"),"name4"),  new
SimpleId("id4"), UUID.randomUUID()  );
+
+                SearchEdge.NodeType.SOURCE ),  new SimpleId("id3"), UUID.randomUUID() );
+
+        final DeIndexRequest deIndexRequest2 = new DeIndexRequest( new String[]{"index2.1",
"index2.1"}, applicationScope,  new SearchEdgeImpl(new SimpleId("testId4"),"name4",
+                SearchEdge.NodeType.SOURCE ),  new SimpleId("id4"), UUID.randomUUID()  );
 
 
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fb81d64b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index ac37a61..0bdf1ca 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -87,7 +87,6 @@ public class EntityIndexTest extends BaseIT {
     @Inject
     public EntityIndex ei;
 
-    //TODO T.N. Remove this when we move the cursor mapping back to core
     @Inject
     @Rule
     public MigrationManagerRule migrationManagerRule;
@@ -108,7 +107,7 @@ public class EntityIndexTest extends BaseIT {
 
 
         final String entityType = "thing";
-        IndexEdge searchEdge = new IndexEdgeImpl( appId, "things", 1 );
+        IndexEdge searchEdge = new IndexEdgeImpl( appId, "things", SearchEdge.NodeType.SOURCE,
1 );
         final SearchTypes searchTypes = SearchTypes.fromTypes( entityType );
 
         insertJsonBlob( entityIndex, entityType, searchEdge, "/sample-large.json", 101, 0
);
@@ -129,7 +128,7 @@ public class EntityIndexTest extends BaseIT {
         ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex( applicationScope
);
 
         final String entityType = "thing";
-        IndexEdge indexEdge = new IndexEdgeImpl( appId, "things", 1 );
+        IndexEdge indexEdge = new IndexEdgeImpl( appId, "things", SearchEdge.NodeType.SOURCE,
 1 );
         final SearchTypes searchTypes = SearchTypes.fromTypes( entityType );
         EntityIndexBatch batch = entityIndex.createBatch();
         Entity entity = new Entity( entityType );
@@ -179,7 +178,7 @@ public class EntityIndexTest extends BaseIT {
         final List<Object> sampleJson = mapper.readValue( is, new TypeReference<List<Object>>()
{} );
         for ( int i = 0; i < threads; i++ ) {
 
-            final IndexEdge indexEdge = new IndexEdgeImpl( appId, "things", i );
+            final IndexEdge indexEdge = new IndexEdgeImpl( appId, "things",  SearchEdge.NodeType.SOURCE,
i );
 
             Thread thread = new Thread( () -> {
                 try {
@@ -239,7 +238,7 @@ public class EntityIndexTest extends BaseIT {
 
 
         final String entityType = "thing";
-        IndexEdge searchEdge = new IndexEdgeImpl( appId, "things", 10 );
+        IndexEdge searchEdge = new IndexEdgeImpl( appId, "things", SearchEdge.NodeType.SOURCE,
 10 );
         final SearchTypes searchTypes = SearchTypes.fromTypes( entityType );
 
         insertJsonBlob( entityIndex, entityType, searchEdge, "/sample-large.json", 101, 0
);
@@ -271,7 +270,7 @@ public class EntityIndexTest extends BaseIT {
 
 
         final String entityType = "thing";
-        IndexEdge searchEdge = new IndexEdgeImpl( appId, "things", 1 );
+        IndexEdge searchEdge = new IndexEdgeImpl( appId, "things",  SearchEdge.NodeType.SOURCE,
1 );
         final SearchTypes searchTypes = SearchTypes.fromTypes( entityType );
 
         insertJsonBlob( entityIndex, entityType, searchEdge, "/sample-large.json", 1, 0 );
@@ -351,7 +350,7 @@ public class EntityIndexTest extends BaseIT {
 
         ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
 
-        IndexEdge searchEdge = new IndexEdgeImpl( appId, "fastcars", 1 );
+        IndexEdge searchEdge = new IndexEdgeImpl( appId, "fastcars", SearchEdge.NodeType.SOURCE,
 1 );
 
         ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex( applicationScope
);
 
@@ -488,7 +487,7 @@ public class EntityIndexTest extends BaseIT {
 
         ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
 
-        IndexEdge indexSCope = new IndexEdgeImpl( ownerId, "user", 10 );
+        IndexEdge indexSCope = new IndexEdgeImpl( ownerId, "user",  SearchEdge.NodeType.SOURCE,
10 );
 
         ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex( applicationScope
);
 
@@ -537,7 +536,7 @@ public class EntityIndexTest extends BaseIT {
         Id ownerId = new SimpleId( "multivaluedtype" );
         ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
 
-        IndexEdge indexScope = new IndexEdgeImpl( ownerId, "user", 10 );
+        IndexEdge indexScope = new IndexEdgeImpl( ownerId, "user",  SearchEdge.NodeType.SOURCE,
10 );
 
         ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex( applicationScope
);
 
@@ -623,7 +622,7 @@ public class EntityIndexTest extends BaseIT {
 
         ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
 
-        IndexEdge indexEdge = new IndexEdgeImpl( ownerId, "users", 10 );
+        IndexEdge indexEdge = new IndexEdgeImpl( ownerId, "users",  SearchEdge.NodeType.SOURCE,
10 );
 
 
         ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex( applicationScope
);

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fb81d64b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java
b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java
index 035d87f..8b928ce 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java
@@ -133,8 +133,6 @@ public class IndexLoadTestsIT extends BaseIT {
         final ApplicationScope scope = new ApplicationScopeImpl( applicationId );
 
 
-        final SearchEdge searchEdge = new SearchEdgeImpl( applicationId, "test" );
-
         batchWriteTPS = metricsFactory.getMeter( IndexLoadTestsIT.class, "write.tps" );
 
         batchWriteTimer = metricsFactory.getTimer( IndexLoadTestsIT.class, "write.timer"
);
@@ -182,7 +180,7 @@ public class IndexLoadTestsIT extends BaseIT {
         final ApplicationScope scope = new ApplicationScopeImpl( applicationId );
 
 
-        final SearchEdge searchEdge = new SearchEdgeImpl( applicationId, "test" );
+        final SearchEdge searchEdge = new SearchEdgeImpl( applicationId, "test",  SearchEdge.NodeType.SOURCE
);
 
         final ApplicationEntityIndex appEntityIndex = entityIndexFactory.createApplicationEntityIndex(
scope );
 
@@ -328,7 +326,8 @@ public class IndexLoadTestsIT extends BaseIT {
 
                 //take our entities and roll them into a batch
                 Observable.from( entities ).collect( () -> entityIndex.createBatch(),
( entityIndexBatch, entity ) -> {
-                    IndexEdge edge = new IndexEdgeImpl( indexEdge.getNodeId(), indexEdge.getEdgeName(),
edgeCounter.incrementAndGet()  );
+                    IndexEdge edge = new IndexEdgeImpl( indexEdge.getNodeId(), indexEdge.getEdgeName(),
+                            SearchEdge.NodeType.SOURCE, edgeCounter.incrementAndGet()  );
                     entityIndexBatch.index( edge, entity );
                 } ).doOnNext( entityIndexBatch -> {
                     log.info( "Indexing next {} in batch", entityIndexBatch.size() );

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fb81d64b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
index 2ae13e0..ada7420 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
@@ -29,6 +29,7 @@ import org.elasticsearch.index.query.QueryBuilder;
 import org.junit.Test;
 
 import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.index.SearchEdge;
 import org.apache.usergrid.persistence.index.exceptions.QueryParseException;
 import org.apache.usergrid.persistence.index.impl.SearchEdgeImpl;
 import org.apache.usergrid.persistence.index.impl.SearchRequestBuilderStrategy;
@@ -438,7 +439,8 @@ public class GrammarTreeTest {
         SearchRequestBuilderStrategy builderStrategy =
                 new SearchRequestBuilderStrategy( null, new ApplicationScopeImpl( new SimpleId(
"test" ) ), null, 100 );
         QueryBuilder qb =
-                builderStrategy.createQueryBuilder( new SearchEdgeImpl( new SimpleId( "owner"
), "app" ), query );
+                builderStrategy.createQueryBuilder( new SearchEdgeImpl( new SimpleId( "owner"
), "app",
+                        SearchEdge.NodeType.SOURCE ), query );
     }
 
 


Mime
View raw message