pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From knogu...@apache.org
Subject svn commit: r1828532 - in /pig/trunk: CHANGES.txt src/org/apache/pig/newplan/logical/visitor/LineageFindRelVisitor.java test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java
Date Fri, 06 Apr 2018 17:07:25 GMT
Author: knoguchi
Date: Fri Apr  6 17:07:25 2018
New Revision: 1828532

URL: http://svn.apache.org/viewvc?rev=1828532&view=rev
Log:
PIG-5333: LoadCaster sometimes not set for complex type (knoguchi)

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/newplan/logical/visitor/LineageFindRelVisitor.java
    pig/trunk/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1828532&r1=1828531&r2=1828532&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri Apr  6 17:07:25 2018
@@ -63,6 +63,8 @@ PIG-5251: Bump joda-time to 2.9.9 (dbist
 OPTIMIZATIONS
  
 BUG FIXES
+PIG-5333: LoadCaster sometimes not set for complex type (knoguchi)
+
 PIG-5328: expressionOperator Divide.equalsZero(DataType.BIGDECIMAL) is invalid (michaelthoward
via knoguchi)
 
 PIG-5320: TestCubeOperator#testRollupBasic is flaky on Spark 2.2 (nkollar via szita)

Modified: pig/trunk/src/org/apache/pig/newplan/logical/visitor/LineageFindRelVisitor.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/visitor/LineageFindRelVisitor.java?rev=1828532&r1=1828531&r2=1828532&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/visitor/LineageFindRelVisitor.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/visitor/LineageFindRelVisitor.java Fri Apr
 6 17:07:25 2018
@@ -139,7 +139,7 @@ public class LineageFindRelVisitor exten
             ){
                 long inpUid = predSchema.getField(innerLoad.getProjection().getColNum()).uid;
                 if(uid2LoadFuncMap.get(inpUid) != null){
-                    addUidLoadFuncToMap(innerLoad.getSchema().getField(0).uid, uid2LoadFuncMap.get(inpUid));
+                    addUidLoadFuncToMap(innerLoad.getSchema().getField(0), uid2LoadFuncMap.get(inpUid));
                 }
                 return;
             }
@@ -165,8 +165,7 @@ public class LineageFindRelVisitor exten
             return;
         }
         for(LogicalFieldSchema fs : schema.getFields()){
-            addUidLoadFuncToMap((Long) fs.uid, funcSpec);
-            setLoadFuncForUids(fs.schema, funcSpec);
+            addUidLoadFuncToMap(fs, funcSpec);
         }
         
     }
@@ -291,9 +290,7 @@ public class LineageFindRelVisitor exten
         //if the group plans are associated with same load function , associate
         //same load fucntion with group column schema
         if (getAssociatedLoadFunc(group)!=null) {
-            addUidLoadFuncToMap(sch.getField(0).uid, rel2InputFuncMap.get(group));
-            if (sch.getField(0).schema!=null)
-                setLoadFuncForUids(sch.getField(0).schema, rel2InputFuncMap.get(group));
+            addUidLoadFuncToMap(sch.getField(0), rel2InputFuncMap.get(group));
         }
         else
             mapMatchLoadFuncToUid(sch.getField(0), groupPlanSchemas);
@@ -311,10 +308,9 @@ public class LineageFindRelVisitor exten
         
         
         for(int i=1; i < sch.size(); i++){
-            long uid = sch.getField(i).uid;
             LogicalRelationalOperator input = (LogicalRelationalOperator) inputs.get(i-1);
             if(getAssociatedLoadFunc(input) != null){
-                addUidLoadFuncToMap(uid, rel2InputFuncMap.get(input));
+                addUidLoadFuncToMap(sch.getField(i), rel2InputFuncMap.get(input));
             }
         }
         
@@ -370,7 +366,7 @@ public class LineageFindRelVisitor exten
                 //get its funcspec and associate it with uid of all fields in the schema
                 FuncSpec funcSpec = uid2LoadFuncMap.get(exp.getFieldSchema().uid);
                 for(LogicalFieldSchema fs : sch.getFields()){
-                    addUidLoadFuncToMap(fs.uid, funcSpec);
+                    addUidLoadFuncToMap(fs, funcSpec);
                 }
             }
         }
@@ -461,7 +457,7 @@ public class LineageFindRelVisitor exten
                   prevLoadFuncSpec  = curLoadFuncSpec;
                 }
                 if( allSameLoader ) {
-                    addUidLoadFuncToMap(logicalFieldSchema.uid,curLoadFuncSpec);
+                    addUidLoadFuncToMap(logicalFieldSchema,curLoadFuncSpec);
                 }
             }
         }
@@ -502,7 +498,7 @@ public class LineageFindRelVisitor exten
             LogicalFieldSchema inField = inputSch.getField(i);
             LogicalFieldSchema outField = outSchema.getField(i);
             if(uid2LoadFuncMap.get(inField.uid) != null){
-                addUidLoadFuncToMap(outField.uid, uid2LoadFuncMap.get(inField.uid));
+                addUidLoadFuncToMap(outField, uid2LoadFuncMap.get(inField.uid));
             }
         }
         
@@ -515,15 +511,17 @@ public class LineageFindRelVisitor exten
      * @param loadFuncSpec
      * @throws VisitorException 
      */
-    private void addUidLoadFuncToMap(long uid, FuncSpec loadFuncSpec)
+    private void addUidLoadFuncToMap(LogicalFieldSchema fs, FuncSpec loadFuncSpec)
     throws VisitorException{
         if(loadFuncSpec == null){
             return;
         }
         //ensure that uid always matches to same load func
-        FuncSpec curFuncSpec = uid2LoadFuncMap.get(uid);
+        FuncSpec curFuncSpec = uid2LoadFuncMap.get(fs.uid);
         if(curFuncSpec == null){
-            uid2LoadFuncMap.put(uid, loadFuncSpec);
+            uid2LoadFuncMap.put(fs.uid, loadFuncSpec);
+            // if field is tuple/map/bag, also set the internal fields
+            setLoadFuncForUids(fs.schema, loadFuncSpec);
         }else if(! haveIdenticalCasters(curFuncSpec,loadFuncSpec)){
             String msg = "Bug: uid mapped to two different load functions : " +
             curFuncSpec + " and " + loadFuncSpec;
@@ -574,7 +572,7 @@ public class LineageFindRelVisitor exten
                 }
             }
             if(allMatch){
-                addUidLoadFuncToMap(outFS.uid, funcSpec1);
+                addUidLoadFuncToMap(outFS, funcSpec1);
             }
         }
         
@@ -624,7 +622,7 @@ public class LineageFindRelVisitor exten
             if(uid2LoadFuncMap.get(uid) == null && (inputRel.getSchema() == null
|| inputRel instanceof LOInnerLoad)){
                 FuncSpec funcSpec = rel2InputFuncMap.get(inputRel);
                 if(funcSpec != null){
-                    addUidLoadFuncToMap(uid, funcSpec);
+                    addUidLoadFuncToMap(proj.getFieldSchema(), funcSpec);
                 }
             }
         }
@@ -645,7 +643,7 @@ public class LineageFindRelVisitor exten
             //find input uid and corresponding load FuncSpec
             long inpUid = inp.getFieldSchema().uid;
             FuncSpec inpLoadFuncSpec = uid2LoadFuncMap.get(inpUid);
-            addUidLoadFuncToMap(exp.getFieldSchema().uid, inpLoadFuncSpec);
+            addUidLoadFuncToMap(exp.getFieldSchema(), inpLoadFuncSpec);
 
         }
 
@@ -689,7 +687,7 @@ public class LineageFindRelVisitor exten
             Integer inputColNum = (Integer)((ConstantExpression) scalarExp.getArguments().get(0)).getValue();
             String inputFile = (String)((ConstantExpression) scalarExp.getArguments().get(1)).getValue();
             
-            long outputUid = scalarExp.getFieldSchema().uid;
+            LogicalFieldSchema outputFS = scalarExp.getFieldSchema();
             boolean foundInput = false; // a variable to do sanity check on num of input
relations
 
             //find the input relation, and use it to get lineage
@@ -711,12 +709,12 @@ public class LineageFindRelVisitor exten
                     if(sch == null){
                         //see if there is a load function associated with the store
                         FuncSpec funcSpec = rel2InputFuncMap.get(inputStore);
-                        addUidLoadFuncToMap(outputUid, funcSpec);
+                        addUidLoadFuncToMap(outputFS, funcSpec);
                     }else{
                         //find input uid and corresponding load func
                         LogicalFieldSchema fs = sch.getField(inputColNum);
                         FuncSpec funcSpec = uid2LoadFuncMap.get(fs.uid);
-                        addUidLoadFuncToMap(outputUid, funcSpec);
+                        addUidLoadFuncToMap(outputFS, funcSpec);
                     }
                 }
             }
@@ -762,9 +760,7 @@ public class LineageFindRelVisitor exten
             }
 
             if( funcSpec != null ) {
-                addUidLoadFuncToMap(op.getFieldSchema().uid, funcSpec);
-                // in case schema is nested, set funcSpec for all
-                setLoadFuncForUids(op.getFieldSchema().schema, funcSpec);
+                addUidLoadFuncToMap(op.getFieldSchema(), funcSpec);
             }
         }
 

Modified: pig/trunk/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java?rev=1828532&r1=1828531&r2=1828532&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java Fri Apr  6 17:07:25
2018
@@ -4174,4 +4174,39 @@ public class TestTypeCheckingValidatorNe
                 + "c = foreach b generate (gpa>3? b1 : b2);";
                 createAndProcessLPlan(query);
         }
+        @Test
+        public void testSplitLineageWithInnerFields() throws Throwable {
+            String query = "A = load 'a' as (field1, field2);"
+            + " B = group A by (field1,field2);"
+            + "split B into C if SIZE(A) > 2, Z otherwise;"
+            + "D = FOREACH C { "
+            + "      D1 = FOREACH A generate (chararray) field1;"
+            + "      GENERATE D1;"
+            + "}";
+            LOForEach outerForeach = getForeachFromPlan(query);
+            LogicalPlan innerPlan = outerForeach.getInnerPlan();
+            LOForEach innerForeach = (LOForEach)innerPlan.getPredecessors(innerPlan.getSinks().get(0)).get(0);
+            LogicalExpressionPlan innerForeachPlan = ((LOGenerate)innerForeach.getInnerPlan().getSinks().get(0)).getOutputPlans().get(0);
+            CastExpression cast = getCastFromExpPlan(innerForeachPlan);
+            checkCastLoadFunc(cast, "org.apache.pig.builtin.PigStorage");
+            /*
+              D: (Name: LOForEach Schema: D1#755:bag{#754:tuple(field1#750:chararray)}) 
         =====> [outerForeach]
+              |   |
+              |   (Name: LOGenerate[false] Schema: D1#755:bag{#754:tuple(field1#750:chararray)})
   ====> innerPlan.getSinks().get(0)
+              |   |   |
+              |   |   D1:(Name: Project Type: bag Uid: 755 Input: 0 Column: (*))
+              |   |
+              |   |---D1: (Name: LOForEach Schema: field1#750:chararray)                
              =====> [innerForeach]
+              |       |   |
+              |       |   (Name: LOGenerate[false] Schema: field1#750:chararray)
+              |       |   |   |
+              |       |   |   (Name: Cast Type: chararray Uid: 750)                     
<========CHECKING HERE
+              |       |   |   |
+              |       |   |   |---field1:(Name: Project Type: bytearray Uid: 750 Input: 0
Column: (*))
+              |       |   |
+              |       |   |---(Name: LOInnerLoad[field1] Schema: field1#750:bytearray)
+              |       |
+              |       |---A: (Name: LOInnerLoad[A] Schema: field1#750:bytearray)
+            */
+        }
 }



Mime
View raw message