spark-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From 吴志龙 (JIRA) <j...@apache.org>
Subject [jira] [Created] (SPARK-19510) org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 44, Column 42: A method named "evaluate" is not declared in any enclosing class nor any supertype, nor through a static import
Date Wed, 08 Feb 2017 09:21:41 GMT
吴志龙 created SPARK-19510:
---------------------------

             Summary: org.codehaus.commons.compiler.CompileException: File 'generated.java',
Line 44, Column 42: A method named "evaluate" is not declared in any enclosing class nor any
supertype, nor through a static import
                 Key: SPARK-19510
                 URL: https://issues.apache.org/jira/browse/SPARK-19510
             Project: Spark
          Issue Type: Bug
          Components: SQL
    Affects Versions: 2.1.0
         Environment: spark 2.1 on yarn client
hadoop 2.6.0
jdk 1.7

            Reporter: 吴志龙


spark sql add function
example:
1、add spark Fql_Base64_Decode.scala
package org.apache.spark.sql.catalyst.expressions
import org.apache.spark.sql.types._
import org.apache.spark.unsafe.types.{ByteArray, UTF8String}
import org.apache.spark.sql.catalyst.expressions.codegen._
import java.net.URLDecoder
/**
 * Base64_Decode 转码
 */
// scalastyle:off line.size.limit
@ExpressionDescription(
  usage = "_FUNC_(Base64) fql_base64_decode",
  extended = """
    Examples:
      > SELECT _FUNC_('VlVdQ0hTV1VFSFRVXkZB');
      return 201607290000239
  """)
case class Fql_Base64_Decode(child: Expression)  extends UnaryExpression with ImplicitCastInputTypes{
  override def dataType: DataType = StringType
  override def inputTypes: Seq[DataType] = Seq(StringType)

  protected override def nullSafeEval(string: Any): Any = {
    val base64 = string.asInstanceOf[UTF8String]
    org.apache.spark.sql.catalyst.expressions.FqlBase64Decode.evaluate(base64.toString())
  }

  override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
    nullSafeCodeGen(ctx, ev, (child) => {
      s"""
         ${ev.value} = Fql_Base64_Decode().evaluate(child.toString());
       """})
  }
  
}

2、add org.apache.spark.sql.catalyst.analysis.FunctionRegistry functions name

expression[Fql_Base64_Decode]("fql_base64_decode")

3、org.apache.spark.sql.functions.scala add functions 
def fql_base64_decode(e: Column): Column = withExpr { Fql_Base64_Decode(e.expr) }

4、compile package 
 mvn -Pyarn -Phadoop-2.6 -Dhadoop.version=2.6.0 -Phadoop-provided -Phive -Phive-thriftserver
-Pnetlib-lgpl -DskipTests clean package
5、replace spark-catalyst_2.11-2.1.0.jar
6、start thrift 
7、beeline connection
8、test spark sql functions
0: jdbc:hive2://6.hadoop.com:10008> select fql_base64_encode('201607290000239');
+-------------------------------------+--+
| fql_base64_encode(201607290000239)  |
+-------------------------------------+--+
| VlVdQ0hTV1VFSFRVXkZB                |
+-------------------------------------+--+
1 row selected (0.11 seconds)
0: jdbc:hive2://6.hadoop.com:10008> 
0: jdbc:hive2://6.hadoop.com:10008> 
0: jdbc:hive2://6.hadoop.com:10008> select * from dp_tmp.test_wuzl;
+-----------------------+--+
|         code          |
+-----------------------+--+
| VlVdQ0hTV1VFSFRVXkZB  |
+-----------------------+--+
1 row selected (0.383 seconds)
0: jdbc:hive2://6.hadoop.com:10008> select fql_base64_encode(code) from dp_tmp.test_wuzl;

Error: org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage
7.0 failed 10 times, most recent failure: Lost task 0.9 in stage 7.0 (TID 54, 9.hadoop.com,
executor 3): java.util.concurrent.ExecutionException: java.lang.Exception: failed to compile:
org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 44, Column 42:
A method named "evaluate" is not declared in any enclosing class nor any supertype, nor through
a static import
/* 001 */ public java.lang.Object generate(Object[] references) {
/* 002 */   return new SpecificUnsafeProjection(references);
/* 003 */ }
/* 004 */
/* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection
{
/* 006 */
/* 007 */   private Object[] references;
/* 008 */   private UnsafeRow result;
/* 009 */   private org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder holder;
/* 010 */   private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter rowWriter;
/* 011 */
/* 012 */   public SpecificUnsafeProjection(Object[] references) {
/* 013 */     this.references = references;
/* 014 */     result = new UnsafeRow(1);
/* 015 */     this.holder = new org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder(result,
32);
/* 016 */     this.rowWriter = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(holder,
1);
/* 017 */
/* 018 */   }
/* 019 */
/* 020 */   public void initialize(int partitionIndex) {
/* 021 */
/* 022 */   }
/* 023 */
/* 024 */
/* 025 */
/* 026 */   // Scala.Function1 need this
/* 027 */   public java.lang.Object apply(java.lang.Object row) {
/* 028 */     return apply((InternalRow) row);
/* 029 */   }
/* 030 */
/* 031 */   public UnsafeRow apply(InternalRow i) {
/* 032 */     holder.reset();
/* 033 */
/* 034 */     rowWriter.zeroOutNullBytes();
/* 035 */
/* 036 */
/* 037 */     boolean isNull1 = i.isNullAt(0);
/* 038 */     UTF8String value1 = isNull1 ? null : (i.getUTF8String(0));
/* 039 */     boolean isNull = isNull1;
/* 040 */     UTF8String value = null;
/* 041 */
/* 042 */     if (!isNull1) {
/* 043 */
/* 044 */       value = new Fql_Base64_Encode().evaluate(child.toString());
/* 045 */
/* 046 */     }
/* 047 */     if (isNull) {
/* 048 */       rowWriter.setNullAt(0);
/* 049 */     } else {
/* 050 */       rowWriter.write(0, value);
/* 051 */     }
/* 052 */     result.setTotalSize(holder.totalSize());
/* 053 */     return result;
/* 054 */   }
/* 055 */ }




--
This message was sent by Atlassian JIRA
(v6.3.15#6346)

---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@spark.apache.org
For additional commands, e-mail: issues-help@spark.apache.org


Mime
View raw message