hive-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Zoltan Haindrich (JIRA)" <j...@apache.org>
Subject [jira] [Updated] (HIVE-21980) Parsing time can be high in case of deeply nested subqueries
Date Thu, 25 Jul 2019 18:28:00 GMT

     [ https://issues.apache.org/jira/browse/HIVE-21980?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]

Zoltan Haindrich updated HIVE-21980:
------------------------------------
    Description: 
for queries which are recursively doing:

{code}
SELECT ...
FROM (SELECT ...
             FROM ( [...]
             ) JOIN
            (SELECT ...
            FROM ( [...] )
JOIN ( [...]
)
{code}


when the issue is present hs2 stops right after starting compiling and the following stack
is there
{code}
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.subQuerySource(HiveParser_FromClauseParser.java:4745)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.synpred1_FromClauseParser_fragment(HiveParser_FromClauseParser.java:6513)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.synpred1_FromClauseParser(HiveParser_FromClauseParser.java:6660)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser$DFA10.specialStateTransition(HiveParser_FromClauseParser.java:6942)
	at org.antlr.runtime.DFA.predict(DFA.java:80)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.atomjoinSource(HiveParser_FromClauseParser.java:1596)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.joinSource(HiveParser_FromClauseParser.java:1859)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.fromSource(HiveParser_FromClauseParser.java:1532)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.fromClause(HiveParser_FromClauseParser.java:1375)
	at org.apache.hadoop.hive.ql.parse.HiveParser.fromClause(HiveParser.java:45755)
	at org.apache.hadoop.hive.ql.parse.HiveParser.atomSelectStatement(HiveParser.java:40337)
	at org.apache.hadoop.hive.ql.parse.HiveParser.selectStatement(HiveParser.java:40589)
	at org.apache.hadoop.hive.ql.parse.HiveParser.regularBody(HiveParser.java:40235)
	at org.apache.hadoop.hive.ql.parse.HiveParser.queryStatementExpressionBody(HiveParser.java:39445)
	at org.apache.hadoop.hive.ql.parse.HiveParser.queryStatementExpression(HiveParser.java:39333)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.subQuerySource(HiveParser_FromClauseParser.java:4745)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.atomjoinSource(HiveParser_FromClauseParser.java:1699)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.joinSource(HiveParser_FromClauseParser.java:1859)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.fromSource(HiveParser_FromClauseParser.java:1532)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.fromClause(HiveParser_FromClauseParser.java:1375)
	at org.apache.hadoop.hive.ql.parse.HiveParser.fromClause(HiveParser.java:45755)
	at org.apache.hadoop.hive.ql.parse.HiveParser.atomSelectStatement(HiveParser.java:40337)
	at org.apache.hadoop.hive.ql.parse.HiveParser.selectStatement(HiveParser.java:40589)
	at org.apache.hadoop.hive.ql.parse.HiveParser.regularBody(HiveParser.java:40235)
	at org.apache.hadoop.hive.ql.parse.HiveParser.queryStatementExpressionBody(HiveParser.java:39445)
	at org.apache.hadoop.hive.ql.parse.HiveParser.queryStatementExpression(HiveParser.java:39333)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.subQuerySource(HiveParser_FromClauseParser.java:4745)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.synpred1_FromClauseParser_fragment(HiveParser_FromClauseParser.java:6513)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.synpred1_FromClauseParser(HiveParser_FromClauseParser.java:6660)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser$DFA10.specialStateTransition(HiveParser_FromClauseParser.java:6942)
	at org.antlr.runtime.DFA.predict(DFA.java:80)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.atomjoinSource(HiveParser_FromClauseParser.java:1596)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.joinSource(HiveParser_FromClauseParser.java:1859)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.fromSource(HiveParser_FromClauseParser.java:1532)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.fromClause(HiveParser_FromClauseParser.java:1375)
	at org.apache.hadoop.hive.ql.parse.HiveParser.fromClause(HiveParser.java:45755)
	at org.apache.hadoop.hive.ql.parse.HiveParser.atomSelectStatement(HiveParser.java:40337)
	at org.apache.hadoop.hive.ql.parse.HiveParser.selectStatement(HiveParser.java:40589)
	at org.apache.hadoop.hive.ql.parse.HiveParser.regularBody(HiveParser.java:40235)
	at org.apache.hadoop.hive.ql.parse.HiveParser.queryStatementExpressionBody(HiveParser.java:39445)
	at org.apache.hadoop.hive.ql.parse.HiveParser.queryStatementExpression(HiveParser.java:39333)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.subQuerySource(HiveParser_FromClauseParser.java:4745)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.atomjoinSource(HiveParser_FromClauseParser.java:1699)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.joinSource(HiveParser_FromClauseParser.java:1859)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.fromSource(HiveParser_FromClauseParser.java:1532)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.fromClause(HiveParser_FromClauseParser.java:1375)
	at org.apache.hadoop.hive.ql.parse.HiveParser.fromClause(HiveParser.java:45755)
	at org.apache.hadoop.hive.ql.parse.HiveParser.atomSelectStatement(HiveParser.java:40337)
	at org.apache.hadoop.hive.ql.parse.HiveParser.selectStatement(HiveParser.java:40589)
	at org.apache.hadoop.hive.ql.parse.HiveParser.regularBody(HiveParser.java:40235)
	at org.apache.hadoop.hive.ql.parse.HiveParser.queryStatementExpressionBody(HiveParser.java:39445)
	at org.apache.hadoop.hive.ql.parse.HiveParser.queryStatementExpression(HiveParser.java:39333)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.subQuerySource(HiveParser_FromClauseParser.java:4745)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.atomjoinSource(HiveParser_FromClauseParser.java:1699)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.joinSource(HiveParser_FromClauseParser.java:1859)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.fromSource(HiveParser_FromClauseParser.java:1532)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.fromClause(HiveParser_FromClauseParser.java:1375)
	at org.apache.hadoop.hive.ql.parse.HiveParser.fromClause(HiveParser.java:45755)
	at org.apache.hadoop.hive.ql.parse.HiveParser.atomSelectStatement(HiveParser.java:40337)
	at org.apache.hadoop.hive.ql.parse.HiveParser.selectStatement(HiveParser.java:40589)
	at org.apache.hadoop.hive.ql.parse.HiveParser.regularBody(HiveParser.java:40235)
	at org.apache.hadoop.hive.ql.parse.HiveParser.queryStatementExpressionBody(HiveParser.java:39445)
	at org.apache.hadoop.hive.ql.parse.HiveParser.queryStatementExpression(HiveParser.java:39333)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.subQuerySource(HiveParser_FromClauseParser.java:4745)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.synpred1_FromClauseParser_fragment(HiveParser_FromClauseParser.java:6513)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.synpred1_FromClauseParser(HiveParser_FromClauseParser.java:6660)
	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser$DFA10.specialStateTransition(HiveParser_FromClauseParser.java:6942)
	at org.antlr.runtime.DFA.predict(DFA.java:80)
{code}


  was:
for queries which are recursively doing:

{code}
SELECT ...
FROM (SELECT ...
             FROM ( [...]
             ) JOIN
            (SELECT ...
            FROM ( [...] )
JOIN ( [...]
)
{code}


> Parsing time can be high in case of deeply nested subqueries
> ------------------------------------------------------------
>
>                 Key: HIVE-21980
>                 URL: https://issues.apache.org/jira/browse/HIVE-21980
>             Project: Hive
>          Issue Type: Improvement
>          Components: Parser
>            Reporter: Zoltan Haindrich
>            Assignee: Zoltan Haindrich
>            Priority: Major
>             Fix For: 4.0.0
>
>         Attachments: HIVE-21980.01.patch, HIVE-21980.01.patch, HIVE-21980.02.patch, HIVE-21980.03.patch
>
>
> for queries which are recursively doing:
> {code}
> SELECT ...
> FROM (SELECT ...
>              FROM ( [...]
>              ) JOIN
>             (SELECT ...
>             FROM ( [...] )
> JOIN ( [...]
> )
> {code}
> when the issue is present hs2 stops right after starting compiling and the following
stack is there
> {code}
> 	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.subQuerySource(HiveParser_FromClauseParser.java:4745)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.synpred1_FromClauseParser_fragment(HiveParser_FromClauseParser.java:6513)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.synpred1_FromClauseParser(HiveParser_FromClauseParser.java:6660)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser$DFA10.specialStateTransition(HiveParser_FromClauseParser.java:6942)
> 	at org.antlr.runtime.DFA.predict(DFA.java:80)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.atomjoinSource(HiveParser_FromClauseParser.java:1596)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.joinSource(HiveParser_FromClauseParser.java:1859)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.fromSource(HiveParser_FromClauseParser.java:1532)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.fromClause(HiveParser_FromClauseParser.java:1375)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser.fromClause(HiveParser.java:45755)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser.atomSelectStatement(HiveParser.java:40337)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser.selectStatement(HiveParser.java:40589)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser.regularBody(HiveParser.java:40235)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser.queryStatementExpressionBody(HiveParser.java:39445)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser.queryStatementExpression(HiveParser.java:39333)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.subQuerySource(HiveParser_FromClauseParser.java:4745)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.atomjoinSource(HiveParser_FromClauseParser.java:1699)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.joinSource(HiveParser_FromClauseParser.java:1859)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.fromSource(HiveParser_FromClauseParser.java:1532)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.fromClause(HiveParser_FromClauseParser.java:1375)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser.fromClause(HiveParser.java:45755)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser.atomSelectStatement(HiveParser.java:40337)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser.selectStatement(HiveParser.java:40589)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser.regularBody(HiveParser.java:40235)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser.queryStatementExpressionBody(HiveParser.java:39445)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser.queryStatementExpression(HiveParser.java:39333)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.subQuerySource(HiveParser_FromClauseParser.java:4745)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.synpred1_FromClauseParser_fragment(HiveParser_FromClauseParser.java:6513)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.synpred1_FromClauseParser(HiveParser_FromClauseParser.java:6660)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser$DFA10.specialStateTransition(HiveParser_FromClauseParser.java:6942)
> 	at org.antlr.runtime.DFA.predict(DFA.java:80)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.atomjoinSource(HiveParser_FromClauseParser.java:1596)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.joinSource(HiveParser_FromClauseParser.java:1859)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.fromSource(HiveParser_FromClauseParser.java:1532)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.fromClause(HiveParser_FromClauseParser.java:1375)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser.fromClause(HiveParser.java:45755)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser.atomSelectStatement(HiveParser.java:40337)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser.selectStatement(HiveParser.java:40589)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser.regularBody(HiveParser.java:40235)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser.queryStatementExpressionBody(HiveParser.java:39445)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser.queryStatementExpression(HiveParser.java:39333)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.subQuerySource(HiveParser_FromClauseParser.java:4745)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.atomjoinSource(HiveParser_FromClauseParser.java:1699)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.joinSource(HiveParser_FromClauseParser.java:1859)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.fromSource(HiveParser_FromClauseParser.java:1532)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.fromClause(HiveParser_FromClauseParser.java:1375)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser.fromClause(HiveParser.java:45755)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser.atomSelectStatement(HiveParser.java:40337)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser.selectStatement(HiveParser.java:40589)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser.regularBody(HiveParser.java:40235)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser.queryStatementExpressionBody(HiveParser.java:39445)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser.queryStatementExpression(HiveParser.java:39333)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.subQuerySource(HiveParser_FromClauseParser.java:4745)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.atomjoinSource(HiveParser_FromClauseParser.java:1699)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.joinSource(HiveParser_FromClauseParser.java:1859)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.fromSource(HiveParser_FromClauseParser.java:1532)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.fromClause(HiveParser_FromClauseParser.java:1375)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser.fromClause(HiveParser.java:45755)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser.atomSelectStatement(HiveParser.java:40337)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser.selectStatement(HiveParser.java:40589)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser.regularBody(HiveParser.java:40235)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser.queryStatementExpressionBody(HiveParser.java:39445)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser.queryStatementExpression(HiveParser.java:39333)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.subQuerySource(HiveParser_FromClauseParser.java:4745)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.synpred1_FromClauseParser_fragment(HiveParser_FromClauseParser.java:6513)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser.synpred1_FromClauseParser(HiveParser_FromClauseParser.java:6660)
> 	at org.apache.hadoop.hive.ql.parse.HiveParser_FromClauseParser$DFA10.specialStateTransition(HiveParser_FromClauseParser.java:6942)
> 	at org.antlr.runtime.DFA.predict(DFA.java:80)
> {code}



--
This message was sent by Atlassian JIRA
(v7.6.14#76016)

Mime
View raw message