sqoop-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "ASF GitHub Bot (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (SQOOP-3396) Add parquet numeric support for Parquet in Hive import
Date Thu, 29 Nov 2018 23:43:00 GMT

    [ https://issues.apache.org/jira/browse/SQOOP-3396?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16704039#comment-16704039
] 

ASF GitHub Bot commented on SQOOP-3396:
---------------------------------------

Github user maugly24 commented on a diff in the pull request:

    https://github.com/apache/sqoop/pull/60#discussion_r237698797
  
    --- Diff: src/test/org/apache/sqoop/hive/numerictypes/NumericTypesHiveImportTest.java
---
    @@ -0,0 +1,202 @@
    +/**
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements.  See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership.  The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License.  You may obtain a copy of the License at
    + * <p>
    + * http://www.apache.org/licenses/LICENSE-2.0
    + * <p>
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +
    +package org.apache.sqoop.hive.numerictypes;
    +
    +import org.apache.sqoop.hive.minicluster.HiveMiniCluster;
    +import org.apache.sqoop.hive.minicluster.NoAuthenticationConfiguration;
    +import org.apache.sqoop.importjob.configuration.HiveTestConfiguration;
    +import org.apache.sqoop.importjob.configuration.MysqlImportJobTestConfiguration;
    +import org.apache.sqoop.importjob.configuration.OracleImportJobTestConfiguration;
    +import org.apache.sqoop.importjob.configuration.OracleImportJobTestConfigurationForNumber;
    +import org.apache.sqoop.importjob.configuration.PostgresqlImportJobTestConfigurationForNumeric;
    +import org.apache.sqoop.importjob.configuration.PostgresqlImportJobTestConfigurationPaddingShouldSucceed;
    +import org.apache.sqoop.importjob.configuration.SqlServerImportJobTestConfiguration;
    +import org.apache.sqoop.testcategories.sqooptest.IntegrationTest;
    +import org.apache.sqoop.testcategories.thirdpartytest.MysqlTest;
    +import org.apache.sqoop.testcategories.thirdpartytest.OracleTest;
    +import org.apache.sqoop.testcategories.thirdpartytest.PostgresqlTest;
    +import org.apache.sqoop.testcategories.thirdpartytest.SqlServerTest;
    +import org.apache.sqoop.testutil.HiveServer2TestUtil;
    +import org.apache.sqoop.testutil.NumericTypesTestUtils;
    +import org.apache.sqoop.testutil.adapter.DatabaseAdapter;
    +import org.apache.sqoop.testutil.adapter.MysqlDatabaseAdapter;
    +import org.apache.sqoop.testutil.adapter.OracleDatabaseAdapter;
    +import org.apache.sqoop.testutil.adapter.PostgresDatabaseAdapter;
    +import org.apache.sqoop.testutil.adapter.SqlServerDatabaseAdapter;
    +import org.apache.sqoop.util.BlockJUnit4ClassRunnerWithParametersFactory;
    +import org.junit.AfterClass;
    +import org.junit.BeforeClass;
    +import org.junit.Rule;
    +import org.junit.experimental.categories.Category;
    +import org.junit.experimental.runners.Enclosed;
    +import org.junit.rules.ExpectedException;
    +import org.junit.runner.RunWith;
    +import org.junit.runners.Parameterized;
    +
    +import java.util.Arrays;
    +
    +import static org.apache.sqoop.testutil.NumericTypesTestUtils.FAIL_WITHOUT_EXTRA_ARGS;
    +import static org.apache.sqoop.testutil.NumericTypesTestUtils.FAIL_WITH_PADDING_ONLY;
    +import static org.apache.sqoop.testutil.NumericTypesTestUtils.SUCCEED_WITHOUT_EXTRA_ARGS;
    +import static org.apache.sqoop.testutil.NumericTypesTestUtils.SUCCEED_WITH_PADDING_ONLY;
    +
    +@RunWith(Enclosed.class)
    +@Category(IntegrationTest.class)
    +public class NumericTypesHiveImportTest {
    +
    +  @Rule
    +  public ExpectedException expectedException = ExpectedException.none();
    +
    +  private static HiveMiniCluster hiveMiniCluster;
    +
    +  private static HiveServer2TestUtil hiveServer2TestUtil;
    +
    +  @BeforeClass
    +  public static void beforeClass() {
    +    startHiveMiniCluster();
    +  }
    +
    +  @AfterClass
    +  public static void afterClass() {
    +    stopHiveMiniCluster();
    +  }
    +
    +  public static void startHiveMiniCluster() {
    +    hiveMiniCluster = new HiveMiniCluster(new NoAuthenticationConfiguration());
    +    hiveMiniCluster.start();
    +    hiveServer2TestUtil = new HiveServer2TestUtil(hiveMiniCluster.getUrl());
    +  }
    +
    +  public static void stopHiveMiniCluster() {
    +    hiveMiniCluster.stop();
    +  }
    +
    +  @Category(MysqlTest.class)
    +  public static class MysqlNumericTypesHiveImportTest extends NumericTypesHiveImportTestBase
{
    +
    +    public MysqlNumericTypesHiveImportTest() {
    +      super(new MysqlImportJobTestConfiguration(), NumericTypesTestUtils.SUCCEED_WITHOUT_EXTRA_ARGS,
NumericTypesTestUtils.SUCCEED_WITH_PADDING_ONLY);
    +    }
    +
    +    @Override
    +    public DatabaseAdapter createAdapter() {
    +      return new MysqlDatabaseAdapter();
    +    }
    +
    +    @Override
    +    protected HiveMiniCluster getHiveMiniCluster() {
    --- End diff --
    
    I wold move getHiveMiniCluster and getHiveServer2TestUtil to the base class, with the
concrete implementation, and just would pass the references in the super constructor call.
Otherwise you'll have to keep this duplication on all 4 underlying implementation (MySQL,
Postgres, MsSQL, Oracle), which sounds quite bad.


> Add parquet numeric support for Parquet in Hive import
> ------------------------------------------------------
>
>                 Key: SQOOP-3396
>                 URL: https://issues.apache.org/jira/browse/SQOOP-3396
>             Project: Sqoop
>          Issue Type: Sub-task
>            Reporter: Fero Szabo
>            Assignee: Fero Szabo
>            Priority: Major
>




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

Mime
View raw message