sqoop-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From szvasas <...@git.apache.org>
Subject [GitHub] sqoop pull request #60: SQOOP-3396: Add parquet numeric support for Parquet ...
Date Mon, 03 Dec 2018 16:25:13 GMT
Github user szvasas commented on a diff in the pull request:

    https://github.com/apache/sqoop/pull/60#discussion_r238334653
  
    --- Diff: src/test/org/apache/sqoop/importjob/numerictypes/NumericTypesImportTestBase.java
---
    @@ -65,240 +46,79 @@
      * 2. Decimal padding during avro or parquet import
      * In case of Oracle and Postgres, Sqoop has to pad the values with 0s to avoid errors.
      */
    -public abstract class NumericTypesImportTestBase<T extends AvroTestConfiguration &
ParquetTestConfiguration> extends ImportJobTestCase implements DatabaseAdapterFactory {
    +public abstract class NumericTypesImportTestBase<T extends ImportJobTestConfiguration>
extends ThirdPartyTestBase<T>  {
     
       public static final Log LOG = LogFactory.getLog(NumericTypesImportTestBase.class.getName());
     
    -  private Configuration conf = new Configuration();
    -
    -  private final T configuration;
    -  private final DatabaseAdapter adapter;
       private final boolean failWithoutExtraArgs;
       private final boolean failWithPadding;
     
    -  // Constants for the basic test case, that doesn't use extra arguments
    -  // that are required to avoid errors, i.e. padding and default precision and scale.
    -  protected final static boolean SUCCEED_WITHOUT_EXTRA_ARGS = false;
    -  protected final static boolean FAIL_WITHOUT_EXTRA_ARGS = true;
    -
    -  // Constants for the test case that has padding specified but not default precision
and scale.
    -  protected final static boolean SUCCEED_WITH_PADDING_ONLY = false;
    -  protected final static boolean FAIL_WITH_PADDING_ONLY = true;
    -
    -  private Path tableDirPath;
    -
       public NumericTypesImportTestBase(T configuration, boolean failWithoutExtraArgs, boolean
failWithPaddingOnly) {
    -    this.adapter = createAdapter();
    -    this.configuration = configuration;
    +    super(configuration);
         this.failWithoutExtraArgs = failWithoutExtraArgs;
         this.failWithPadding = failWithPaddingOnly;
       }
     
    -  @Rule
    -  public ExpectedException thrown = ExpectedException.none();
    -
    -  @Override
    -  protected Configuration getConf() {
    -    return conf;
    -  }
    -
    -  @Override
    -  protected boolean useHsqldbTestServer() {
    -    return false;
    -  }
    -
    -  @Override
    -  protected String getConnectString() {
    -    return adapter.getConnectionString();
    -  }
    -
    -  @Override
    -  protected SqoopOptions getSqoopOptions(Configuration conf) {
    -    SqoopOptions opts = new SqoopOptions(conf);
    -    adapter.injectConnectionParameters(opts);
    -    return opts;
    -  }
    -
    -  @Override
    -  protected void dropTableIfExists(String table) throws SQLException {
    -    adapter.dropTableIfExists(table, getManager());
    -  }
    -
       @Before
       public void setUp() {
         super.setUp();
    -    String[] names = configuration.getNames();
    -    String[] types = configuration.getTypes();
    -    createTableWithColTypesAndNames(names, types, new String[0]);
    -    List<String[]> inputData = configuration.getSampleData();
    -    for (String[] input  : inputData) {
    -      insertIntoTable(names, types, input);
    -    }
         tableDirPath = new Path(getWarehouseDir() + "/" + getTableName());
       }
     
    -  @After
    -  public void tearDown() {
    -    try {
    -      dropTableIfExists(getTableName());
    -    } catch (SQLException e) {
    -      LOG.warn("Error trying to drop table on tearDown: " + e);
    -    }
    -    super.tearDown();
    -  }
    +  public Path tableDirPath;
     
    -  private ArgumentArrayBuilder getArgsBuilder(SqoopOptions.FileLayout fileLayout) {
    -    ArgumentArrayBuilder builder = new ArgumentArrayBuilder();
    -    if (AvroDataFile.equals(fileLayout)) {
    -      builder.withOption("as-avrodatafile");
    -    }
    -    else if (ParquetFile.equals(fileLayout)) {
    -      builder.withOption("as-parquetfile");
    -    }
    +  @Rule
    +  public ExpectedException thrown = ExpectedException.none();
    +
    +  abstract public ArgumentArrayBuilder getArgsBuilder();
    +  abstract public void verify();
     
    +  public ArgumentArrayBuilder includeCommonOptions(ArgumentArrayBuilder builder) {
         return builder.withCommonHadoopFlags(true)
             .withOption("warehouse-dir", getWarehouseDir())
             .withOption("num-mappers", "1")
             .withOption("table", getTableName())
             .withOption("connect", getConnectString());
       }
     
    -  /**
    -   * Adds properties to the given arg builder for decimal precision and scale.
    -   * @param builder
    -   */
    -  private void addPrecisionAndScale(ArgumentArrayBuilder builder) {
    -    builder.withProperty("sqoop.avro.logical_types.decimal.default.precision", "38");
    -    builder.withProperty("sqoop.avro.logical_types.decimal.default.scale", "3");
    -  }
    -
    -  /**
    -   * Enables padding for decimals in avro and parquet import.
    -   * @param builder
    -   */
    -  private void addPadding(ArgumentArrayBuilder builder) {
    -    builder.withProperty("sqoop.avro.decimal_padding.enable", "true");
    -  }
    -
    -  private void addEnableAvroDecimal(ArgumentArrayBuilder builder) {
    -    builder.withProperty("sqoop.avro.logical_types.decimal.enable", "true");
    -  }
    -
    -  private void addEnableParquetDecimal(ArgumentArrayBuilder builder) {
    -    builder.withProperty("sqoop.parquet.logical_types.decimal.enable", "true");
    -  }
     
    -  private void configureJunitToExpectFailure(boolean failWithPadding) {
    -    if (failWithPadding) {
    +  @Test
    +  public void testImportWithoutPadding() throws IOException {
    +    if(failWithoutExtraArgs){
           thrown.expect(IOException.class);
           thrown.expectMessage("Failure during job; return status 1");
    +      NumericTypesTestUtils.configureJunitToExpectFailure(thrown);
         }
    -  }
    -
    -  @Test
    -  public void testAvroImportWithoutPadding() throws IOException {
    -    configureJunitToExpectFailure(failWithoutExtraArgs);
    -    ArgumentArrayBuilder builder = getArgsBuilder(AvroDataFile);
    -    addEnableAvroDecimal(builder);
    +    ArgumentArrayBuilder builder = getArgsBuilder();
         String[] args = builder.build();
         runImport(args);
         if (!failWithoutExtraArgs) {
    -      verify(AvroDataFile);
    -    }
    -  }
    -
    -  @Test
    -  public void testAvroImportWithPadding() throws IOException {
    -    configureJunitToExpectFailure(failWithPadding);
    -    ArgumentArrayBuilder builder = getArgsBuilder(AvroDataFile);
    -    addEnableAvroDecimal(builder);
    -    addPadding(builder);
    -    runImport(builder.build());
    -    if (!failWithPadding) {
    -      verify(AvroDataFile);
    +      verify();
         }
       }
     
       @Test
    -  public void testAvroImportWithDefaultPrecisionAndScale() throws  IOException {
    -    ArgumentArrayBuilder builder = getArgsBuilder(AvroDataFile);
    -    addEnableAvroDecimal(builder);
    -    addPadding(builder);
    -    addPrecisionAndScale(builder);
    -    runImport(builder.build());
    -    verify(AvroDataFile);
    -  }
    -
    -  @Test
    -  public void testParquetImportWithoutPadding() throws IOException {
    -    configureJunitToExpectFailure(failWithoutExtraArgs);
    -    ArgumentArrayBuilder builder = getArgsBuilder(ParquetFile);
    -    addEnableParquetDecimal(builder);
    -    String[] args = builder.build();
    -    runImport(args);
    -    if (!failWithoutExtraArgs) {
    -      verify(ParquetFile);
    +  public void testImportWithPadding() throws IOException {
    +    if(failWithPadding){
    +      thrown.expect(IOException.class);
    +      thrown.expectMessage("Failure during job; return status 1");
    +      NumericTypesTestUtils.configureJunitToExpectFailure(thrown);
    --- End diff --
    
    This method seems to be doing the same thing as the previous line.


---

Mime
View raw message