chukwa-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ey...@apache.org
Subject chukwa git commit: CHUKWA-784. Improve CharFileTailingAdaptorUTF8NewLineEscaped and LocalWriter logic to send proper data chunk. (Eric Yang)
Date Sun, 11 Oct 2015 19:43:40 GMT
Repository: chukwa
Updated Branches:
  refs/heads/master a990fc54a -> 93a1f75c4


CHUKWA-784. Improve CharFileTailingAdaptorUTF8NewLineEscaped and LocalWriter
                logic to send proper data chunk. (Eric Yang)


Project: http://git-wip-us.apache.org/repos/asf/chukwa/repo
Commit: http://git-wip-us.apache.org/repos/asf/chukwa/commit/93a1f75c
Tree: http://git-wip-us.apache.org/repos/asf/chukwa/tree/93a1f75c
Diff: http://git-wip-us.apache.org/repos/asf/chukwa/diff/93a1f75c

Branch: refs/heads/master
Commit: 93a1f75c45f7333dcba2980f5ec9ebbdc64bb9d3
Parents: a990fc5
Author: Eric Yang <eyang@apache.org>
Authored: Sun Oct 11 12:42:48 2015 -0700
Committer: Eric Yang <eyang@apache.org>
Committed: Sun Oct 11 12:43:31 2015 -0700

----------------------------------------------------------------------
 .../writer/parquet/ChukwaAvroSchema.java        |  17 +++
 .../writer/TestChukwaWriters.java               | 122 -------------------
 2 files changed, 17 insertions(+), 122 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/chukwa/blob/93a1f75c/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/parquet/ChukwaAvroSchema.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/parquet/ChukwaAvroSchema.java
b/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/parquet/ChukwaAvroSchema.java
index eaad1bc..cc113d3 100644
--- a/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/parquet/ChukwaAvroSchema.java
+++ b/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/parquet/ChukwaAvroSchema.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package org.apache.hadoop.chukwa.datacollection.writer.parquet;
 
 import org.apache.avro.Schema;

http://git-wip-us.apache.org/repos/asf/chukwa/blob/93a1f75c/src/test/java/org/apache/hadoop/chukwa/datacollection/writer/TestChukwaWriters.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/chukwa/datacollection/writer/TestChukwaWriters.java
b/src/test/java/org/apache/hadoop/chukwa/datacollection/writer/TestChukwaWriters.java
index 8d0dd46..28a48d7 100644
--- a/src/test/java/org/apache/hadoop/chukwa/datacollection/writer/TestChukwaWriters.java
+++ b/src/test/java/org/apache/hadoop/chukwa/datacollection/writer/TestChukwaWriters.java
@@ -174,128 +174,6 @@ public class TestChukwaWriters extends TestCase{
     return null;    
   }
 
-//  /**
-//   * Test to check if the .chukwa files are closing at the time we expect them
-//   * to close. This test sets the rotateInterval and offsetInterval to small
-//   * values, reads the filename of the first .chukwa file, extracts the
-//   * timestamp from its name, calculates the timestamp when the next .chukwa
-//   * file should be closed, sleeps for some time (enough for producing the next
-//   * .chukwa file), reads the timestamp on the second .chukwa file, and
-//   * compares the expected close timestamp with the actual closing timestamp of
-//   * the second file.
-//   */
-//  public void testParquetWriterFixedCloseInterval() {
-//    try {
-//      long rotateInterval = 10000;
-//      long intervalOffset = 3000;
-//
-//      File tempDir = new File(System.getProperty("test.build.data", "/tmp"));
-//      if (!tempDir.exists()) {
-//        tempDir.mkdirs();
-//      }
-//
-//      String outputDirectory = tempDir.getPath() + "/testChukwaWriters_testChukwaParquetWriterFixedCloseInterval_"
+
-//              System.currentTimeMillis() + "/";
-//      String parquetWriterOutputDir = outputDirectory +"/parquetWriter/parquetOutputDir";
-//
-//      File directory = new File(parquetWriterOutputDir);
-//      // if some files already exist in this directory then delete them. Files
-//      // may exist due to an old test run.
-//      File[] files = directory.listFiles();
-//      if (files != null) {
-//        for(File file: files) {
-//          file.delete();
-//        }
-//      }
-//
-//      Configuration confParquetWriter = new Configuration();
-//      confParquetWriter.set("chukwaCollector.rotateInterval", String.valueOf(rotateInterval));
-//      confParquetWriter.set("writer.hdfs.filesystem", "file:///");
-//      confParquetWriter.set(ChukwaParquetWriter.OUTPUT_DIR_OPT, parquetWriterOutputDir
);
-//      confParquetWriter.set("chukwaCollector.isFixedTimeRotatorScheme", "true");
-//      confParquetWriter.set("chukwaCollector.fixedTimeIntervalOffset", String.valueOf(intervalOffset));
-//
-//      ChukwaWriter parquetWriter = new ChukwaParquetWriter(confParquetWriter);
-//
-//      // we do not want our test to fail due to a lag in calling the
-//      // scheduleNextRotation() method and creating of first .chukwa file.
-//      // So, we will make sure that the rotation starts in the middle (approx)
-//      // of the rotateInterval
-//      long currentTime = System.currentTimeMillis();
-//      long currentTimeInSec = currentTime/1000;
-//      long timeAfterPrevRotateInterval = currentTimeInSec % rotateInterval;
-//      if(timeAfterPrevRotateInterval > (rotateInterval - 2)){
-//        Thread.sleep(2000);
-//      }
-//
-//      String [] fileNames = directory.list();
-//      String firstFileName = "";
-//      String initialTimestamp = "";
-//      // extracting the close time of first .chukwa file. This timestamp can be
-//      // extracted from the file name. An example filename is
-//      // 20110531122600002_<host-name>_5f836ece1302899d9a0727e.chukwa
-//      for(String file: fileNames) {
-//        if ( file.endsWith(".chukwa") ){
-//          // set a flag so that later we can identify that this file has been
-//          // visited
-//          firstFileName = file;
-//          // getting just the timestamp part i.e. 20110531122600002 in the
-//          // example filename mentioned in the above comment
-//          initialTimestamp = file.split("_")[0];
-//          // stripping off the millisecond part of timestamp. The timestamp
-//          // now becomes 20110531122600
-//          initialTimestamp = initialTimestamp.substring(0, initialTimestamp.length()-3);
-//          break;
-//        }
-//      }
-//
-//      SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMddhhmmss");
-//      Date initialDate = formatter.parse(initialTimestamp);
-//	    long initialDateInMillis = initialDate.getTime();
-//
-//      // calculate the expected close time of the next .chukwa file.
-//      long prevRoundedInterval = initialDateInMillis - (initialDateInMillis %
-//              rotateInterval);
-//      long expectedNextCloseDate = prevRoundedInterval +
-//              rotateInterval + intervalOffset;
-//
-//      // sleep for a time interval equal to (rotateInterval + offsetInterval).
-//      // Only one more .chukwa file will be will be produced in this time
-//      // interval.
-//      long sleepTime = rotateInterval + intervalOffset;
-//
-//      Thread.sleep(sleepTime);
-//      fileNames = directory.list();
-//      String nextTimestamp = "";
-//      // extract the timestamp of the second .chukwa file
-//      for(String file: fileNames) {
-//        if ( file.endsWith(".chukwa") && !file.equals(firstFileName)){
-//          nextTimestamp = file.split("_")[0];
-//          nextTimestamp = nextTimestamp.substring(0, nextTimestamp.length()-3);
-//          break;
-//        }
-//      }
-//
-//      Date nextDate = formatter.parse(nextTimestamp);
-//      System.out.println("initialTimestamp:"+nextTimestamp);
-//      long nextDateInMillis = nextDate.getTime();
-//
-//      long threshold = 500; //milliseconds
-//
-//      // test will be successful only if the timestamp on the second .chukwa
-//      // file is very close (differs by < 500 ms) to the expected closing
-//      // timestamp we calculated.
-//      Assert.assertTrue("File not closed at expected time",
-//              (nextDateInMillis - expectedNextCloseDate < threshold));
-//      parquetWriter.close();
-//
-//    } catch (Throwable e) {
-//      e.printStackTrace();
-//      Assert.fail("Exception in TestChukwaWriters - " +
-//              "testParquetFileFixedCloseInterval()," + e.getMessage());
-//    }
-//}
-
   /**
    * Test to check the calculation of the delay interval for rotation in
    * ParquetFileWriter. It uses an array of known currentTimestamps and their


Mime
View raw message