Repository: oozie
Updated Branches:
refs/heads/master 800ea758e -> 0601fc228
OOZIE-3038 Make all Oozie JUnit tests pass on dist_test (andras.piros via gezapeti)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/0601fc22
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/0601fc22
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/0601fc22
Branch: refs/heads/master
Commit: 0601fc228ce2d5a9610efd7644e5adab611f8a64
Parents: 800ea75
Author: Gezapeti Cseh <gezapeti@gmail.com>
Authored: Mon Aug 28 17:23:15 2017 +0200
Committer: Gezapeti Cseh <gezapeti@gmail.com>
Committed: Mon Aug 28 17:23:18 2017 +0200
----------------------------------------------------------------------
.../oozie/action/ssh/TestSshActionExecutor.java | 30 +++++++
.../apache/oozie/servlet/TestAdminServlet.java | 5 +-
.../oozie/servlet/TestV1AdminServlet.java | 5 +-
.../java/org/apache/oozie/test/XTestCase.java | 28 ++++--
.../apache/oozie/util/TestGraphGenerator.java | 95 ++++++--------------
release-log.txt | 1 +
6 files changed, 87 insertions(+), 77 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/0601fc22/core/src/test/java/org/apache/oozie/action/ssh/TestSshActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/ssh/TestSshActionExecutor.java b/core/src/test/java/org/apache/oozie/action/ssh/TestSshActionExecutor.java
index 16dd29a..00daa77 100644
--- a/core/src/test/java/org/apache/oozie/action/ssh/TestSshActionExecutor.java
+++ b/core/src/test/java/org/apache/oozie/action/ssh/TestSshActionExecutor.java
@@ -159,6 +159,10 @@ public class TestSshActionExecutor extends XFsTestCase {
@Override
protected void setUp() throws Exception {
+ if (!isSshPresent()) {
+ return;
+ }
+
super.setUp();
services = new Services();
services.init();
@@ -170,6 +174,19 @@ public class TestSshActionExecutor extends XFsTestCase {
fs.delete(path, true);
}
+ private boolean isSshPresent() {
+ int exitValue;
+ try {
+ final Process process = Runtime.getRuntime().exec("ssh -V");
+ process.waitFor();
+ exitValue = process.exitValue();
+ } catch (final IOException | InterruptedException e) {
+ exitValue = 1;
+ }
+
+ return exitValue == 0;
+ }
+
protected String getActionXMLSchema() {
return "uri:oozie-workflow:0.1";
}
@@ -647,7 +664,20 @@ public class TestSshActionExecutor extends XFsTestCase {
@Override
protected void tearDown() throws Exception {
+ if (!isSshPresent()) {
+ return;
+ }
+
services.destroy();
super.tearDown();
}
+
+ @Override
+ protected void runTest() throws Throwable {
+ if (!isSshPresent()) {
+ return;
+ }
+
+ super.runTest();
+ }
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/0601fc22/core/src/test/java/org/apache/oozie/servlet/TestAdminServlet.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/servlet/TestAdminServlet.java b/core/src/test/java/org/apache/oozie/servlet/TestAdminServlet.java
index 595cb26..0d7b362 100644
--- a/core/src/test/java/org/apache/oozie/servlet/TestAdminServlet.java
+++ b/core/src/test/java/org/apache/oozie/servlet/TestAdminServlet.java
@@ -71,7 +71,10 @@ public class TestAdminServlet extends DagServletTestCase {
assertEquals(HttpServletResponse.SC_OK, conn.getResponseCode());
assertTrue(conn.getHeaderField("content-type").startsWith(RestConstants.JSON_CONTENT_TYPE));
JSONObject json = (JSONObject) JSONValue.parse(new InputStreamReader(conn.getInputStream()));
- assertTrue(json.containsKey(Shell.WINDOWS ? "USERNAME" : "USER"));
+ assertTrue("USERNAME, USER or HOME property not found in json",
+ json.containsKey(Shell.WINDOWS ? "USERNAME" : "USER") ||
+ json.containsKey("HOME")
+ );
return null;
}
});
http://git-wip-us.apache.org/repos/asf/oozie/blob/0601fc22/core/src/test/java/org/apache/oozie/servlet/TestV1AdminServlet.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/servlet/TestV1AdminServlet.java b/core/src/test/java/org/apache/oozie/servlet/TestV1AdminServlet.java
index 1e66537..0113751 100644
--- a/core/src/test/java/org/apache/oozie/servlet/TestV1AdminServlet.java
+++ b/core/src/test/java/org/apache/oozie/servlet/TestV1AdminServlet.java
@@ -75,7 +75,10 @@ public class TestV1AdminServlet extends DagServletTestCase {
assertEquals(HttpServletResponse.SC_OK, conn.getResponseCode());
assertTrue(conn.getHeaderField("content-type").startsWith(RestConstants.JSON_CONTENT_TYPE));
JSONObject json = (JSONObject) JSONValue.parse(new InputStreamReader(conn.getInputStream()));
- assertTrue(json.containsKey(Shell.WINDOWS ? "USERNAME" : "USER"));
+ assertTrue("USERNAME, USER or HOME property not found in json",
+ json.containsKey(Shell.WINDOWS ? "USERNAME" : "USER") ||
+ json.containsKey("HOME")
+ );
return null;
}
});
http://git-wip-us.apache.org/repos/asf/oozie/blob/0601fc22/core/src/test/java/org/apache/oozie/test/XTestCase.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/test/XTestCase.java b/core/src/test/java/org/apache/oozie/test/XTestCase.java
index 50e2452..584aa12 100644
--- a/core/src/test/java/org/apache/oozie/test/XTestCase.java
+++ b/core/src/test/java/org/apache/oozie/test/XTestCase.java
@@ -336,10 +336,11 @@ public abstract class XTestCase extends TestCase {
testCaseConfDir = createTestCaseSubDir("conf");
// load test Oozie site
- String oozieTestDB = System.getProperty("oozie.test.db", "hsqldb");
- String defaultOozieSize =
- new File(OOZIE_SRC_DIR, "core/src/test/resources/" + oozieTestDB + "-oozie-site.xml").getAbsolutePath();
- String customOozieSite = System.getProperty("oozie.test.config.file", defaultOozieSize);
+ final String oozieTestDB = System.getProperty("oozie.test.db", "hsqldb");
+ final String oozieSiteFileName = oozieTestDB + "-oozie-site.xml";
+ final String defaultOozieSite =
+ new File(OOZIE_SRC_DIR, "core/src/test/resources/" + oozieSiteFileName).getAbsolutePath();
+ final String customOozieSite = System.getProperty("oozie.test.config.file", defaultOozieSite);
File source = new File(customOozieSite);
if(!source.isAbsolute()) {
source = new File(OOZIE_SRC_DIR, customOozieSite);
@@ -347,11 +348,28 @@ public abstract class XTestCase extends TestCase {
source = source.getAbsoluteFile();
InputStream oozieSiteSourceStream = null;
if (source.exists()) {
+ log.info("Reading Oozie test resource from file. [source.name={0}]", source.getName());
oozieSiteSourceStream = new FileInputStream(source);
}
else {
// If we can't find it, try using the class loader (useful if we're using XTestCase
from outside core)
- URL sourceURL = getClass().getClassLoader().getResource(oozieTestDB + "-oozie-site.xml");
+ log.info("Oozie test resource file doesn't exist. [source.name={0}]", source.getName());
+ final String testResourceName;
+ if (customOozieSite.lastIndexOf(Path.SEPARATOR) > -1) {
+ final String customOozieSiteFileName = customOozieSite.substring(customOozieSite.lastIndexOf(Path.SEPARATOR)
+ 1);
+ if (customOozieSiteFileName.equals(oozieSiteFileName)) {
+ testResourceName = oozieSiteFileName;
+ }
+ else {
+ testResourceName = customOozieSiteFileName;
+ }
+ }
+ else {
+ testResourceName = oozieSiteFileName;
+ }
+ log.info("Reading Oozie test resource from classpath. [testResourceName={0};source.name={1}]",
+ testResourceName, source.getName());
+ final URL sourceURL = getClass().getClassLoader().getResource(testResourceName);
if (sourceURL != null) {
oozieSiteSourceStream = sourceURL.openStream();
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/0601fc22/core/src/test/java/org/apache/oozie/util/TestGraphGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/util/TestGraphGenerator.java b/core/src/test/java/org/apache/oozie/util/TestGraphGenerator.java
index a92d468..002e925 100644
--- a/core/src/test/java/org/apache/oozie/util/TestGraphGenerator.java
+++ b/core/src/test/java/org/apache/oozie/util/TestGraphGenerator.java
@@ -27,9 +27,6 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
-import java.nio.MappedByteBuffer;
-import java.nio.channels.FileChannel;
-import java.nio.charset.Charset;
public class TestGraphGenerator extends XTestCase {
@@ -37,18 +34,18 @@ public class TestGraphGenerator extends XTestCase {
try {
new GraphGenerator(null, null);
}
- catch (IllegalArgumentException iae) {
+ catch (final IllegalArgumentException iae) {
Assert.assertTrue("Construction with illegal args failed as expected: " + iae.getMessage(),
true);
}
try {
new GraphGenerator("<workflow></workflow>", null);
}
- catch (IllegalArgumentException iae) {
+ catch (final IllegalArgumentException iae) {
Assert.assertTrue("Construction with illegal args failed as expected: " + iae.getMessage(),
true);
}
Assert.assertNotNull(new GraphGenerator("<workflow></workflow>", new
WorkflowJobBean()));
Assert.assertNotNull(new GraphGenerator(null, new WorkflowJobBean()));
- WorkflowJobBean jsonWFJob = new WorkflowJobBean();
+ final WorkflowJobBean jsonWFJob = new WorkflowJobBean();
jsonWFJob.setAppName("My Test App");
jsonWFJob.setId("My Test ID");
Assert.assertNotNull(new GraphGenerator("<workflow></workflow>", jsonWFJob));
@@ -57,94 +54,52 @@ public class TestGraphGenerator extends XTestCase {
}
public void testWrite() {
- WorkflowJobBean jsonWFJob = new WorkflowJobBean();
+ final WorkflowJobBean jsonWFJob = new WorkflowJobBean();
jsonWFJob.setAppName("My Test App");
jsonWFJob.setId("My Test ID");
- String png1 = "src/test/resources/tmp1.png";
- String png2 = "src/test/resources/tmp2.png";
- try {
- GraphGenerator g = new GraphGenerator(readFile("src/test/resources/graphWF.xml"),
jsonWFJob);
- g.write(new FileOutputStream(new File(png1)));
- }
- catch (Exception e) {
- Assert.fail("Write PNG failed for graphWF.xml: " + e.getMessage());
- }
+ generateAndAssertPng(jsonWFJob, "graphWF.xml", false);
- File f1 = new File(png1);
- try {
- // Check if a valid file was written
- Assert.assertNotNull(ImageIO.read(f1));
- }
- catch (IOException io) {
- Assert.fail("Not a valid PNG: " + io.getMessage());
- }
+ generateAndAssertPng(jsonWFJob, "graphWF.xml", true);
try {
- GraphGenerator g = new GraphGenerator(readFile("src/test/resources/graphWF.xml"),
jsonWFJob, true);
- g.write(new FileOutputStream(new File(png2)));
+ final String content = IOUtils.getResourceAsString("invalidGraphWF.xml", -1);
+ final GraphGenerator g = new GraphGenerator(content, jsonWFJob, true);
+ g.write(new org.apache.hadoop.io.IOUtils.NullOutputStream());
}
- catch (Exception e) {
- Assert.fail("Write PNG failed for graphWF.xml: " + e.getMessage());
- }
-
- File f2 = new File(png2);
- try {
- // Check if a valid file was written
- Assert.assertNotNull(ImageIO.read(f2));
- }
- catch (IOException io) {
- Assert.fail("Not a valid PNG: " + io.getMessage());
+ catch (final Exception e) {
+ Assert.fail("Write PNG failed for invalidGraphWF.xml: " + e.getMessage());
}
+ }
- Assert.assertTrue(f1.length() < f2.length());
- f1.delete();
- f2.delete();
-
+ private void generateAndAssertPng(final WorkflowJobBean workflowJob, final String path,
final boolean showKill) {
try {
- GraphGenerator g = new GraphGenerator(readFile("src/test/resources/invalidGraphWF.xml"),
jsonWFJob, true);
- g.write(new FileOutputStream(new File("src/test/resources/invalid.png")));
+ final File outputPng = File.createTempFile("graph-output", path);
+ final String content = IOUtils.getResourceAsString(path, -1);
+ final GraphGenerator g = new GraphGenerator(content, workflowJob);
+ g.write(new FileOutputStream(outputPng));
+ Assert.assertNotNull("PNG read error", ImageIO.read(new FileInputStream(outputPng)));
}
- catch (Exception e) {
- Assert.fail("Write PNG failed for invalidGraphWF.xml: " + e.getMessage());
+ catch (final Exception e) {
+ Assert.fail(String.format("Read or write PNG without kill failed for %s: %s",
path, e.getMessage()));
}
- new File("src/test/resources/invalid.png").delete();
}
public void testJobDAGLimit_more() throws IOException {
- WorkflowJobBean jsonWFJob = new WorkflowJobBean();
+ final WorkflowJobBean jsonWFJob = new WorkflowJobBean();
jsonWFJob.setAppName("My Test App");
jsonWFJob.setId("My Test ID");
- String txt = "src/test/resources/tmp1.txt";
try {
- GraphGenerator g = new GraphGenerator(readFile("src/test/resources/graphWF_26_actions.xml"),
jsonWFJob);
- g.write(new FileOutputStream(new File(txt)));
+ final String content = IOUtils.getResourceAsString("graphWF_26_actions.xml",
-1);
+ final GraphGenerator g = new GraphGenerator(content, jsonWFJob);
+ g.write(new FileOutputStream(File.createTempFile("graph-output", "over-limit")));
Assert.fail("This should not get executed");
}
- catch (Exception e) {
+ catch (final Exception e) {
Assert.assertTrue(e.getMessage().startsWith(
"Can't display the graph. Number of actions are more than display limit"));
}
-
- File f1 = new File(txt);
- f1.delete();
-
- }
-
- private static String readFile(String path) throws IOException {
- File f = new File(path);
- System.out.println("Reading input file " + f.getAbsolutePath());
- FileInputStream stream = new FileInputStream(f);
- try {
- FileChannel fc = stream.getChannel();
- MappedByteBuffer bb = fc.map(FileChannel.MapMode.READ_ONLY, 0, fc.size());
- /* Instead of using default, pass in a decoder. */
- return Charset.defaultCharset().decode(bb).toString();
- }
- finally {
- stream.close();
- }
}
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/0601fc22/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 1db2874..c2b3683 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 5.0.0 release (trunk - unreleased)
+OOZIE-3038 Make all Oozie JUnit tests pass on dist_test (andras.piros via gezapeti)
OOZIE-2940 Possible NullPointerException in WorkflowActionBean (dionusos via gezapeti)
OOZIE-3036 Spark 2.2.0 support: tell Spark not to get any delegation tokens (andras.piros
via gezapeti)
OOZIE-3028 Oozie Pig Action fails with no python dependencies (dbist13 via rohini)
|