sqoop-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Manish Meshram <manishmeshra...@gmail.com>
Subject Re: Connection refused while Job Submission - Sqoop Client API
Date Tue, 24 Sep 2013 05:52:19 GMT
Hi Abraham,

That is another problem,I cannot use the commands you said because when I
start client on the same machine(centOS-remote for me) on which server is
running, the server automatically stops.
But when I started the server on centOS(which is remote for me and I am
accessing it through putty)
and I have a sqoop Client API java program in my system(windows 7).
It does not stops the server and my connection established successfully.
Also MySQL server is installed on my system(windows 7) and I have given
remote access to ip address of CentOS system,
that is why my connection get established in 1st step.
I am attaching the source and the output I am getting, please have a look.

*Program:*
*
*
public class TestSqoopClient {
 private static void printMessage(List<MForm> formList) {
  for(MForm form : formList) {
    List<MInput<?>> inputlist = form.getInputs();
    if (form.getValidationMessage() != null) {
      System.out.println("Form message: " + form.getValidationMessage());
    }
    for (MInput minput : inputlist) {
      if (minput.getValidationStatus() == Status.ACCEPTABLE) {
        System.out.println("Warning:" + minput.getValidationMessage());
      } else if (minput.getValidationStatus() == Status.UNACCEPTABLE) {
        System.out.println("Error:" + minput.getValidationMessage());
      }
    }
  }
}

public static void main(String[] args) {
 try
{
String url = "http://192.168.3.156:12000/sqoop/";
SqoopClient client = new SqoopClient(url);
 /*********Connection**********/
//Dummy connection object
MConnection newCon = client.newConnection(1);

//Get connection and framework forms. Set name for connection
MConnectionForms conForms = newCon.getConnectorPart();
MConnectionForms frameworkForms = newCon.getFrameworkPart();
newCon.setName("MyConnection");
 /*sample for below code
 * MStringInput m = conForms.getStringInput("connection.connectionString");
m.setValue("");*/
 //Set connection forms values
conForms.getStringInput("connection.connectionString").setValue("jdbc:mysql://
10.10.10.103:3306/mydb");
conForms.getStringInput("connection.jdbcDriver").setValue("com.mysql.jdbc.Driver");
conForms.getStringInput("connection.username").setValue("********");
conForms.getStringInput("connection.password").setValue("********");   //I
masked it.

frameworkForms.getIntegerInput("security.maxConnections").setValue(0);

Status status  = client.createConnection(newCon);
if(status.canProceed()) {
 System.out.println("Created. New Connection ID : "
+newCon.getPersistenceId());
} else {
 System.out.println("Check for status and forms error ");
}
 //for checking the errors/warnings
printMessage(newCon.getConnectorPart().getForms());
printMessage(newCon.getFrameworkPart().getForms());
 /*********Job**********/
//Creating dummy job object
MJob newjob = client.newJob(1, org.apache.sqoop.model.MJob.Type.IMPORT);
MJobForms connectorForm = newjob.getConnectorPart();
MJobForms frameworkForm = newjob.getFrameworkPart();
 newjob.setName("ImportJob");
//Database configuration
connectorForm.getStringInput("table.schemaName").setValue("");
 //Input either table name or sql
connectorForm.getStringInput("table.tableName").setValue("student");
//connectorForm.getStringInput("table.sql").setValue("select id,name from
student");
 connectorForm.getStringInput("table.columns").setValue("id,name");
connectorForm.getStringInput("table.partitionColumn").setValue("id");
 //Set boundary value only if required
//connectorForm.getStringInput("table.boundaryQuery").setValue("");

//Output configurations
frameworkForm.getEnumInput("output.storageType").setValue("HDFS");
frameworkForm.getEnumInput("output.outputFormat").setValue("TEXT_FILE");//Other
option: SEQUENCE_FILE/TEXT_FILE
frameworkForm.getStringInput("output.outputDirectory").setValue("/home/hadoop/tmp/hdfs/data");

//Job resources
frameworkForm.getIntegerInput("throttling.extractors").setValue(1);
frameworkForm.getIntegerInput("throttling.loaders").setValue(1);

Status status1 = client.createJob(newjob);
if(status1.canProceed()) {
 System.out.println("New Job ID: "+ newjob.getPersistenceId());
} else {
 System.out.println("Check for status and forms error ");
}
 printMessage(newjob.getConnectorPart().getForms());
printMessage(newjob.getFrameworkPart().getForms());
 /*****Job Submission******/
//Job submission start
long jid = newjob.getPersistenceId();
System.out.println(jid);
 MSubmission submission = client.startSubmission(jid);
 System.out.println("Status : " + submission.getStatus());
 if(submission.getStatus().isRunning() && submission.getProgress() != -1) {
  System.out.println("Progress : " + String.format("%.2f %%",
submission.getProgress() * 100));
}
 System.out.println("Hadoop job id :" + submission.getExternalId());
System.out.println("Job link : " + submission.getExternalLink());
 Counters counters = submission.getCounters();
 if(counters != null) {
  System.out.println("Counters:");
  for(CounterGroup group : counters) {
    System.out.print("\t");
    System.out.println(group.getName());
    for(Counter counter : group) {
      System.out.print("\t\t");
      System.out.print(counter.getName());
      System.out.print(": ");
      System.out.println(counter.getValue());
    }
  }
}
if(submission.getExceptionInfo() != null) {
  System.out.println("Exception info : " +submission.getExceptionInfo());
}


//Check job status
MSubmission submission1 = client.getSubmissionStatus(1);
if(submission1.getStatus().isRunning() && submission1.getProgress() != -1) {
  System.out.println("Progress : " + String.format("%.2f %%",
submission.getProgress() * 100));
}

//Stop a running job
//submission1.stopSubmission(jid);

}
catch(Exception e)
{
e.printStackTrace();
}
 }
}



*Output:*

Created. New Connection ID : 33
New Job ID: 11
11
org.apache.sqoop.common.SqoopException: CLIENT_0006:Server has returned
exception
at
org.apache.sqoop.client.request.Request$ServerExceptionFilter.handle(Request.java:102)
at com.sun.jersey.api.client.WebResource.handle(WebResource.java:670)
at com.sun.jersey.api.client.WebResource.access$200(WebResource.java:74)
at com.sun.jersey.api.client.WebResource$Builder.post(WebResource.java:563)
at org.apache.sqoop.client.request.Request.post(Request.java:66)
at
org.apache.sqoop.client.request.SubmissionRequest.create(SubmissionRequest.java:46)
at
org.apache.sqoop.client.request.SqoopRequests.createSubmission(SqoopRequests.java:134)
at org.apache.sqoop.client.SqoopClient.startSubmission(SqoopClient.java:353)
at com.TestSqoopClient.main(TestSqoopClient.java:126)
Caused by: java.lang.Throwable: GENERIC_JDBC_CONNECTOR_0001:Unable to get a
connection
at
org.apache.sqoop.connector.jdbc.GenericJdbcExecutor.<init>(GenericJdbcExecutor.java:47)
at
org.apache.sqoop.connector.jdbc.GenericJdbcImportInitializer.configureJdbcProperties(GenericJdbcImportInitializer.java:73)
at
org.apache.sqoop.connector.jdbc.GenericJdbcImportInitializer.initialize(GenericJdbcImportInitializer.java:46)
at
org.apache.sqoop.connector.jdbc.GenericJdbcImportInitializer.initialize(GenericJdbcImportInitializer.java:37)
at
org.apache.sqoop.framework.FrameworkManager.submit(FrameworkManager.java:434)
at
org.apache.sqoop.handler.SubmissionRequestHandler.submissionSubmit(SubmissionRequestHandler.java:112)
at
org.apache.sqoop.handler.SubmissionRequestHandler.handleActionEvent(SubmissionRequestHandler.java:98)
at
org.apache.sqoop.handler.SubmissionRequestHandler.handleEvent(SubmissionRequestHandler.java:68)
at
org.apache.sqoop.server.v1.SubmissionServlet.handlePostRequest(SubmissionServlet.java:44)
at
org.apache.sqoop.server.SqoopProtocolServlet.doPost(SqoopProtocolServlet.java:63)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:637)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:717)
at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)
at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at
org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:233)
at
org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:191)
at
org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:127)
at
org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103)
at
org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)
at
org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:293)
at
org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:861)
at
org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.process(Http11Protocol.java:606)
at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:489)
at java.lang.Thread.run(Thread.java:724)
Caused by: java.lang.Throwable: Communications link failure

The last packet sent successfully to the server was 0 milliseconds ago. The
driver has not received any packets from the server.
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at
sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)
at
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:526)
at com.mysql.jdbc.Util.handleNewInstance(Util.java:411)
at com.mysql.jdbc.SQLError.createCommunicationsException(SQLError.java:1121)
at com.mysql.jdbc.MysqlIO.<init>(MysqlIO.java:355)
at com.mysql.jdbc.ConnectionImpl.coreConnect(ConnectionImpl.java:2479)
at com.mysql.jdbc.ConnectionImpl.connectOneTryOnly(ConnectionImpl.java:2516)
at com.mysql.jdbc.ConnectionImpl.createNewIO(ConnectionImpl.java:2301)
at com.mysql.jdbc.ConnectionImpl.<init>(ConnectionImpl.java:834)
at com.mysql.jdbc.JDBC4Connection.<init>(JDBC4Connection.java:47)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at
sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)
at
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:526)
at com.mysql.jdbc.Util.handleNewInstance(Util.java:411)
at com.mysql.jdbc.ConnectionImpl.getInstance(ConnectionImpl.java:416)
at
com.mysql.jdbc.NonRegisteringDriver.connect(NonRegisteringDriver.java:317)
at java.sql.DriverManager.getConnection(DriverManager.java:571)
at java.sql.DriverManager.getConnection(DriverManager.java:215)
at
org.apache.sqoop.connector.jdbc.GenericJdbcExecutor.<init>(GenericJdbcExecutor.java:40)
... 23 more
Caused by: java.net.ConnectException: Connection refused
at java.net.PlainSocketImpl.socketConnect(Native Method)
at
java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)
at
java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)
at
java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)
at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
at java.net.Socket.connect(Socket.java:579)
at java.net.Socket.connect(Socket.java:528)
at java.net.Socket.<init>(Socket.java:425)
at java.net.Socket.<init>(Socket.java:241)
at
com.mysql.jdbc.StandardSocketFactory.connect(StandardSocketFactory.java:259)
at com.mysql.jdbc.MysqlIO.<init>(MysqlIO.java:305)
... 38 more


Thanks,

       Manish


On Mon, Sep 23, 2013 at 11:11 PM, Abraham Elmahrek <abe@cloudera.com> wrote:

> Hey Manish,
>
> It looks like your job is using a connection that is pointing to a
> database server that is down or doesn't exist. Could you validate your
> connection parameters and verify you're using the correct connection in the
> job? Please provide the job and connection details via the "show
> connection" and "show job" commands. That will definitely assist us in
> figuring out what went wrong in your setup.
>
> -Abe
>
>
> On Mon, Sep 23, 2013 at 3:45 AM, Manish Meshram <manishmeshram36@gmail.com
> > wrote:
>
>> Hi,
>>
>> I am using sqoop Client API for importing rdbms data to hdfs.
>> I am using Hadoop 1.2.1(single node)
>> and sqoop 1.99.2
>> Hadoop is remote for me installed on CentOS and I am accessing it through
>> putty.
>>
>> I have gone through Sqoop client API guide -
>> http://sqoop.apache.org/docs/1.99.2/ClientAPI.html
>>
>> As there are 3 steps in total:
>> 1. Create connection.....(done)
>> 2. Create Job..........(done)
>> 3. Submit Job.........(GETTING ERROR)
>>
>> Steps 1 and 2 are working fine for me.
>> But during Step 3 i.e. while submitting the job I am getting the
>> following exception
>> in my eclipse IDE:
>>
>>
>>  org.apache.sqoop.common.SqoopException: CLIENT_0006:Server has returned
>> exception
>> at
>> org.apache.sqoop.client.request.Request$ServerExceptionFilter.handle(Request.java:102)
>>  at com.sun.jersey.api.client.WebResource.handle(WebResource.java:670)
>> at com.sun.jersey.api.client.WebResource.access$200(WebResource.java:74)
>>  at
>> com.sun.jersey.api.client.WebResource$Builder.post(WebResource.java:563)
>> at org.apache.sqoop.client.request.Request.post(Request.java:66)
>>  at
>> org.apache.sqoop.client.request.SubmissionRequest.create(SubmissionRequest.java:46)
>> at
>> org.apache.sqoop.client.request.SqoopRequests.createSubmission(SqoopRequests.java:134)
>>  at
>> org.apache.sqoop.client.SqoopClient.startSubmission(SqoopClient.java:353)
>> at com.TestSqoopClient.main(TestSqoopClient.java:126)
>> Caused by: java.lang.Throwable: GENERIC_JDBC_CONNECTOR_0001:Unable to get
>> a connection
>> at
>> org.apache.sqoop.connector.jdbc.GenericJdbcExecutor.<init>(GenericJdbcExecutor.java:47)
>>  at
>> org.apache.sqoop.connector.jdbc.GenericJdbcImportInitializer.configureJdbcProperties(GenericJdbcImportInitializer.java:73)
>> at
>> org.apache.sqoop.connector.jdbc.GenericJdbcImportInitializer.initialize(GenericJdbcImportInitializer.java:46)
>>  at
>> org.apache.sqoop.connector.jdbc.GenericJdbcImportInitializer.initialize(GenericJdbcImportInitializer.java:37)
>> at
>> org.apache.sqoop.framework.FrameworkManager.submit(FrameworkManager.java:434)
>>  at
>> org.apache.sqoop.handler.SubmissionRequestHandler.submissionSubmit(SubmissionRequestHandler.java:112)
>> at
>> org.apache.sqoop.handler.SubmissionRequestHandler.handleActionEvent(SubmissionRequestHandler.java:98)
>>  at
>> org.apache.sqoop.handler.SubmissionRequestHandler.handleEvent(SubmissionRequestHandler.java:68)
>> at
>> org.apache.sqoop.server.v1.SubmissionServlet.handlePostRequest(SubmissionServlet.java:44)
>>  at
>> org.apache.sqoop.server.SqoopProtocolServlet.doPost(SqoopProtocolServlet.java:63)
>> at javax.servlet.http.HttpServlet.service(HttpServlet.java:637)
>>  at javax.servlet.http.HttpServlet.service(HttpServlet.java:717)
>> at
>> org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)
>>  at
>> org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
>> at
>> org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:233)
>>  at
>> org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:191)
>> at
>> org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:127)
>>  at
>> org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103)
>> at
>> org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)
>>  at
>> org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:293)
>> at
>> org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:861)
>>  at
>> org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.process(Http11Protocol.java:606)
>> at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:489)
>>  at java.lang.Thread.run(Thread.java:724)
>> Caused by: java.lang.Throwable: Communications link failure
>>
>> The last packet sent successfully to the server was 0 milliseconds ago.
>> The driver has not received any packets from the server.
>>  at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
>> at
>> sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)
>>  at
>> sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
>> at java.lang.reflect.Constructor.newInstance(Constructor.java:526)
>>  at com.mysql.jdbc.Util.handleNewInstance(Util.java:411)
>> at
>> com.mysql.jdbc.SQLError.createCommunicationsException(SQLError.java:1121)
>>  at com.mysql.jdbc.MysqlIO.<init>(MysqlIO.java:355)
>> at com.mysql.jdbc.ConnectionImpl.coreConnect(ConnectionImpl.java:2479)
>>  at
>> com.mysql.jdbc.ConnectionImpl.connectOneTryOnly(ConnectionImpl.java:2516)
>> at com.mysql.jdbc.ConnectionImpl.createNewIO(ConnectionImpl.java:2301)
>>  at com.mysql.jdbc.ConnectionImpl.<init>(ConnectionImpl.java:834)
>> at com.mysql.jdbc.JDBC4Connection.<init>(JDBC4Connection.java:47)
>>  at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
>> at
>> sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)
>>  at
>> sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
>> at java.lang.reflect.Constructor.newInstance(Constructor.java:526)
>>  at com.mysql.jdbc.Util.handleNewInstance(Util.java:411)
>> at com.mysql.jdbc.ConnectionImpl.getInstance(ConnectionImpl.java:416)
>>  at
>> com.mysql.jdbc.NonRegisteringDriver.connect(NonRegisteringDriver.java:317)
>> at java.sql.DriverManager.getConnection(DriverManager.java:571)
>>  at java.sql.DriverManager.getConnection(DriverManager.java:215)
>> at
>> org.apache.sqoop.connector.jdbc.GenericJdbcExecutor.<init>(GenericJdbcExecutor.java:40)
>>  ... 23 more
>> Caused by: java.net.ConnectException: Connection refused
>> at java.net.PlainSocketImpl.socketConnect(Native Method)
>>  at
>> java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)
>> at
>> java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)
>>  at
>> java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)
>> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
>>  at java.net.Socket.connect(Socket.java:579)
>> at java.net.Socket.connect(Socket.java:528)
>> at java.net.Socket.<init>(Socket.java:425)
>>  at java.net.Socket.<init>(Socket.java:241)
>> at
>> com.mysql.jdbc.StandardSocketFactory.connect(StandardSocketFactory.java:259)
>>  at com.mysql.jdbc.MysqlIO.<init>(MysqlIO.java:305)
>> ... 38 more
>>
>
>

Mime
View raw message