sqoop-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Abraham Elmahrek <...@cloudera.com>
Subject Re: Connection refused while Job Submission - Sqoop Client API
Date Tue, 24 Sep 2013 06:24:07 GMT
Manish,

Have you given your mapreduce cluster access to your database? it will be
using the credentials in the Connection you created to distribute the
process of importing into HDFS.

-Abe


On Mon, Sep 23, 2013 at 10:52 PM, Manish Meshram
<manishmeshram36@gmail.com>wrote:

> Hi Abraham,
>
> That is another problem,I cannot use the commands you said because when I
> start client on the same machine(centOS-remote for me) on which server is
> running, the server automatically stops.
> But when I started the server on centOS(which is remote for me and I am
> accessing it through putty)
> and I have a sqoop Client API java program in my system(windows 7).
> It does not stops the server and my connection established successfully.
> Also MySQL server is installed on my system(windows 7) and I have given
> remote access to ip address of CentOS system,
> that is why my connection get established in 1st step.
> I am attaching the source and the output I am getting, please have a look.
>
> *Program:*
> *
> *
> public class TestSqoopClient {
>  private static void printMessage(List<MForm> formList) {
>   for(MForm form : formList) {
>     List<MInput<?>> inputlist = form.getInputs();
>     if (form.getValidationMessage() != null) {
>       System.out.println("Form message: " + form.getValidationMessage());
>     }
>     for (MInput minput : inputlist) {
>       if (minput.getValidationStatus() == Status.ACCEPTABLE) {
>         System.out.println("Warning:" + minput.getValidationMessage());
>       } else if (minput.getValidationStatus() == Status.UNACCEPTABLE) {
>         System.out.println("Error:" + minput.getValidationMessage());
>       }
>     }
>   }
> }
>
> public static void main(String[] args) {
>  try
>  {
> String url = "http://192.168.3.156:12000/sqoop/";
>  SqoopClient client = new SqoopClient(url);
>   /*********Connection**********/
> //Dummy connection object
> MConnection newCon = client.newConnection(1);
>
> //Get connection and framework forms. Set name for connection
> MConnectionForms conForms = newCon.getConnectorPart();
>  MConnectionForms frameworkForms = newCon.getFrameworkPart();
> newCon.setName("MyConnection");
>   /*sample for below code
>  * MStringInput m = conForms.getStringInput("connection.connectionString");
>  m.setValue("");*/
>  //Set connection forms values
>
> conForms.getStringInput("connection.connectionString").setValue("jdbc:mysql://
> 10.10.10.103:3306/mydb");
>
> conForms.getStringInput("connection.jdbcDriver").setValue("com.mysql.jdbc.Driver");
> conForms.getStringInput("connection.username").setValue("********");
>  conForms.getStringInput("connection.password").setValue("********");
> //I masked it.
>
> frameworkForms.getIntegerInput("security.maxConnections").setValue(0);
>
> Status status  = client.createConnection(newCon);
> if(status.canProceed()) {
>  System.out.println("Created. New Connection ID : "
> +newCon.getPersistenceId());
>  } else {
>  System.out.println("Check for status and forms error ");
> }
>  //for checking the errors/warnings
> printMessage(newCon.getConnectorPart().getForms());
>  printMessage(newCon.getFrameworkPart().getForms());
>   /*********Job**********/
> //Creating dummy job object
> MJob newjob = client.newJob(1, org.apache.sqoop.model.MJob.Type.IMPORT);
>  MJobForms connectorForm = newjob.getConnectorPart();
> MJobForms frameworkForm = newjob.getFrameworkPart();
>  newjob.setName("ImportJob");
> //Database configuration
>  connectorForm.getStringInput("table.schemaName").setValue("");
>  //Input either table name or sql
>  connectorForm.getStringInput("table.tableName").setValue("student");
> //connectorForm.getStringInput("table.sql").setValue("select id,name from
> student");
>  connectorForm.getStringInput("table.columns").setValue("id,name");
> connectorForm.getStringInput("table.partitionColumn").setValue("id");
>  //Set boundary value only if required
> //connectorForm.getStringInput("table.boundaryQuery").setValue("");
>
> //Output configurations
> frameworkForm.getEnumInput("output.storageType").setValue("HDFS");
>  frameworkForm.getEnumInput("output.outputFormat").setValue("TEXT_FILE");//Other
> option: SEQUENCE_FILE/TEXT_FILE
>
> frameworkForm.getStringInput("output.outputDirectory").setValue("/home/hadoop/tmp/hdfs/data");
>
> //Job resources
> frameworkForm.getIntegerInput("throttling.extractors").setValue(1);
>  frameworkForm.getIntegerInput("throttling.loaders").setValue(1);
>
> Status status1 = client.createJob(newjob);
>  if(status1.canProceed()) {
>  System.out.println("New Job ID: "+ newjob.getPersistenceId());
>  } else {
>  System.out.println("Check for status and forms error ");
> }
>  printMessage(newjob.getConnectorPart().getForms());
> printMessage(newjob.getFrameworkPart().getForms());
>   /*****Job Submission******/
> //Job submission start
>  long jid = newjob.getPersistenceId();
> System.out.println(jid);
>  MSubmission submission = client.startSubmission(jid);
>  System.out.println("Status : " + submission.getStatus());
>  if(submission.getStatus().isRunning() && submission.getProgress() != -1)
> {
>   System.out.println("Progress : " + String.format("%.2f %%",
> submission.getProgress() * 100));
>  }
>  System.out.println("Hadoop job id :" + submission.getExternalId());
>  System.out.println("Job link : " + submission.getExternalLink());
>  Counters counters = submission.getCounters();
>  if(counters != null) {
>   System.out.println("Counters:");
>   for(CounterGroup group : counters) {
>     System.out.print("\t");
>     System.out.println(group.getName());
>     for(Counter counter : group) {
>       System.out.print("\t\t");
>       System.out.print(counter.getName());
>       System.out.print(": ");
>       System.out.println(counter.getValue());
>     }
>   }
> }
> if(submission.getExceptionInfo() != null) {
>   System.out.println("Exception info : " +submission.getExceptionInfo());
> }
>
>
> //Check job status
> MSubmission submission1 = client.getSubmissionStatus(1);
> if(submission1.getStatus().isRunning() && submission1.getProgress() != -1)
> {
>   System.out.println("Progress : " + String.format("%.2f %%",
> submission.getProgress() * 100));
> }
>
> //Stop a running job
> //submission1.stopSubmission(jid);
>
> }
>  catch(Exception e)
> {
> e.printStackTrace();
>  }
>  }
> }
>
>
>
> *Output:*
>
> Created. New Connection ID : 33
> New Job ID: 11
> 11
> org.apache.sqoop.common.SqoopException: CLIENT_0006:Server has returned
> exception
>  at
> org.apache.sqoop.client.request.Request$ServerExceptionFilter.handle(Request.java:102)
> at com.sun.jersey.api.client.WebResource.handle(WebResource.java:670)
>  at com.sun.jersey.api.client.WebResource.access$200(WebResource.java:74)
> at com.sun.jersey.api.client.WebResource$Builder.post(WebResource.java:563)
>  at org.apache.sqoop.client.request.Request.post(Request.java:66)
> at
> org.apache.sqoop.client.request.SubmissionRequest.create(SubmissionRequest.java:46)
>  at
> org.apache.sqoop.client.request.SqoopRequests.createSubmission(SqoopRequests.java:134)
> at
> org.apache.sqoop.client.SqoopClient.startSubmission(SqoopClient.java:353)
>  at com.TestSqoopClient.main(TestSqoopClient.java:126)
> Caused by: java.lang.Throwable: GENERIC_JDBC_CONNECTOR_0001:Unable to get
> a connection
> at
> org.apache.sqoop.connector.jdbc.GenericJdbcExecutor.<init>(GenericJdbcExecutor.java:47)
>  at
> org.apache.sqoop.connector.jdbc.GenericJdbcImportInitializer.configureJdbcProperties(GenericJdbcImportInitializer.java:73)
> at
> org.apache.sqoop.connector.jdbc.GenericJdbcImportInitializer.initialize(GenericJdbcImportInitializer.java:46)
>  at
> org.apache.sqoop.connector.jdbc.GenericJdbcImportInitializer.initialize(GenericJdbcImportInitializer.java:37)
> at
> org.apache.sqoop.framework.FrameworkManager.submit(FrameworkManager.java:434)
>  at
> org.apache.sqoop.handler.SubmissionRequestHandler.submissionSubmit(SubmissionRequestHandler.java:112)
> at
> org.apache.sqoop.handler.SubmissionRequestHandler.handleActionEvent(SubmissionRequestHandler.java:98)
>  at
> org.apache.sqoop.handler.SubmissionRequestHandler.handleEvent(SubmissionRequestHandler.java:68)
> at
> org.apache.sqoop.server.v1.SubmissionServlet.handlePostRequest(SubmissionServlet.java:44)
>  at
> org.apache.sqoop.server.SqoopProtocolServlet.doPost(SqoopProtocolServlet.java:63)
> at javax.servlet.http.HttpServlet.service(HttpServlet.java:637)
>  at javax.servlet.http.HttpServlet.service(HttpServlet.java:717)
> at
> org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)
>  at
> org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
> at
> org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:233)
>  at
> org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:191)
> at
> org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:127)
>  at
> org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103)
> at
> org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)
>  at
> org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:293)
> at
> org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:861)
>  at
> org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.process(Http11Protocol.java:606)
> at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:489)
>  at java.lang.Thread.run(Thread.java:724)
> Caused by: java.lang.Throwable: Communications link failure
>
> The last packet sent successfully to the server was 0 milliseconds ago.
> The driver has not received any packets from the server.
>  at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
> at
> sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)
>  at
> sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
> at java.lang.reflect.Constructor.newInstance(Constructor.java:526)
>  at com.mysql.jdbc.Util.handleNewInstance(Util.java:411)
> at
> com.mysql.jdbc.SQLError.createCommunicationsException(SQLError.java:1121)
>  at com.mysql.jdbc.MysqlIO.<init>(MysqlIO.java:355)
> at com.mysql.jdbc.ConnectionImpl.coreConnect(ConnectionImpl.java:2479)
>  at
> com.mysql.jdbc.ConnectionImpl.connectOneTryOnly(ConnectionImpl.java:2516)
> at com.mysql.jdbc.ConnectionImpl.createNewIO(ConnectionImpl.java:2301)
>  at com.mysql.jdbc.ConnectionImpl.<init>(ConnectionImpl.java:834)
> at com.mysql.jdbc.JDBC4Connection.<init>(JDBC4Connection.java:47)
>  at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
> at
> sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)
>  at
> sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
> at java.lang.reflect.Constructor.newInstance(Constructor.java:526)
>  at com.mysql.jdbc.Util.handleNewInstance(Util.java:411)
> at com.mysql.jdbc.ConnectionImpl.getInstance(ConnectionImpl.java:416)
>  at
> com.mysql.jdbc.NonRegisteringDriver.connect(NonRegisteringDriver.java:317)
> at java.sql.DriverManager.getConnection(DriverManager.java:571)
>  at java.sql.DriverManager.getConnection(DriverManager.java:215)
> at
> org.apache.sqoop.connector.jdbc.GenericJdbcExecutor.<init>(GenericJdbcExecutor.java:40)
>  ... 23 more
> Caused by: java.net.ConnectException: Connection refused
> at java.net.PlainSocketImpl.socketConnect(Native Method)
>  at
> java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)
> at
> java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)
>  at
> java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)
> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
>  at java.net.Socket.connect(Socket.java:579)
> at java.net.Socket.connect(Socket.java:528)
> at java.net.Socket.<init>(Socket.java:425)
>  at java.net.Socket.<init>(Socket.java:241)
> at
> com.mysql.jdbc.StandardSocketFactory.connect(StandardSocketFactory.java:259)
>  at com.mysql.jdbc.MysqlIO.<init>(MysqlIO.java:305)
> ... 38 more
>
>
> Thanks,
>
>        Manish
>
>
> On Mon, Sep 23, 2013 at 11:11 PM, Abraham Elmahrek <abe@cloudera.com>wrote:
>
>> Hey Manish,
>>
>> It looks like your job is using a connection that is pointing to a
>> database server that is down or doesn't exist. Could you validate your
>> connection parameters and verify you're using the correct connection in the
>> job? Please provide the job and connection details via the "show
>> connection" and "show job" commands. That will definitely assist us in
>> figuring out what went wrong in your setup.
>>
>> -Abe
>>
>>
>> On Mon, Sep 23, 2013 at 3:45 AM, Manish Meshram <
>> manishmeshram36@gmail.com> wrote:
>>
>>> Hi,
>>>
>>> I am using sqoop Client API for importing rdbms data to hdfs.
>>> I am using Hadoop 1.2.1(single node)
>>> and sqoop 1.99.2
>>> Hadoop is remote for me installed on CentOS and I am accessing it
>>> through putty.
>>>
>>> I have gone through Sqoop client API guide -
>>> http://sqoop.apache.org/docs/1.99.2/ClientAPI.html
>>>
>>> As there are 3 steps in total:
>>> 1. Create connection.....(done)
>>> 2. Create Job..........(done)
>>> 3. Submit Job.........(GETTING ERROR)
>>>
>>> Steps 1 and 2 are working fine for me.
>>> But during Step 3 i.e. while submitting the job I am getting the
>>> following exception
>>> in my eclipse IDE:
>>>
>>>
>>>  org.apache.sqoop.common.SqoopException: CLIENT_0006:Server has returned
>>> exception
>>> at
>>> org.apache.sqoop.client.request.Request$ServerExceptionFilter.handle(Request.java:102)
>>>  at com.sun.jersey.api.client.WebResource.handle(WebResource.java:670)
>>> at com.sun.jersey.api.client.WebResource.access$200(WebResource.java:74)
>>>  at
>>> com.sun.jersey.api.client.WebResource$Builder.post(WebResource.java:563)
>>> at org.apache.sqoop.client.request.Request.post(Request.java:66)
>>>  at
>>> org.apache.sqoop.client.request.SubmissionRequest.create(SubmissionRequest.java:46)
>>> at
>>> org.apache.sqoop.client.request.SqoopRequests.createSubmission(SqoopRequests.java:134)
>>>  at
>>> org.apache.sqoop.client.SqoopClient.startSubmission(SqoopClient.java:353)
>>> at com.TestSqoopClient.main(TestSqoopClient.java:126)
>>> Caused by: java.lang.Throwable: GENERIC_JDBC_CONNECTOR_0001:Unable to
>>> get a connection
>>> at
>>> org.apache.sqoop.connector.jdbc.GenericJdbcExecutor.<init>(GenericJdbcExecutor.java:47)
>>>  at
>>> org.apache.sqoop.connector.jdbc.GenericJdbcImportInitializer.configureJdbcProperties(GenericJdbcImportInitializer.java:73)
>>> at
>>> org.apache.sqoop.connector.jdbc.GenericJdbcImportInitializer.initialize(GenericJdbcImportInitializer.java:46)
>>>  at
>>> org.apache.sqoop.connector.jdbc.GenericJdbcImportInitializer.initialize(GenericJdbcImportInitializer.java:37)
>>> at
>>> org.apache.sqoop.framework.FrameworkManager.submit(FrameworkManager.java:434)
>>>  at
>>> org.apache.sqoop.handler.SubmissionRequestHandler.submissionSubmit(SubmissionRequestHandler.java:112)
>>> at
>>> org.apache.sqoop.handler.SubmissionRequestHandler.handleActionEvent(SubmissionRequestHandler.java:98)
>>>  at
>>> org.apache.sqoop.handler.SubmissionRequestHandler.handleEvent(SubmissionRequestHandler.java:68)
>>> at
>>> org.apache.sqoop.server.v1.SubmissionServlet.handlePostRequest(SubmissionServlet.java:44)
>>>  at
>>> org.apache.sqoop.server.SqoopProtocolServlet.doPost(SqoopProtocolServlet.java:63)
>>> at javax.servlet.http.HttpServlet.service(HttpServlet.java:637)
>>>  at javax.servlet.http.HttpServlet.service(HttpServlet.java:717)
>>> at
>>> org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)
>>>  at
>>> org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
>>> at
>>> org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:233)
>>>  at
>>> org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:191)
>>> at
>>> org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:127)
>>>  at
>>> org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103)
>>> at
>>> org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)
>>>  at
>>> org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:293)
>>> at
>>> org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:861)
>>>  at
>>> org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.process(Http11Protocol.java:606)
>>> at
>>> org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:489)
>>>  at java.lang.Thread.run(Thread.java:724)
>>> Caused by: java.lang.Throwable: Communications link failure
>>>
>>> The last packet sent successfully to the server was 0 milliseconds ago.
>>> The driver has not received any packets from the server.
>>>  at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native
>>> Method)
>>> at
>>> sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)
>>>  at
>>> sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
>>> at java.lang.reflect.Constructor.newInstance(Constructor.java:526)
>>>  at com.mysql.jdbc.Util.handleNewInstance(Util.java:411)
>>> at
>>> com.mysql.jdbc.SQLError.createCommunicationsException(SQLError.java:1121)
>>>  at com.mysql.jdbc.MysqlIO.<init>(MysqlIO.java:355)
>>> at com.mysql.jdbc.ConnectionImpl.coreConnect(ConnectionImpl.java:2479)
>>>  at
>>> com.mysql.jdbc.ConnectionImpl.connectOneTryOnly(ConnectionImpl.java:2516)
>>> at com.mysql.jdbc.ConnectionImpl.createNewIO(ConnectionImpl.java:2301)
>>>  at com.mysql.jdbc.ConnectionImpl.<init>(ConnectionImpl.java:834)
>>> at com.mysql.jdbc.JDBC4Connection.<init>(JDBC4Connection.java:47)
>>>  at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native
>>> Method)
>>> at
>>> sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)
>>>  at
>>> sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
>>> at java.lang.reflect.Constructor.newInstance(Constructor.java:526)
>>>  at com.mysql.jdbc.Util.handleNewInstance(Util.java:411)
>>> at com.mysql.jdbc.ConnectionImpl.getInstance(ConnectionImpl.java:416)
>>>  at
>>> com.mysql.jdbc.NonRegisteringDriver.connect(NonRegisteringDriver.java:317)
>>> at java.sql.DriverManager.getConnection(DriverManager.java:571)
>>>  at java.sql.DriverManager.getConnection(DriverManager.java:215)
>>> at
>>> org.apache.sqoop.connector.jdbc.GenericJdbcExecutor.<init>(GenericJdbcExecutor.java:40)
>>>  ... 23 more
>>> Caused by: java.net.ConnectException: Connection refused
>>> at java.net.PlainSocketImpl.socketConnect(Native Method)
>>>  at
>>> java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)
>>> at
>>> java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)
>>>  at
>>> java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)
>>> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
>>>  at java.net.Socket.connect(Socket.java:579)
>>> at java.net.Socket.connect(Socket.java:528)
>>> at java.net.Socket.<init>(Socket.java:425)
>>>  at java.net.Socket.<init>(Socket.java:241)
>>> at
>>> com.mysql.jdbc.StandardSocketFactory.connect(StandardSocketFactory.java:259)
>>>  at com.mysql.jdbc.MysqlIO.<init>(MysqlIO.java:305)
>>> ... 38 more
>>>
>>
>>
>

Mime
View raw message