sqoop-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Manish Meshram <manishmeshra...@gmail.com>
Subject Re: Connection refused while Job Submission - Sqoop Client API
Date Tue, 24 Sep 2013 07:12:46 GMT
Hi Abraham,

   Can you tell me how to give mapreduce cluster access to database.
   I guess I haven't given it.

Thanks,
   Manish


On Tue, Sep 24, 2013 at 11:54 AM, Abraham Elmahrek <abe@cloudera.com> wrote:

> Manish,
>
> Have you given your mapreduce cluster access to your database? it will be
> using the credentials in the Connection you created to distribute the
> process of importing into HDFS.
>
> -Abe
>
>
> On Mon, Sep 23, 2013 at 10:52 PM, Manish Meshram <
> manishmeshram36@gmail.com> wrote:
>
>> Hi Abraham,
>>
>> That is another problem,I cannot use the commands you said because when I
>> start client on the same machine(centOS-remote for me) on which server is
>> running, the server automatically stops.
>> But when I started the server on centOS(which is remote for me and I am
>> accessing it through putty)
>> and I have a sqoop Client API java program in my system(windows 7).
>> It does not stops the server and my connection established successfully.
>> Also MySQL server is installed on my system(windows 7) and I have given
>> remote access to ip address of CentOS system,
>> that is why my connection get established in 1st step.
>> I am attaching the source and the output I am getting, please have a look.
>>
>> *Program:*
>> *
>> *
>> public class TestSqoopClient {
>>  private static void printMessage(List<MForm> formList) {
>>   for(MForm form : formList) {
>>     List<MInput<?>> inputlist = form.getInputs();
>>     if (form.getValidationMessage() != null) {
>>       System.out.println("Form message: " + form.getValidationMessage());
>>     }
>>     for (MInput minput : inputlist) {
>>       if (minput.getValidationStatus() == Status.ACCEPTABLE) {
>>         System.out.println("Warning:" + minput.getValidationMessage());
>>       } else if (minput.getValidationStatus() == Status.UNACCEPTABLE) {
>>         System.out.println("Error:" + minput.getValidationMessage());
>>       }
>>     }
>>   }
>> }
>>
>> public static void main(String[] args) {
>>  try
>>  {
>> String url = "http://192.168.3.156:12000/sqoop/";
>>  SqoopClient client = new SqoopClient(url);
>>   /*********Connection**********/
>> //Dummy connection object
>> MConnection newCon = client.newConnection(1);
>>
>> //Get connection and framework forms. Set name for connection
>> MConnectionForms conForms = newCon.getConnectorPart();
>>  MConnectionForms frameworkForms = newCon.getFrameworkPart();
>> newCon.setName("MyConnection");
>>   /*sample for below code
>>  * MStringInput m =
>> conForms.getStringInput("connection.connectionString");
>>  m.setValue("");*/
>>  //Set connection forms values
>>
>> conForms.getStringInput("connection.connectionString").setValue("jdbc:mysql://
>> 10.10.10.103:3306/mydb");
>>
>> conForms.getStringInput("connection.jdbcDriver").setValue("com.mysql.jdbc.Driver");
>> conForms.getStringInput("connection.username").setValue("********");
>>  conForms.getStringInput("connection.password").setValue("********");
>> //I masked it.
>>
>> frameworkForms.getIntegerInput("security.maxConnections").setValue(0);
>>
>> Status status  = client.createConnection(newCon);
>> if(status.canProceed()) {
>>  System.out.println("Created. New Connection ID : "
>> +newCon.getPersistenceId());
>>  } else {
>>  System.out.println("Check for status and forms error ");
>> }
>>  //for checking the errors/warnings
>> printMessage(newCon.getConnectorPart().getForms());
>>  printMessage(newCon.getFrameworkPart().getForms());
>>   /*********Job**********/
>> //Creating dummy job object
>> MJob newjob = client.newJob(1, org.apache.sqoop.model.MJob.Type.IMPORT);
>>  MJobForms connectorForm = newjob.getConnectorPart();
>> MJobForms frameworkForm = newjob.getFrameworkPart();
>>  newjob.setName("ImportJob");
>> //Database configuration
>>  connectorForm.getStringInput("table.schemaName").setValue("");
>>  //Input either table name or sql
>>  connectorForm.getStringInput("table.tableName").setValue("student");
>> //connectorForm.getStringInput("table.sql").setValue("select id,name from
>> student");
>>  connectorForm.getStringInput("table.columns").setValue("id,name");
>> connectorForm.getStringInput("table.partitionColumn").setValue("id");
>>  //Set boundary value only if required
>> //connectorForm.getStringInput("table.boundaryQuery").setValue("");
>>
>> //Output configurations
>> frameworkForm.getEnumInput("output.storageType").setValue("HDFS");
>>  frameworkForm.getEnumInput("output.outputFormat").setValue("TEXT_FILE");//Other
>> option: SEQUENCE_FILE/TEXT_FILE
>>
>> frameworkForm.getStringInput("output.outputDirectory").setValue("/home/hadoop/tmp/hdfs/data");
>>
>> //Job resources
>> frameworkForm.getIntegerInput("throttling.extractors").setValue(1);
>>  frameworkForm.getIntegerInput("throttling.loaders").setValue(1);
>>
>> Status status1 = client.createJob(newjob);
>>  if(status1.canProceed()) {
>>  System.out.println("New Job ID: "+ newjob.getPersistenceId());
>>  } else {
>>  System.out.println("Check for status and forms error ");
>> }
>>  printMessage(newjob.getConnectorPart().getForms());
>> printMessage(newjob.getFrameworkPart().getForms());
>>   /*****Job Submission******/
>> //Job submission start
>>  long jid = newjob.getPersistenceId();
>> System.out.println(jid);
>>  MSubmission submission = client.startSubmission(jid);
>>  System.out.println("Status : " + submission.getStatus());
>>  if(submission.getStatus().isRunning() && submission.getProgress() !=
>> -1) {
>>   System.out.println("Progress : " + String.format("%.2f %%",
>> submission.getProgress() * 100));
>>  }
>>  System.out.println("Hadoop job id :" + submission.getExternalId());
>>  System.out.println("Job link : " + submission.getExternalLink());
>>  Counters counters = submission.getCounters();
>>  if(counters != null) {
>>   System.out.println("Counters:");
>>   for(CounterGroup group : counters) {
>>     System.out.print("\t");
>>     System.out.println(group.getName());
>>     for(Counter counter : group) {
>>       System.out.print("\t\t");
>>       System.out.print(counter.getName());
>>       System.out.print(": ");
>>       System.out.println(counter.getValue());
>>     }
>>   }
>> }
>> if(submission.getExceptionInfo() != null) {
>>   System.out.println("Exception info : " +submission.getExceptionInfo());
>> }
>>
>>
>> //Check job status
>> MSubmission submission1 = client.getSubmissionStatus(1);
>> if(submission1.getStatus().isRunning() && submission1.getProgress() !=
>> -1) {
>>   System.out.println("Progress : " + String.format("%.2f %%",
>> submission.getProgress() * 100));
>> }
>>
>> //Stop a running job
>> //submission1.stopSubmission(jid);
>>
>> }
>>  catch(Exception e)
>> {
>> e.printStackTrace();
>>  }
>>  }
>> }
>>
>>
>>
>> *Output:*
>>
>> Created. New Connection ID : 33
>> New Job ID: 11
>> 11
>> org.apache.sqoop.common.SqoopException: CLIENT_0006:Server has returned
>> exception
>>  at
>> org.apache.sqoop.client.request.Request$ServerExceptionFilter.handle(Request.java:102)
>> at com.sun.jersey.api.client.WebResource.handle(WebResource.java:670)
>>  at com.sun.jersey.api.client.WebResource.access$200(WebResource.java:74)
>> at
>> com.sun.jersey.api.client.WebResource$Builder.post(WebResource.java:563)
>>  at org.apache.sqoop.client.request.Request.post(Request.java:66)
>> at
>> org.apache.sqoop.client.request.SubmissionRequest.create(SubmissionRequest.java:46)
>>  at
>> org.apache.sqoop.client.request.SqoopRequests.createSubmission(SqoopRequests.java:134)
>> at
>> org.apache.sqoop.client.SqoopClient.startSubmission(SqoopClient.java:353)
>>  at com.TestSqoopClient.main(TestSqoopClient.java:126)
>> Caused by: java.lang.Throwable: GENERIC_JDBC_CONNECTOR_0001:Unable to get
>> a connection
>> at
>> org.apache.sqoop.connector.jdbc.GenericJdbcExecutor.<init>(GenericJdbcExecutor.java:47)
>>  at
>> org.apache.sqoop.connector.jdbc.GenericJdbcImportInitializer.configureJdbcProperties(GenericJdbcImportInitializer.java:73)
>> at
>> org.apache.sqoop.connector.jdbc.GenericJdbcImportInitializer.initialize(GenericJdbcImportInitializer.java:46)
>>  at
>> org.apache.sqoop.connector.jdbc.GenericJdbcImportInitializer.initialize(GenericJdbcImportInitializer.java:37)
>> at
>> org.apache.sqoop.framework.FrameworkManager.submit(FrameworkManager.java:434)
>>  at
>> org.apache.sqoop.handler.SubmissionRequestHandler.submissionSubmit(SubmissionRequestHandler.java:112)
>> at
>> org.apache.sqoop.handler.SubmissionRequestHandler.handleActionEvent(SubmissionRequestHandler.java:98)
>>  at
>> org.apache.sqoop.handler.SubmissionRequestHandler.handleEvent(SubmissionRequestHandler.java:68)
>> at
>> org.apache.sqoop.server.v1.SubmissionServlet.handlePostRequest(SubmissionServlet.java:44)
>>  at
>> org.apache.sqoop.server.SqoopProtocolServlet.doPost(SqoopProtocolServlet.java:63)
>> at javax.servlet.http.HttpServlet.service(HttpServlet.java:637)
>>  at javax.servlet.http.HttpServlet.service(HttpServlet.java:717)
>> at
>> org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)
>>  at
>> org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
>> at
>> org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:233)
>>  at
>> org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:191)
>> at
>> org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:127)
>>  at
>> org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103)
>> at
>> org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)
>>  at
>> org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:293)
>> at
>> org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:861)
>>  at
>> org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.process(Http11Protocol.java:606)
>> at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:489)
>>  at java.lang.Thread.run(Thread.java:724)
>> Caused by: java.lang.Throwable: Communications link failure
>>
>> The last packet sent successfully to the server was 0 milliseconds ago.
>> The driver has not received any packets from the server.
>>  at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
>> at
>> sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)
>>  at
>> sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
>> at java.lang.reflect.Constructor.newInstance(Constructor.java:526)
>>  at com.mysql.jdbc.Util.handleNewInstance(Util.java:411)
>> at
>> com.mysql.jdbc.SQLError.createCommunicationsException(SQLError.java:1121)
>>  at com.mysql.jdbc.MysqlIO.<init>(MysqlIO.java:355)
>> at com.mysql.jdbc.ConnectionImpl.coreConnect(ConnectionImpl.java:2479)
>>  at
>> com.mysql.jdbc.ConnectionImpl.connectOneTryOnly(ConnectionImpl.java:2516)
>> at com.mysql.jdbc.ConnectionImpl.createNewIO(ConnectionImpl.java:2301)
>>  at com.mysql.jdbc.ConnectionImpl.<init>(ConnectionImpl.java:834)
>> at com.mysql.jdbc.JDBC4Connection.<init>(JDBC4Connection.java:47)
>>  at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
>> at
>> sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)
>>  at
>> sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
>> at java.lang.reflect.Constructor.newInstance(Constructor.java:526)
>>  at com.mysql.jdbc.Util.handleNewInstance(Util.java:411)
>> at com.mysql.jdbc.ConnectionImpl.getInstance(ConnectionImpl.java:416)
>>  at
>> com.mysql.jdbc.NonRegisteringDriver.connect(NonRegisteringDriver.java:317)
>> at java.sql.DriverManager.getConnection(DriverManager.java:571)
>>  at java.sql.DriverManager.getConnection(DriverManager.java:215)
>> at
>> org.apache.sqoop.connector.jdbc.GenericJdbcExecutor.<init>(GenericJdbcExecutor.java:40)
>>  ... 23 more
>> Caused by: java.net.ConnectException: Connection refused
>> at java.net.PlainSocketImpl.socketConnect(Native Method)
>>  at
>> java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)
>> at
>> java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)
>>  at
>> java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)
>> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
>>  at java.net.Socket.connect(Socket.java:579)
>> at java.net.Socket.connect(Socket.java:528)
>> at java.net.Socket.<init>(Socket.java:425)
>>  at java.net.Socket.<init>(Socket.java:241)
>> at
>> com.mysql.jdbc.StandardSocketFactory.connect(StandardSocketFactory.java:259)
>>  at com.mysql.jdbc.MysqlIO.<init>(MysqlIO.java:305)
>> ... 38 more
>>
>>
>>  Thanks,
>>
>>        Manish
>>
>>
>> On Mon, Sep 23, 2013 at 11:11 PM, Abraham Elmahrek <abe@cloudera.com>wrote:
>>
>>> Hey Manish,
>>>
>>> It looks like your job is using a connection that is pointing to a
>>> database server that is down or doesn't exist. Could you validate your
>>> connection parameters and verify you're using the correct connection in the
>>> job? Please provide the job and connection details via the "show
>>> connection" and "show job" commands. That will definitely assist us in
>>> figuring out what went wrong in your setup.
>>>
>>> -Abe
>>>
>>>
>>> On Mon, Sep 23, 2013 at 3:45 AM, Manish Meshram <
>>> manishmeshram36@gmail.com> wrote:
>>>
>>>> Hi,
>>>>
>>>> I am using sqoop Client API for importing rdbms data to hdfs.
>>>> I am using Hadoop 1.2.1(single node)
>>>> and sqoop 1.99.2
>>>> Hadoop is remote for me installed on CentOS and I am accessing it
>>>> through putty.
>>>>
>>>> I have gone through Sqoop client API guide -
>>>> http://sqoop.apache.org/docs/1.99.2/ClientAPI.html
>>>>
>>>> As there are 3 steps in total:
>>>> 1. Create connection.....(done)
>>>> 2. Create Job..........(done)
>>>> 3. Submit Job.........(GETTING ERROR)
>>>>
>>>> Steps 1 and 2 are working fine for me.
>>>> But during Step 3 i.e. while submitting the job I am getting the
>>>> following exception
>>>> in my eclipse IDE:
>>>>
>>>>
>>>>  org.apache.sqoop.common.SqoopException: CLIENT_0006:Server has
>>>> returned exception
>>>> at
>>>> org.apache.sqoop.client.request.Request$ServerExceptionFilter.handle(Request.java:102)
>>>>  at com.sun.jersey.api.client.WebResource.handle(WebResource.java:670)
>>>> at com.sun.jersey.api.client.WebResource.access$200(WebResource.java:74)
>>>>  at
>>>> com.sun.jersey.api.client.WebResource$Builder.post(WebResource.java:563)
>>>> at org.apache.sqoop.client.request.Request.post(Request.java:66)
>>>>  at
>>>> org.apache.sqoop.client.request.SubmissionRequest.create(SubmissionRequest.java:46)
>>>> at
>>>> org.apache.sqoop.client.request.SqoopRequests.createSubmission(SqoopRequests.java:134)
>>>>  at
>>>> org.apache.sqoop.client.SqoopClient.startSubmission(SqoopClient.java:353)
>>>> at com.TestSqoopClient.main(TestSqoopClient.java:126)
>>>> Caused by: java.lang.Throwable: GENERIC_JDBC_CONNECTOR_0001:Unable to
>>>> get a connection
>>>> at
>>>> org.apache.sqoop.connector.jdbc.GenericJdbcExecutor.<init>(GenericJdbcExecutor.java:47)
>>>>  at
>>>> org.apache.sqoop.connector.jdbc.GenericJdbcImportInitializer.configureJdbcProperties(GenericJdbcImportInitializer.java:73)
>>>> at
>>>> org.apache.sqoop.connector.jdbc.GenericJdbcImportInitializer.initialize(GenericJdbcImportInitializer.java:46)
>>>>  at
>>>> org.apache.sqoop.connector.jdbc.GenericJdbcImportInitializer.initialize(GenericJdbcImportInitializer.java:37)
>>>> at
>>>> org.apache.sqoop.framework.FrameworkManager.submit(FrameworkManager.java:434)
>>>>  at
>>>> org.apache.sqoop.handler.SubmissionRequestHandler.submissionSubmit(SubmissionRequestHandler.java:112)
>>>> at
>>>> org.apache.sqoop.handler.SubmissionRequestHandler.handleActionEvent(SubmissionRequestHandler.java:98)
>>>>  at
>>>> org.apache.sqoop.handler.SubmissionRequestHandler.handleEvent(SubmissionRequestHandler.java:68)
>>>> at
>>>> org.apache.sqoop.server.v1.SubmissionServlet.handlePostRequest(SubmissionServlet.java:44)
>>>>  at
>>>> org.apache.sqoop.server.SqoopProtocolServlet.doPost(SqoopProtocolServlet.java:63)
>>>> at javax.servlet.http.HttpServlet.service(HttpServlet.java:637)
>>>>  at javax.servlet.http.HttpServlet.service(HttpServlet.java:717)
>>>> at
>>>> org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)
>>>>  at
>>>> org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
>>>> at
>>>> org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:233)
>>>>  at
>>>> org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:191)
>>>> at
>>>> org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:127)
>>>>  at
>>>> org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103)
>>>> at
>>>> org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)
>>>>  at
>>>> org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:293)
>>>> at
>>>> org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:861)
>>>>  at
>>>> org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.process(Http11Protocol.java:606)
>>>> at
>>>> org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:489)
>>>>  at java.lang.Thread.run(Thread.java:724)
>>>> Caused by: java.lang.Throwable: Communications link failure
>>>>
>>>> The last packet sent successfully to the server was 0 milliseconds ago.
>>>> The driver has not received any packets from the server.
>>>>  at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native
>>>> Method)
>>>> at
>>>> sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)
>>>>  at
>>>> sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
>>>> at java.lang.reflect.Constructor.newInstance(Constructor.java:526)
>>>>  at com.mysql.jdbc.Util.handleNewInstance(Util.java:411)
>>>> at
>>>> com.mysql.jdbc.SQLError.createCommunicationsException(SQLError.java:1121)
>>>>  at com.mysql.jdbc.MysqlIO.<init>(MysqlIO.java:355)
>>>> at com.mysql.jdbc.ConnectionImpl.coreConnect(ConnectionImpl.java:2479)
>>>>  at
>>>> com.mysql.jdbc.ConnectionImpl.connectOneTryOnly(ConnectionImpl.java:2516)
>>>> at com.mysql.jdbc.ConnectionImpl.createNewIO(ConnectionImpl.java:2301)
>>>>  at com.mysql.jdbc.ConnectionImpl.<init>(ConnectionImpl.java:834)
>>>> at com.mysql.jdbc.JDBC4Connection.<init>(JDBC4Connection.java:47)
>>>>  at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native
>>>> Method)
>>>> at
>>>> sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)
>>>>  at
>>>> sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
>>>> at java.lang.reflect.Constructor.newInstance(Constructor.java:526)
>>>>  at com.mysql.jdbc.Util.handleNewInstance(Util.java:411)
>>>> at com.mysql.jdbc.ConnectionImpl.getInstance(ConnectionImpl.java:416)
>>>>  at
>>>> com.mysql.jdbc.NonRegisteringDriver.connect(NonRegisteringDriver.java:317)
>>>> at java.sql.DriverManager.getConnection(DriverManager.java:571)
>>>>  at java.sql.DriverManager.getConnection(DriverManager.java:215)
>>>> at
>>>> org.apache.sqoop.connector.jdbc.GenericJdbcExecutor.<init>(GenericJdbcExecutor.java:40)
>>>>  ... 23 more
>>>> Caused by: java.net.ConnectException: Connection refused
>>>> at java.net.PlainSocketImpl.socketConnect(Native Method)
>>>>  at
>>>> java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)
>>>> at
>>>> java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)
>>>>  at
>>>> java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)
>>>> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
>>>>  at java.net.Socket.connect(Socket.java:579)
>>>> at java.net.Socket.connect(Socket.java:528)
>>>> at java.net.Socket.<init>(Socket.java:425)
>>>>  at java.net.Socket.<init>(Socket.java:241)
>>>> at
>>>> com.mysql.jdbc.StandardSocketFactory.connect(StandardSocketFactory.java:259)
>>>>  at com.mysql.jdbc.MysqlIO.<init>(MysqlIO.java:305)
>>>> ... 38 more
>>>>
>>>
>>>
>>
>

Mime
View raw message