sqoop-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From 翟玉勇 (JIRA) <j...@apache.org>
Subject [jira] [Commented] (SQOOP-3180) when mysql one column must not but insert has null map hang
Date Fri, 05 May 2017 08:11:04 GMT

    [ https://issues.apache.org/jira/browse/SQOOP-3180?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15997911#comment-15997911
] 

翟玉勇 commented on SQOOP-3180:
----------------------------

{code}
Full thread dump Java HotSpot(TM) 64-Bit Server VM (24.55-b03 mixed mode):

"Attach Listener" daemon prio=10 tid=0x00007fcac4004000 nid=0x4d91 waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"Abandoned connection cleanup thread" daemon prio=10 tid=0x00007fcad19d7000 nid=0x3f25 in
Object.wait() [0x00007fcab1d36000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
	at java.lang.Object.wait(Native Method)
	at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135)
	- locked <0x00000000f4fae480> (a java.lang.ref.ReferenceQueue$Lock)
	at com.mysql.jdbc.AbandonedConnectionCleanupThread.run(AbandonedConnectionCleanupThread.java:41)

"org.apache.hadoop.hdfs.PeerCache@7051242d" daemon prio=10 tid=0x00007fcad1b3a000 nid=0x3f20
waiting on condition [0x00007fcab1e37000]
   java.lang.Thread.State: TIMED_WAITING (sleeping)
	at java.lang.Thread.sleep(Native Method)
	at org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:255)
	at org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46)
	at org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124)
	at java.lang.Thread.run(Thread.java:745)

"communication thread" daemon prio=10 tid=0x00007fcad196a000 nid=0x3f0a in Object.wait() [0x00007fcab2039000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
	at java.lang.Object.wait(Native Method)
	at org.apache.hadoop.mapred.Task$TaskReporter.run(Task.java:765)
	- locked <0x00000000f2007658> (a java.lang.Object)
	at java.lang.Thread.run(Thread.java:745)

"Thread-6" daemon prio=10 tid=0x00007fcad198c000 nid=0x3f09 runnable [0x00007fcab213a000]
   java.lang.Thread.State: RUNNABLE
	at org.apache.hadoop.net.unix.DomainSocketWatcher.doPoll0(Native Method)
	at org.apache.hadoop.net.unix.DomainSocketWatcher.access$900(DomainSocketWatcher.java:52)
	at org.apache.hadoop.net.unix.DomainSocketWatcher$2.run(DomainSocketWatcher.java:509)
	at java.lang.Thread.run(Thread.java:745)

"Thread for syncLogs" daemon prio=10 tid=0x00007fcad135c800 nid=0x3ee1 waiting on condition
[0x00007fcab223b000]
   java.lang.Thread.State: TIMED_WAITING (parking)
	at sun.misc.Unsafe.park(Native Method)
	- parking to wait for  <0x00000000fc059358> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
	at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090)
	at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807)
	at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	at java.lang.Thread.run(Thread.java:745)

"org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner" daemon prio=10
tid=0x00007fcad0e8c000 nid=0x3ea1 in Object.wait() [0x00007fcab2347000]
   java.lang.Thread.State: WAITING (on object monitor)
	at java.lang.Object.wait(Native Method)
	- waiting on <0x00000000fc020518> (a java.lang.ref.ReferenceQueue$Lock)
	at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135)
	- locked <0x00000000fc020518> (a java.lang.ref.ReferenceQueue$Lock)
	at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151)
	at org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:3093)
	at java.lang.Thread.run(Thread.java:745)

"IPC Parameter Sending Thread #0" daemon prio=10 tid=0x00007fcad1012800 nid=0x3e72 waiting
on condition [0x00007fcab2448000]
   java.lang.Thread.State: TIMED_WAITING (parking)
	at sun.misc.Unsafe.park(Native Method)
	- parking to wait for  <0x00000000fc058908> (a java.util.concurrent.SynchronousQueue$TransferStack)
	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
	at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
	at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:359)
	at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:942)
	at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	at java.lang.Thread.run(Thread.java:745)

"IPC Client (1575367607) connection to /10.0.250.175:55571 from job_1493953508663_14532" daemon
prio=10 tid=0x00007fcad0fc9000 nid=0x3e71 in Object.wait() [0x00007fcab2549000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
	at java.lang.Object.wait(Native Method)
	at org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:929)
	- locked <0x00000000fc028870> (a org.apache.hadoop.ipc.Client$Connection)
	at org.apache.hadoop.ipc.Client$Connection.run(Client.java:974)

"Timer for 'MapTask' metrics system" daemon prio=10 tid=0x00007fcad1893000 nid=0x3e3d in Object.wait()
[0x00007fcab2a96000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
	at java.lang.Object.wait(Native Method)
	at java.util.TimerThread.mainLoop(Timer.java:552)
	- locked <0x00000000fc040070> (a java.util.TaskQueue)
	at java.util.TimerThread.run(Timer.java:505)

"Service Thread" daemon prio=10 tid=0x00007fcad00ca000 nid=0x3d2a runnable [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C2 CompilerThread1" daemon prio=10 tid=0x00007fcad00c7800 nid=0x3d29 waiting on condition
[0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C2 CompilerThread0" daemon prio=10 tid=0x00007fcad00c4000 nid=0x3d28 waiting on condition
[0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"Signal Dispatcher" daemon prio=10 tid=0x00007fcad00ba000 nid=0x3d27 runnable [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"Finalizer" daemon prio=10 tid=0x00007fcad00a3800 nid=0x3d1d in Object.wait() [0x00007fcabbcfb000]
   java.lang.Thread.State: WAITING (on object monitor)
	at java.lang.Object.wait(Native Method)
	- waiting on <0x00000000fc0011b8> (a java.lang.ref.ReferenceQueue$Lock)
	at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135)
	- locked <0x00000000fc0011b8> (a java.lang.ref.ReferenceQueue$Lock)
	at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151)
	at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:189)

"Reference Handler" daemon prio=10 tid=0x00007fcad009f000 nid=0x3d1c in Object.wait() [0x00007fcabbdfc000]
   java.lang.Thread.State: WAITING (on object monitor)
	at java.lang.Object.wait(Native Method)
	- waiting on <0x00000000fc000b78> (a java.lang.ref.Reference$Lock)
	at java.lang.Object.wait(Object.java:503)
	at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:133)
	- locked <0x00000000fc000b78> (a java.lang.ref.Reference$Lock)

"main" prio=10 tid=0x00007fcad0014000 nid=0x3d03 waiting on condition [0x00007fcad53a8000]
   java.lang.Thread.State: WAITING (parking)
	at sun.misc.Unsafe.park(Native Method)
	- parking to wait for  <0x00000000f5895060> (a java.util.concurrent.SynchronousQueue$TransferStack)
	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
	at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458)
	at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:359)
	at java.util.concurrent.SynchronousQueue.put(SynchronousQueue.java:878)
	at org.apache.sqoop.mapreduce.AsyncSqlOutputFormat$AsyncSqlExecThread.put(AsyncSqlOutputFormat.java:305)
	at org.apache.sqoop.mapreduce.AsyncSqlRecordWriter.execUpdate(AsyncSqlRecordWriter.java:161)
	at org.apache.sqoop.mapreduce.AsyncSqlRecordWriter.close(AsyncSqlRecordWriter.java:184)
	at org.apache.hadoop.mapred.MapTask$NewDirectOutputCollector.close(MapTask.java:670)
	at org.apache.hadoop.mapred.MapTask.closeQuietly(MapTask.java:2024)
	at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:797)
	at org.apache.hadoop.mapred.MapTask.run(MapTask.java:341)
	at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:164)
	at java.security.AccessController.doPrivileged(Native Method)
	at javax.security.auth.Subject.doAs(Subject.java:415)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1693)
	at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158)

"VM Thread" prio=10 tid=0x00007fcad009d000 nid=0x3d1b runnable 
{code}

> when mysql one column must not but insert has null map hang 
> ------------------------------------------------------------
>
>                 Key: SQOOP-3180
>                 URL: https://issues.apache.org/jira/browse/SQOOP-3180
>             Project: Sqoop
>          Issue Type: Bug
>    Affects Versions: 1.4.4
>            Reporter: 翟玉勇
>
> {code}
> 2017-05-05 14:35:55,301 INFO [main] org.apache.hadoop.metrics2.impl.MetricsConfig: loaded
properties from hadoop-metrics2.properties
> 2017-05-05 14:35:55,377 INFO [main] org.apache.hadoop.metrics2.impl.MetricsSystemImpl:
Scheduled snapshot period at 10 second(s).
> 2017-05-05 14:35:55,377 INFO [main] org.apache.hadoop.metrics2.impl.MetricsSystemImpl:
MapTask metrics system started
> 2017-05-05 14:35:55,385 INFO [main] org.apache.hadoop.mapred.YarnChild: Executing with
tokens:
> 2017-05-05 14:35:55,385 INFO [main] org.apache.hadoop.mapred.YarnChild: Kind: mapreduce.job,
Service: job_1493953508663_14532, Ident: (org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier@692b53b4)
> 2017-05-05 14:35:55,478 INFO [main] org.apache.hadoop.mapred.YarnChild: Sleeping for
0ms before retrying again. Got null now.
> 2017-05-05 14:35:55,806 INFO [main] org.apache.hadoop.mapred.YarnChild: mapreduce.cluster.local.dir
for child: /data1/nodemanager/usercache/master/appcache/application_1493953508663_14532,/data2/nodemanager/usercache/master/appcache/application_1493953508663_14532,/data3/nodemanager/usercache/master/appcache/application_1493953508663_14532,/data4/nodemanager/usercache/master/appcache/application_1493953508663_14532,/data5/nodemanager/usercache/master/appcache/application_1493953508663_14532,/data6/nodemanager/usercache/master/appcache/application_1493953508663_14532,/data7/nodemanager/usercache/master/appcache/application_1493953508663_14532,/data8/nodemanager/usercache/master/appcache/application_1493953508663_14532,/data9/nodemanager/usercache/master/appcache/application_1493953508663_14532,/data10/nodemanager/usercache/master/appcache/application_1493953508663_14532,/data11/nodemanager/usercache/master/appcache/application_1493953508663_14532,/data12/nodemanager/usercache/master/appcache/application_1493953508663_14532
> 2017-05-05 14:35:56,388 INFO [main] org.apache.hadoop.conf.Configuration.deprecation:
session.id is deprecated. Instead, use dfs.metrics.session-id
> 2017-05-05 14:35:56,919 INFO [main] org.apache.hadoop.mapred.Task:  Using ResourceCalculatorProcessTree
: [ ]
> 2017-05-05 14:35:57,229 INFO [main] org.apache.hadoop.mapred.MapTask: Processing split:
Paths:/data/sqoop_buffer/export_buffer/tmp_st_tms_typecode_bod_hummerteam_sale_20170505_143426_5424/000000_0:0+136738
> 2017-05-05 14:35:57,235 INFO [main] org.apache.hadoop.conf.Configuration.deprecation:
map.input.file is deprecated. Instead, use mapreduce.map.input.file
> 2017-05-05 14:35:57,235 INFO [main] org.apache.hadoop.conf.Configuration.deprecation:
map.input.start is deprecated. Instead, use mapreduce.map.input.start
> 2017-05-05 14:35:57,235 INFO [main] org.apache.hadoop.conf.Configuration.deprecation:
map.input.length is deprecated. Instead, use mapreduce.map.input.length
> 2017-05-05 14:35:57,685 INFO [Thread-12] org.apache.sqoop.mapreduce.AutoProgressMapper:
Auto-progress thread is finished. keepGoing=false
> 2017-05-05 14:35:57,695 INFO [Thread-11] org.apache.sqoop.mapreduce.AsyncSqlOutputFormat:
Start AsyncSqlExecThread thread to insert data
> 2017-05-05 14:35:57,695 INFO [main] org.apache.sqoop.mapreduce.AsyncSqlRecordWriter:
Start execUpdate,commit is true,stopThread is true,records length is 98batch is false
> 2017-05-05 14:35:57,923 INFO [main] org.apache.sqoop.mapreduce.AsyncSqlOutputFormat:
Init AsyncDBOperation: isBatch is false,commit is true,stopThread is true
> 2017-05-05 14:35:57,987 INFO [Thread-11] org.apache.sqoop.mapreduce.AsyncSqlOutputFormat:
Insert thread stmt is com.mysql.jdbc.JDBC4PreparedStatement@51072bc2: INSERT INTO st_tms_typecode_bod_hummerteam_sale
(column)values(my data)
> 2017-05-05 14:35:57,988 INFO [Thread-11] org.apache.sqoop.mapreduce.AsyncSqlOutputFormat:
Start try block
> 2017-05-05 14:35:57,988 INFO [Thread-11] org.apache.sqoop.mapreduce.AsyncSqlOutputFormat:
Execute without batch mode
> 2017-05-05 14:35:58,052 ERROR [Thread-11] org.apache.sqoop.mapreduce.AsyncSqlOutputFormat:
ERROR 3
> 2017-05-05 14:35:58,052 ERROR [Thread-11] org.apache.sqoop.mapreduce.AsyncSqlOutputFormat:
Got exception in update thread: com.mysql.jdbc.exceptions.jdbc4.MySQLIntegrityConstraintViolationException:
Column 'total_pleased_rate_count' cannot be null
> 	at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
> 	at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)
> 	at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
> 	at java.lang.reflect.Constructor.newInstance(Constructor.java:526)
> 	at com.mysql.jdbc.Util.handleNewInstance(Util.java:408)
> 	at com.mysql.jdbc.Util.getInstance(Util.java:383)
> 	at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:1049)
> 	at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:4226)
> 	at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:4158)
> 	at com.mysql.jdbc.MysqlIO.sendCommand(MysqlIO.java:2615)
> 	at com.mysql.jdbc.MysqlIO.sqlQueryDirect(MysqlIO.java:2776)
> 	at com.mysql.jdbc.ConnectionImpl.execSQL(ConnectionImpl.java:2840)
> 	at com.mysql.jdbc.PreparedStatement.executeInternal(PreparedStatement.java:2082)
> 	at com.mysql.jdbc.PreparedStatement.execute(PreparedStatement.java:1302)
> 	at org.apache.sqoop.mapreduce.AsyncSqlOutputFormat$AsyncSqlExecThread.run(AsyncSqlOutputFormat.java:243)
> 2017-05-05 14:35:58,056 INFO [main] org.apache.sqoop.mapreduce.AsyncSqlRecordWriter:
Start execUpdate,commit is true,stopThread is true,records length is 0batch is false
> 2017-05-05 14:35:58,056 INFO [main] org.apache.sqoop.mapreduce.AsyncSqlOutputFormat:
Init AsyncDBOperation: isBatch is false,commit is true,stopThread is true
> {code}



--
This message was sent by Atlassian JIRA
(v6.3.15#6346)

Mime
View raw message