cloudstack-users-cn mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From asicando <asica...@gmail.com>
Subject 回复: Re: 主存储宕机,导致实例无法启动
Date Mon, 28 Jan 2013 05:13:05 GMT
该host是DELL R720 centos-6.3-x86_64 kvm
这个是cs日志文件
2013-01-28 12:39:35,414 DEBUG [agent.transport.Request] (Job-Executor-142:job-302) Seq 15-1278086023:
Sending  { Cmd , MgmtId: 90520734270602, via: 15, Ver: v1, Flags: 100111, [{"routing.SavePasswordCommand":{"password":"fnirq_cnffjbeq","vmIpAddress":"192.168.35.31","vmName":"hadoop-test-01","accessDetails":{"router.guest.ip":"192.168.35.48","zone.network.type":"Basic","router.ip":"169.254.1.174","router.name":"r-41-VM"},"wait":0}},{"routing.VmDataCommand":{"vmIpAddress":"192.168.35.31","vmName":"hadoop-test-01","accessDetails":{"router.guest.ip":"192.168.35.48","zone.network.type":"Basic","router.ip":"169.254.1.174","router.name":"r-41-VM"},"wait":0}}]
}
2013-01-28 12:39:35,536 DEBUG [agent.transport.Request] (StatsCollector-1:null) Seq 11-2114980761:
Received:  { Ans: , MgmtId: 90520734270602, via: 11, Ver: v1, Flags: 10, { GetHostStatsAnswer
} }
2013-01-28 12:39:35,788 DEBUG [cloud.async.AsyncJobManagerImpl] (catalina-exec-1:null) Async
job-293 completed
2013-01-28 12:39:36,164 DEBUG [agent.transport.Request] (StatsCollector-1:null) Seq 13-1921256474:
Received:  { Ans: , MgmtId: 90520734270602, via: 13, Ver: v1, Flags: 10, { GetHostStatsAnswer
} }
2013-01-28 12:39:36,326 DEBUG [agent.transport.Request] (StatsCollector-3:null) Seq 11-2114980762:
Received:  { Ans: , MgmtId: 90520734270602, via: 11, Ver: v1, Flags: 10, { GetVmStatsAnswer
} }
2013-01-28 12:39:36,753 DEBUG [agent.transport.Request] (StatsCollector-1:null) Seq 15-1278086024:
Received:  { Ans: , MgmtId: 90520734270602, via: 15, Ver: v1, Flags: 10, { GetHostStatsAnswer
} }
2013-01-28 12:39:36,768 DEBUG [agent.manager.DirectAgentAttache] (DirectAgent-347:null) Seq
16-258541398: Executing request
2013-01-28 12:39:37,087 DEBUG [agent.manager.DirectAgentAttache] (DirectAgent-347:null) Seq
16-258541398: Response Received: 
2013-01-28 12:39:37,088 DEBUG [agent.transport.Request] (StatsCollector-1:null) Seq 16-258541398:
Received:  { Ans: , MgmtId: 90520734270602, via: 16, Ver: v1, Flags: 10, { GetHostStatsAnswer
} }
2013-01-28 12:39:37,679 DEBUG [agent.transport.Request] (StatsCollector-3:null) Seq 13-1921256475:
Received:  { Ans: , MgmtId: 90520734270602, via: 13, Ver: v1, Flags: 10, { GetVmStatsAnswer
} }
2013-01-28 12:39:37,743 DEBUG [cloud.async.AsyncJobManagerImpl] (catalina-exec-21:null) Async
job-280 completed
2013-01-28 12:39:37,848 DEBUG [agent.transport.Request] (StatsCollector-3:null) Seq 15-1278086025:
Received:  { Ans: , MgmtId: 90520734270602, via: 15, Ver: v1, Flags: 10, { GetVmStatsAnswer
} }
2013-01-28 12:39:37,855 DEBUG [cloud.async.AsyncJobManagerImpl] (catalina-exec-25:null) Async
job-291 completed
2013-01-28 12:39:40,913 DEBUG [cloud.async.AsyncJobManagerImpl] (catalina-exec-7:null) Async
job-293 completed
2013-01-28 12:39:41,500 DEBUG [cloud.server.StatsCollector] (StatsCollector-2:null) StorageCollector
is running...
2013-01-28 12:39:41,564 DEBUG [agent.transport.Request] (StatsCollector-2:null) Seq 19-1004538187:
Received:  { Ans: , MgmtId: 90520734270602, via: 19, Ver: v1, Flags: 10, { GetStorageStatsAnswer
} }
2013-01-28 12:39:41,702 DEBUG [agent.transport.Request] (StatsCollector-2:null) Seq 11-2114980763:
Received:  { Ans: , MgmtId: 90520734270602, via: 11, Ver: v1, Flags: 10, { GetStorageStatsAnswer
} }
2013-01-28 12:39:41,736 DEBUG [agent.transport.Request] (StatsCollector-2:null) Seq 8-1352207496:
Received:  { Ans: , MgmtId: 90520734270602, via: 8, Ver: v1, Flags: 10, { GetStorageStatsAnswer
} }
2013-01-28 12:39:41,758 DEBUG [agent.transport.Request] (StatsCollector-2:null) Seq 11-2114980764:
Received:  { Ans: , MgmtId: 90520734270602, via: 11, Ver: v1, Flags: 10, { GetStorageStatsAnswer
} }
2013-01-28 12:39:41,779 DEBUG [agent.transport.Request] (StatsCollector-2:null) Seq 13-1921256476:
Received:  { Ans: , MgmtId: 90520734270602, via: 13, Ver: v1, Flags: 10, { GetStorageStatsAnswer
} }
2013-01-28 12:39:41,803 DEBUG [agent.transport.Request] (StatsCollector-2:null) Seq 15-1278086026:
Received:  { Ans: , MgmtId: 90520734270602, via: 15, Ver: v1, Flags: 10, { GetStorageStatsAnswer
} }
2013-01-28 12:39:41,825 DEBUG [agent.manager.DirectAgentAttache] (DirectAgent-7:null) Seq
16-258541399: Executing request
2013-01-28 12:39:42,331 DEBUG [agent.manager.DirectAgentAttache] (DirectAgent-7:null) Seq
16-258541399: Response Received: 
2013-01-28 12:39:42,332 DEBUG [agent.transport.Request] (StatsCollector-2:null) Seq 16-258541399:
Received:  { Ans: , MgmtId: 90520734270602, via: 16, Ver: v1, Flags: 10, { GetStorageStatsAnswer
} }
2013-01-28 12:39:41,825 DEBUG [agent.manager.DirectAgentAttache] (DirectAgent-7:null) Seq
16-258541399: Executing request
2013-01-28 12:39:42,331 DEBUG [agent.manager.DirectAgentAttache] (DirectAgent-7:null) Seq
16-258541399: Response Received: 
2013-01-28 12:39:42,332 DEBUG [agent.transport.Request] (StatsCollector-2:null) Seq 16-258541399:
Received:  { Ans: , MgmtId: 90520734270602, via: 16, Ver: v1, Flags: 10, { GetStorageStatsAnswer
} }
2013-01-28 12:39:42,613 DEBUG [agent.transport.Request] (AgentManager-Handler-3:null) Seq
15-1278086023: Processing:  { Ans: , MgmtId: 90520734270602, via: 15, Ver: v1, Flags: 110,
[{"Answer":{"result":true,"wait":0}},{"Answer":{"result":true,"wait":0}}] }
2013-01-28 12:39:42,613 DEBUG [agent.manager.AgentAttache] (AgentManager-Handler-3:null) Seq
15-1278086023: No more commands found
2013-01-28 12:39:42,613 DEBUG [agent.transport.Request] (Job-Executor-142:job-302) Seq 15-1278086023:
Received:  { Ans: , MgmtId: 90520734270602, via: 15, Ver: v1, Flags: 110, { Answer, Answer
} }
2013-01-28 12:39:42,613 DEBUG [cloud.network.NetworkManagerImpl] (Job-Executor-142:job-302)
Asking Ovs to prepare for Nic[22-16-8797487a-d757-4c0e-b4b3-f2e9d6b3c5b9-192.168.35.31]
2013-01-28 12:39:42,613 DEBUG [cloud.network.NetworkManagerImpl] (Job-Executor-142:job-302)
Asking ExternalDhcpServer to prepare for Nic[22-16-8797487a-d757-4c0e-b4b3-f2e9d6b3c5b9-192.168.35.31]
2013-01-28 12:39:42,626 DEBUG [cloud.network.NetworkManagerImpl] (Job-Executor-142:job-302)
Asking BareMetal to prepare for Nic[22-16-8797487a-d757-4c0e-b4b3-f2e9d6b3c5b9-192.168.35.31]
2013-01-28 12:39:42,626 DEBUG [cloud.network.NetworkManagerImpl] (Job-Executor-142:job-302)
Asking SecurityGroupProvider to prepare for Nic[22-16-8797487a-d757-4c0e-b4b3-f2e9d6b3c5b9-192.168.35.31]
2013-01-28 12:39:42,637 DEBUG [cloud.network.NetworkManagerImpl] (Job-Executor-142:job-302)
Asking CiscoNexus1000vVSM to prepare for Nic[22-16-8797487a-d757-4c0e-b4b3-f2e9d6b3c5b9-192.168.35.31]
2013-01-28 12:39:42,637 DEBUG [cloud.network.NetworkManagerImpl] (Job-Executor-142:job-302)
Asking VpcVirtualRouter to prepare for Nic[22-16-8797487a-d757-4c0e-b4b3-f2e9d6b3c5b9-192.168.35.31]
2013-01-28 12:39:42,637 WARN  [network.element.VpcVirtualRouterElement] (Job-Executor-142:job-302)
Network Ntwk[204|Guest|6] is not associated with any VPC
2013-01-28 12:39:42,647 DEBUG [cloud.network.NetworkManagerImpl] (Job-Executor-142:job-302)
Asking NiciraNvp to prepare for Nic[22-16-8797487a-d757-4c0e-b4b3-f2e9d6b3c5b9-192.168.35.31]
2013-01-28 12:39:42,668 DEBUG [cloud.storage.StorageManagerImpl] (Job-Executor-142:job-302)
Checking if we need to prepare 1 volumes for VM[User|hadoop-test-01]
2013-01-28 12:39:42,668 DEBUG [cloud.storage.StorageManagerImpl] (Job-Executor-142:job-302)
No need to recreate the volume: Vol[23|vm=16|ROOT], since it already has a pool assigned:
206, adding disk to VM
2013-01-28 12:39:42,702 DEBUG [agent.transport.Request] (Job-Executor-142:job-302) Seq 8-1352207497:
Sending  { Cmd , MgmtId: 90520734270602, via: 8, Ver: v1, Flags: 100111, [{"StartCommand":{"vm":{"id":16,"name":"i-2-16-VM","type":"User","cpus":2,"speed":1000,"minRam":4194304000,"maxRam":4194304000,"arch":"x86_64","os":"CentOS
6.0 (64-bit)","bootArgs":"","rebootOnCrash":false,"enableHA":false,"limitCpuUse":false,"vncPassword":"a395ea36a972bb85","params":{},"uuid":"757769bf-cff6-427b-97ea-b95a77d9a047","disks":[{"id":23,"name":"ROOT-16","mountPoint":"/var/lib/libvirt/images/","path":"93fd06b1-1249-41f5-9f79-ae00dec0200b","size":32212254720,"type":"ROOT","storagePoolType":"Filesystem","storagePoolUuid":"f3f25da4-737a-4ca5-a627-eb2317da2259","deviceId":0},{"id":16,"name":"Centos-6.3-x86_64-template","size":0,"type":"ISO","storagePoolType":"ISO","deviceId":3}],"nics":[{"deviceId":0,"networkRateMbps":200,"defaultNic":true,"uuid":"6112665c-f63c-4851-b192-0b00fba919db","ip":"192.168.35.31","netmask":"255.255.255.0","gateway":"192.168.35.254","mac":"06:2a:20:00:00:22","dns1":"192.168.33.3","dns2":"","broadcastType":"Vlan","type":"Guest","broadcastUri":"vlan://untagged","isolationUri":"ec2://untagged","isSecurityGroupEnabled":false,"name":"cloudbr1"}]},"wait":0}}]
}
2013-01-28 12:39:42,869 DEBUG [cloud.async.AsyncJobManagerImpl] (catalina-exec-19:null) Async
job-280 completed
2013-01-28 12:39:42,996 DEBUG [cloud.async.AsyncJobManagerImpl] (catalina-exec-13:null) Async
job-291 completed
2013-01-28 12:39:43,399 DEBUG [agent.transport.Request] (AgentManager-Handler-6:null) Seq
8-1352207497: Processing:  { Ans: , MgmtId: 90520734270602, via: 8, Ver: v1, Flags: 110, [{"Answer":{"result":false,"details":"java.lang.NullPointerException\n\tat
com.cloud.hypervisor.kvm.resource.LibvirtComputingResource.cleanupVMNetworks(LibvirtComputingResource.java:3922)\n\tat
com.cloud.hypervisor.kvm.resource.LibvirtComputingResource.handleVmStartFailure(LibvirtComputingResource.java:2709)\n\tat
com.cloud.hypervisor.kvm.resource.LibvirtComputingResource.execute(LibvirtComputingResource.java:2834)\n\tat
com.cloud.hypervisor.kvm.resource.LibvirtComputingResource.executeRequest(LibvirtComputingResource.java:1035)\n\tat
com.cloud.agent.Agent.processRequest(Agent.java:518)\n\tat com.cloud.agent.Agent$AgentRequestHandler.doTask(Agent.java:831)\n\tat
com.cloud.utils.nio.Task.run(Task.java:83)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)\n\tat
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)\n\tat java.lang.Thread.run(Thread.java:679)\n","wait":0}}]
}
2013-01-28 12:39:43,399 DEBUG [agent.manager.AgentAttache] (AgentManager-Handler-6:null) Seq
8-1352207497: No more commands found
2013-01-28 12:39:43,399 DEBUG [agent.transport.Request] (Job-Executor-142:job-302) Seq 8-1352207497:
Received:  { Ans: , MgmtId: 90520734270602, via: 8, Ver: v1, Flags: 110, { Answer } }
2013-01-28 12:39:43,408 ERROR [cloud.vm.VirtualMachineManagerImpl] (Job-Executor-142:job-302)
Failed to start instance VM[User|hadoop-test-01]
com.cloud.utils.exception.CloudRuntimeException: Unable to get answer that is of class com.cloud.agent.api.StartAnswer
        at com.cloud.agent.manager.Commands.getAnswer(Commands.java:80)
        at com.cloud.vm.VirtualMachineManagerImpl.advanceStart(VirtualMachineManagerImpl.java:783)
        at com.cloud.vm.VirtualMachineManagerImpl.start(VirtualMachineManagerImpl.java:472)
        at com.cloud.vm.UserVmManagerImpl.startVirtualMachine(UserVmManagerImpl.java:2929)
        at com.cloud.vm.UserVmManagerImpl.startVirtualMachine(UserVmManagerImpl.java:1842)
        at com.cloud.event.ActionEventCallback.intercept(ActionEventCallback.java:36)
        at com.cloud.api.commands.StartVMCmd.execute(StartVMCmd.java:123)
        at com.cloud.api.ApiDispatcher.dispatch(ApiDispatcher.java:138)
        at com.cloud.async.AsyncJobManagerImpl$1.run(AsyncJobManagerImpl.java:432)
        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
        at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
        at java.util.concurrent.FutureTask.run(FutureTask.java:166)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
        at java.lang.Thread.run(Thread.java:679)
2013-01-28 12:39:43,416 DEBUG [cloud.vm.VirtualMachineManagerImpl] (Job-Executor-142:job-302)
Cleaning up resources for the vm VM[User|hadoop-test-01] in Starting state
2013-01-28 12:39:43,419 DEBUG [agent.transport.Request] (Job-Executor-142:job-302) Seq 8-1352207498:
Sending  { Cmd , MgmtId: 90520734270602, via: 8, Ver: v1, Flags: 100111, [{"StopCommand":{"isProxy":false,"vmName":"i-2-16-VM","wait":0}}]
}
2013-01-28 12:39:43,693 DEBUG [agent.transport.Request] (AgentManager-Handler-11:null) Seq
8-1352207498: Processing:  { Ans: , MgmtId: 90520734270602, via: 8, Ver: v1, Flags: 110, [{"StopAnswer":{"vncPort":0,"result":true,"wait":0}}]
}
2013-01-28 12:39:43,694 DEBUG [agent.manager.AgentAttache] (AgentManager-Handler-11:null)
Seq 8-1352207498: No more commands found
2013-01-28 12:39:43,694 DEBUG [agent.transport.Request] (Job-Executor-142:job-302) Seq 8-1352207498:
Received:  { Ans: , MgmtId: 90520734270602, via: 8, Ver: v1, Flags: 110, { StopAnswer } }

libvirt的log文件
2013-01-28 04:56:17.414+0000: 1332: error : virSecurityDACSetOwnership:125 : unable to set
user and group to '0:0' on '/var/lib/libvirt/images/9fc3070a-5ae6-42c2-a134-025c59a2153f':
No such file or directory
2013-01-28 04:56:17.614+0000: 1332: error : virSecurityDACRestoreSecurityFileLabel:143 : cannot
resolve symlink /var/lib/libvirt/images/9fc3070a-5ae6-42c2-a134-025c59a2153f: No such file
or directory
2013-01-28 04:56:17.614+0000: 1332: error : virSecurityDACRestoreSecurityFileLabel:143 : cannot
resolve symlink /mnt/c8f87069-c6c7-3e30-8ed2-49200d92eee4/9bb1dbc3-b58b-48fb-8746-b36177e64cb0:
No such file or directory
2013-01-28 04:56:45.365+0000: 1331: warning : qemuDomainObjTaint:1297 : Domain id=39 name='i-2-16-VM'
uuid=38480573-71c0-3a16-a05e-8c476d32a806 is tainted: high-privileges
2013-01-28 04:56:45.457+0000: 1331: error : virSecurityDACSetOwnership:125 : unable to set
user and group to '0:0' on '/var/lib/libvirt/images/9fc3070a-5ae6-42c2-a134-025c59a2153f':
No such file or directory
2013-01-28 04:56:45.657+0000: 1331: error : virSecurityDACRestoreSecurityFileLabel:143 : cannot
resolve symlink /var/lib/libvirt/images/9fc3070a-5ae6-42c2-a134-025c59a2153f: No such file
or directory
2013-01-28 04:56:45.657+0000: 1331: error : virSecurityDACRestoreSecurityFileLabel:143 : cannot
resolve symlink /mnt/c8f87069-c6c7-3e30-8ed2-49200d92eee4/9bb1dbc3-b58b-48fb-8746-b36177e64cb0:
No such file or directory

可是我在该实例所在的host里 cat 该虚拟机的配置文件 i-2-16-VM.xml
source file='/var/lib/libvirt/images/93fd06b1-1249-41f5-9f79-ae00dec0200b' 



Rick Luo 

发件人: tanthalas
发送时间: 2013-01-28 12:21
收件人: cloudstack-users-cn
主题: Re: 回复: Re: 主存储宕机,导致实例无法启动
给的log信息还不够,再往上翻翻。
这是什么类型的hypervisor?如果vm不能启动,一般会从hypervisor层返回错误信息。

2013-01-28 


刘宇超  Richard Liu




发件人: asicando 
发送时间: 2013-01-28  11:45:02 
收件人: cloudstack-users-cn 
抄送: 
主题: 回复: Re: 主存储宕机,导致实例无法启动 

恩是的,不知道为啥,都重启过,而且就那一台实例出问题无法启动,其他用本地存储的实例都没有问题。
Rick Luo 
发件人: 马营
发送时间: 2013-01-28 11:32
收件人: cloudstack-users-cn
主题: Re: 主存储宕机,导致实例无法启动
按理说,主存储宕机,本地存储的vm应该不受影响
或许也可以尝试一下重启agent
2013/1/28 linuxbqj@gmail.com <linuxbqj@gmail.com>
> 重启NFS和cloudstack 管理节点试试呢?
>
>
> 2013/1/28 asicando <asicando@gmail.com>
>
> > 大家好
> > cs-4.0,主nfs宕机,导致大部分vm停止,多数实例都能启动,只有一台实例无法启动,该实例使用本地存储,页面启动报错信息
> > Resource [Host:8] is unreachable: Host 8: Unable to start instance due to
> > Unable to get answer that is of class com.cloud.agent.api.StartAnswer
> > 日志信息如下:
> > 2013-01-28 11:13:43,178 ERROR [cloud.vm.VirtualMachineManagerImpl]
> > (Job-Executor-136:job-296) Failed to start instance
> VM[User|hadoop-test-01]
> > com.cloud.utils.exception.CloudRuntimeException: Unable to get answer
> that
> > is of class com.cloud.agent.api.StartAnswer
> >         at com.cloud.agent.manager.Commands.getAnswer(Commands.java:80)
> >         at
> >
> com.cloud.vm.VirtualMachineManagerImpl.advanceStart(VirtualMachineManagerImpl.java:783)
> >         at
> >
> com.cloud.vm.VirtualMachineManagerImpl.start(VirtualMachineManagerImpl.java:472)
> >         at
> >
> com.cloud.vm.UserVmManagerImpl.startVirtualMachine(UserVmManagerImpl.java:2929)
> >         at
> >
> com.cloud.vm.UserVmManagerImpl.startVirtualMachine(UserVmManagerImpl.java:1842)
> >         at
> >
> com.cloud.event.ActionEventCallback.intercept(ActionEventCallback.java:36)
> >         at com.cloud.api.commands.StartVMCmd.execute(StartVMCmd.java:123)
> >         at com.cloud.api.ApiDispatcher.dispatch(ApiDispatcher.java:138)
> >         at
> > com.cloud.async.AsyncJobManagerImpl$1.run(AsyncJobManagerImpl.java:432)
> >         at
> > java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
> >         at
> > java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
> >         at java.util.concurrent.FutureTask.run(FutureTask.java:166)
> >         at
> >
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
> >         at
> >
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
> >         at java.lang.Thread.run(Thread.java:679)
> > 2013-01-28 11:13:43,186 DEBUG [cloud.vm.VirtualMachineManagerImpl]
> > (Job-Executor-136:job-296) Cleaning up resources for the vm
> > VM[User|biansutao-test-01] in Starting state
> >
> >
> >
> >
> >
> > Rick Luo
>
>
>
>
> --
> 白清杰 (Born Bai)
>
> Beijing Open Vision Information Technology Co.,Ltd
>
> Mail: linuxbqj@gmail.com
>
Mime
  • Unnamed multipart/alternative (inline, None, 0 bytes)
View raw message