1.数据库启动集群报错
[root@db1 bin]# ./crs_stat -t -v Name Type R/RA F/FT Target State Host ---------------------------------------------------------------------- ora.BACKUP.dg ora....up.type 0/5 0/ ONLINE ONLINE db1 ora.DATA.dg ora....up.type 0/5 0/ ONLINE ONLINE db1 ora....ER.lsnr ora....er.type 0/5 0/ ONLINE OFFLINE ora....N1.lsnr ora....er.type 0/5 0/0 ONLINE OFFLINE ora.OCR.dg ora....up.type 0/5 0/ ONLINE ONLINE db1 ora.asm ora.asm.type 0/5 0/ ONLINE ONLINE db1 ora.cvu ora.cvu.type 0/5 0/0 ONLINE ONLINE db1 ora....SM1.asm application 0/5 0/0 ONLINE ONLINE db1 ora....B1.lsnr application 0/5 0/0 ONLINE OFFLINE ora.db1.gsd application 0/5 0/0 OFFLINE OFFLINE ora.db1.ons application 0/3 0/0 ONLINE OFFLINE ora.db1.vip ora....t1.type 0/0 0/0 ONLINE ONLINE db1 ora....SM2.asm application 0/5 0/0 ONLINE ONLINE db2 ora....B2.lsnr application 0/5 0/0 ONLINE OFFLINE ora.db2.gsd application 0/5 0/0 OFFLINE OFFLINE ora.db2.ons application 0/3 0/0 ONLINE OFFLINE ora.db2.vip ora....t1.type 0/0 0/0 ONLINE ONLINE db2 ora.gsd ora.gsd.type 0/5 0/ OFFLINE OFFLINE ora....network ora....rk.type 0/5 0/ ONLINE ONLINE db1 ora.oc4j ora.oc4j.type 0/1 0/2 ONLINE OFFLINE ora.ons ora.ons.type 0/3 0/ ONLINE OFFLINE ora.racdb.db ora....se.type 0/2 0/1 ONLINE OFFLINE ora....ry.acfs ora....fs.type 0/5 0/ ONLINE ONLINE db1 ora.scan1.vip ora....ip.type 0/0 0/0 ONLINE OFFLINE [root@db1 bin]# ./srvctl status listener Listener LISTENER is enabled Listener LISTENER is not running [root@db1 bin]# ./srvctl start listener PRCR-1079 : Failed to start resource ora.LISTENER.lsnr CRS-5016: Process "/u01/app/11.2.0/grid/bin/lsnrctl" spawned by agent "/u01/app/11.2.0/grid/bin/oraagent.bin" for action "start" failed: details at "(:CLSN00010:)" in "/u01/app/11.2.0/grid/log/db2/agent/crsd/oraagent_grid/oraagent_grid.log" CRS-5016: Process "/u01/app/11.2.0/grid/bin/lsnrctl" spawned by agent "/u01/app/11.2.0/grid/bin/oraagent.bin" for action "start" failed: details at "(:CLSN00010:)" in "/u01/app/11.2.0/grid/log/db1/agent/crsd/oraagent_grid/oraagent_grid.log" CRS-5016: Process "/u01/app/11.2.0/grid/bin/lsnrctl" spawned by agent "/u01/app/11.2.0/grid/bin/oraagent.bin" for action "start" failed: details at "(:CLSN00010:)" in "/u01/app/11.2.0/grid/log/db2/agent/crsd/oraagent_grid/oraagent_grid.log" CRS-5016: Process "/u01/app/11.2.0/grid/bin/lsnrctl" spawned by agent "/u01/app/11.2.0/grid/bin/oraagent.bin" for action "start" failed: details at "(:CLSN00010:)" in "/u01/app/11.2.0/grid/log/db1/agent/crsd/oraagent_grid/oraagent_grid.log" CRS-2674: Start of 'ora.LISTENER.lsnr' on 'db2' failed CRS-2674: Start of 'ora.LISTENER.lsnr' on 'db1' failed
[root@db1 oraagent_grid]# tail -50 oraagent_grid.log
2018-05-22 09:53:56.551: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] Adding Environment Variables TNS_ADMIN=/u01/app/11.2.0/grid/network/admin/
2018-05-22 09:53:56.551: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] Adding Environment variable from USR_ORA_ENV ORACLE_BASE=/u01/app/grid
2018-05-22 09:53:56.551: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] Utils:execCmd action = 3 flags = 38 ohome = (null) cmdname = lsnrctl.
2018-05-22 09:53:56.551: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] getOracleHomeAttrib: oracle_home = /u01/app/11.2.0/grid
2018-05-22 09:53:56.653: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:)
2018-05-22 09:53:56.653: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:)LSNRCTL for Linux: Version 11.2.0.4.0 - Production on 22-MAY-2018 09:53:56
2018-05-22 09:53:56.653: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:)
2018-05-22 09:53:56.653: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:)Copyright (c) 1991, 2013, Oracle. All rights reserved.
2018-05-22 09:53:56.653: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:)
2018-05-22 09:53:56.653: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:)Connecting to (DESCRIPTION=(ADDRESS=(PROTOCOL=IPC)(KEY=LISTENER)))
2018-05-22 09:53:56.653: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:)
2018-05-22 09:53:58.255: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:)TNS-12541: TNS:no listener
2018-05-22 09:53:58.255: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:) TNS-12560: TNS:protocol adapter error
2018-05-22 09:53:58.255: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:) TNS-00511: No listener
2018-05-22 09:53:58.255: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:) Linux Error: 111: Connection refused
2018-05-22 09:53:58.255: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:)
2018-05-22 09:53:58.256: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:)Utils:execCmd scls_process_join() uret 1
2018-05-22 09:53:58.256: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] execCmd ret = 1
2018-05-22 09:53:58.256: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:)
2018-05-22 09:53:58.256: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:)LSNRCTL for Linux: Version 11.2.0.4.0 - Production on 22-MAY-2018 09:53:56
2018-05-22 09:53:58.256: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:)
2018-05-22 09:53:58.256: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:)Copyright (c) 1991, 2013, Oracle. All rights reserved.
2018-05-22 09:53:58.256: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:)
2018-05-22 09:53:58.256: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:)Connecting to (DESCRIPTION=(ADDRESS=(PROTOCOL=IPC)(KEY=LISTENER)))
2018-05-22 09:53:58.256: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:)TNS-12541: TNS:no listener
2018-05-22 09:53:58.256: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:) TNS-12560: TNS:protocol adapter error
2018-05-22 09:53:58.256: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:) TNS-00511: No listener
2018-05-22 09:53:58.256: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:) Linux Error: 111: Connection refused
2018-05-22 09:53:58.256: [ora.LISTENER.lsnr][3503482624]{1:4489:169} [check] (:CLSN00010:)
2018-05-22 09:53:58.256: [ AGFW][3501381376]{1:4489:169} ora.LISTENER.lsnr db1 1 state changed from: STARTING to: OFFLINE
2018-05-22 09:53:58.257: [ AGFW][3501381376]{1:4489:169} Agent sending last reply for: RESOURCE_START[ora.LISTENER.lsnr db1 1] ID 4098:850
2018-05-22 09:54:07.297: [CLSFRAME][3522594592] TM [MultiThread] is changing desired thread # to 5. Current # is 6
2018-05-22 09:54:07.298: [CLSFRAME][2876454656]{0:1:8} Worker thread is exiting in TM [MultiThread] to meet the desired count of 5. New count is 5
2018-05-22 09:54:37.120: [ AGFW][3501381376]{1:4489:2} Agent received the message: AGENT_HB[Engine] ID 12293:956
2018-05-22 09:54:37.120: [ AGFW][3501381376]{1:4489:2} Deleting the resource: ora.LISTENER_SCAN1.lsnr 1 1
2018-05-22 09:54:37.272: [ora.asm][2882758400]{1:4489:2} [check] AsmProxyAgent:setAttrsFromSrcRes 2 m_oracleSid +ASM1 cmdId 260 checkType 3 flag 32
2018-05-22 09:54:37.272: [ora.asm][2882758400]{1:4489:2} [check] CrsCmd::ClscrsCmdData::stat entity 1 statflag 32 useFilter 0
2018-05-22 09:54:37.299: [ USRTHRD][2882758400]{1:4489:2} Thread:ASM DedicatedThreadstart {
2018-05-22 09:54:37.299: [ USRTHRD][2882758400]{1:4489:2} Thread:ASM DedicatedThreadstart }
2018-05-22 09:54:37.299: [ USRTHRD][2876454656]{1:4489:2} ASM Dedicated Thread {
2018-05-22 09:54:37.299: [ USRTHRD][2876454656]{1:4489:2} AsmProxyAgent::setGenUsrOraInst updating GEN_USR_ORA_INST_NAME@SERVERNAME(db1) to +ASM1
2018-05-22 09:54:37.299: [ora.asm][2882758400]{1:4489:2} [check] CrsCmd::ClscrsCmdData::stat entity 1 statflag 33 useFilter 0
2018-05-22 09:54:37.300: [ USRTHRD][2876454656]{1:4489:2} CrsCmd::ClscrsCmdData::stat entity 1 statflag 32 useFilter 0
2018-05-22 09:54:37.320: [ora.asm][2882758400]{1:4489:2} [check] AsmProxyAgent::check clsagfw_res_status 0
2018-05-22 09:54:37.327: [ USRTHRD][2876454656]{1:4489:2} setResAttrib: attr GEN_USR_ORA_INST_NAME@SERVERNAME(db1) clsagfw attribVal +ASM1 clscrs tmpAttrValue +ASM1
2018-05-22 09:54:37.327: [ USRTHRD][2876454656]{1:4489:2} setResAttrib clsagfw_modify_attribute attr GEN_USR_ORA_INST_NAME@SERVERNAME(db1) value +ASM1 retCode 0
2018-05-22 09:54:37.327: [ USRTHRD][2876454656]{1:4489:2} ASM Dedicated Thread }
2018-05-22 09:54:37.327: [ USRTHRD][2876454656]{1:4489:2} Thread:ASM DedicatedThreadisRunning is reset to false here
2018-05-22 09:55:07.122: [ AGFW][3501381376]{1:4489:2} Agent received the message: AGENT_HB[Engine] ID 12293:975
查看/etc/hosts文件，发现127.0.0.1被注释，打开注释，重启crs
[root@db1 oraagent_grid]# cat /etc/hosts #127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 #::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 [root@db1 oraagent_grid]# vi /etc/hosts #127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 #::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
[root@db1 oraagent_grid]# cd /u01/app/11.2.0/grid/bin [root@db1 bin]# ./crsctl stop cluster -all CRS-2673: Attempting to stop 'ora.crsd' on 'db1' CRS-2673: Attempting to stop 'ora.crsd' on 'db2' CRS-2790: Starting shutdown of Cluster Ready Services-managed resources on 'db1' CRS-2673: Attempting to stop 'ora.scan1.vip' on 'db1' CRS-2673: Attempting to stop 'ora.db1.vip' on 'db1' CRS-2673: Attempting to stop 'ora.OCR.dg' on 'db1' CRS-2673: Attempting to stop 'ora.registry.acfs' on 'db1' CRS-2673: Attempting to stop 'ora.racdb.db' on 'db1' CRS-2673: Attempting to stop 'ora.oc4j' on 'db1' CRS-2673: Attempting to stop 'ora.cvu' on 'db1' CRS-2790: Starting shutdown of Cluster Ready Services-managed resources on 'db2' CRS-2673: Attempting to stop 'ora.db2.vip' on 'db2' CRS-2673: Attempting to stop 'ora.OCR.dg' on 'db2' CRS-2673: Attempting to stop 'ora.registry.acfs' on 'db2' CRS-2673: Attempting to stop 'ora.racdb.db' on 'db2' CRS-2677: Stop of 'ora.cvu' on 'db1' succeeded CRS-2677: Stop of 'ora.db2.vip' on 'db2' succeeded CRS-2677: Stop of 'ora.racdb.db' on 'db1' succeeded CRS-2673: Attempting to stop 'ora.BACKUP.dg' on 'db1' CRS-2673: Attempting to stop 'ora.DATA.dg' on 'db1' CRS-2677: Stop of 'ora.racdb.db' on 'db2' succeeded CRS-2673: Attempting to stop 'ora.BACKUP.dg' on 'db2' CRS-2673: Attempting to stop 'ora.DATA.dg' on 'db2' CRS-2677: Stop of 'ora.registry.acfs' on 'db2' succeeded CRS-2677: Stop of 'ora.registry.acfs' on 'db1' succeeded CRS-2677: Stop of 'ora.scan1.vip' on 'db1' succeeded CRS-2677: Stop of 'ora.BACKUP.dg' on 'db1' succeeded CRS-2677: Stop of 'ora.DATA.dg' on 'db2' succeeded CRS-2677: Stop of 'ora.DATA.dg' on 'db1' succeeded CRS-2677: Stop of 'ora.BACKUP.dg' on 'db2' succeeded CRS-2677: Stop of 'ora.db1.vip' on 'db1' succeeded CRS-2677: Stop of 'ora.oc4j' on 'db1' succeeded CRS-2677: Stop of 'ora.OCR.dg' on 'db2' succeeded CRS-2673: Attempting to stop 'ora.asm' on 'db2' CRS-2677: Stop of 'ora.OCR.dg' on 'db1' succeeded CRS-2673: Attempting to stop 'ora.asm' on 'db1' CRS-2677: Stop of 'ora.asm' on 'db2' succeeded CRS-2673: Attempting to stop 'ora.net1.network' on 'db2' CRS-2677: Stop of 'ora.net1.network' on 'db2' succeeded CRS-2792: Shutdown of Cluster Ready Services-managed resources on 'db2' has completed CRS-2677: Stop of 'ora.asm' on 'db1' succeeded CRS-2673: Attempting to stop 'ora.net1.network' on 'db1' CRS-2677: Stop of 'ora.net1.network' on 'db1' succeeded CRS-2792: Shutdown of Cluster Ready Services-managed resources on 'db1' has completed CRS-2677: Stop of 'ora.crsd' on 'db2' succeeded CRS-2673: Attempting to stop 'ora.ctssd' on 'db2' CRS-2673: Attempting to stop 'ora.evmd' on 'db2' CRS-2673: Attempting to stop 'ora.asm' on 'db2' CRS-2677: Stop of 'ora.crsd' on 'db1' succeeded CRS-2673: Attempting to stop 'ora.ctssd' on 'db1' CRS-2673: Attempting to stop 'ora.evmd' on 'db1' CRS-2673: Attempting to stop 'ora.asm' on 'db1' CRS-2677: Stop of 'ora.evmd' on 'db2' succeeded CRS-2677: Stop of 'ora.evmd' on 'db1' succeeded CRS-2677: Stop of 'ora.ctssd' on 'db2' succeeded CRS-2677: Stop of 'ora.ctssd' on 'db1' succeeded CRS-2677: Stop of 'ora.asm' on 'db2' succeeded CRS-2673: Attempting to stop 'ora.cluster_interconnect.haip' on 'db2' CRS-2677: Stop of 'ora.asm' on 'db1' succeeded CRS-2673: Attempting to stop 'ora.cluster_interconnect.haip' on 'db1' CRS-2677: Stop of 'ora.cluster_interconnect.haip' on 'db1' succeeded CRS-2673: Attempting to stop 'ora.cssd' on 'db1' CRS-2677: Stop of 'ora.cluster_interconnect.haip' on 'db2' succeeded CRS-2673: Attempting to stop 'ora.cssd' on 'db2' CRS-2677: Stop of 'ora.cssd' on 'db1' succeeded CRS-2677: Stop of 'ora.cssd' on 'db2' succeeded [root@db1 bin]# ./crsctl start cluster -all CRS-2672: Attempting to start 'ora.cssdmonitor' on 'db1' CRS-2672: Attempting to start 'ora.cssdmonitor' on 'db2' CRS-2676: Start of 'ora.cssdmonitor' on 'db2' succeeded CRS-2672: Attempting to start 'ora.cssd' on 'db2' CRS-2672: Attempting to start 'ora.diskmon' on 'db2' CRS-2676: Start of 'ora.diskmon' on 'db2' succeeded CRS-2676: Start of 'ora.cssdmonitor' on 'db1' succeeded CRS-2672: Attempting to start 'ora.cssd' on 'db1' CRS-2672: Attempting to start 'ora.diskmon' on 'db1' CRS-2676: Start of 'ora.diskmon' on 'db1' succeeded CRS-2676: Start of 'ora.cssd' on 'db2' succeeded CRS-2672: Attempting to start 'ora.ctssd' on 'db2' CRS-2676: Start of 'ora.cssd' on 'db1' succeeded CRS-2672: Attempting to start 'ora.ctssd' on 'db1' CRS-2672: Attempting to start 'ora.cluster_interconnect.haip' on 'db1' CRS-2676: Start of 'ora.ctssd' on 'db2' succeeded CRS-2672: Attempting to start 'ora.evmd' on 'db2' CRS-2672: Attempting to start 'ora.cluster_interconnect.haip' on 'db2' CRS-2676: Start of 'ora.ctssd' on 'db1' succeeded CRS-2672: Attempting to start 'ora.evmd' on 'db1' CRS-2676: Start of 'ora.evmd' on 'db2' succeeded CRS-2676: Start of 'ora.evmd' on 'db1' succeeded CRS-2676: Start of 'ora.cluster_interconnect.haip' on 'db1' succeeded CRS-2672: Attempting to start 'ora.asm' on 'db1' CRS-2676: Start of 'ora.cluster_interconnect.haip' on 'db2' succeeded CRS-2672: Attempting to start 'ora.asm' on 'db2' CRS-2676: Start of 'ora.asm' on 'db1' succeeded CRS-2672: Attempting to start 'ora.crsd' on 'db1' CRS-2676: Start of 'ora.asm' on 'db2' succeeded CRS-2672: Attempting to start 'ora.crsd' on 'db2' CRS-2676: Start of 'ora.crsd' on 'db1' succeeded CRS-2676: Start of 'ora.crsd' on 'db2' succeeded [root@db1 bin]# ./crs_stat -t -v Name Type R/RA F/FT Target State Host ---------------------------------------------------------------------- ora.BACKUP.dg ora....up.type 0/5 0/ ONLINE ONLINE db1 ora.DATA.dg ora....up.type 0/5 0/ ONLINE ONLINE db1 ora....ER.lsnr ora....er.type 0/5 0/ ONLINE ONLINE db1 ora....N1.lsnr ora....er.type 0/5 0/0 ONLINE ONLINE db1 ora.OCR.dg ora....up.type 0/5 0/ ONLINE ONLINE db1 ora.asm ora.asm.type 0/5 0/ ONLINE ONLINE db1 ora.cvu ora.cvu.type 0/5 0/0 ONLINE ONLINE db1 ora....SM1.asm application 0/5 0/0 ONLINE ONLINE db1 ora....B1.lsnr application 0/5 0/0 ONLINE ONLINE db1 ora.db1.gsd application 0/5 0/0 OFFLINE OFFLINE ora.db1.ons application 0/3 0/0 ONLINE ONLINE db1 ora.db1.vip ora....t1.type 0/0 0/0 ONLINE ONLINE db1 ora....SM2.asm application 0/5 0/0 ONLINE ONLINE db2 ora....B2.lsnr application 0/5 0/0 ONLINE ONLINE db2 ora.db2.gsd application 0/5 0/0 OFFLINE OFFLINE ora.db2.ons application 0/3 0/0 ONLINE ONLINE db2 ora.db2.vip ora....t1.type 0/0 0/0 ONLINE ONLINE db2 ora.gsd ora.gsd.type 0/5 0/ OFFLINE OFFLINE ora....network ora....rk.type 0/5 0/ ONLINE ONLINE db1 ora.oc4j ora.oc4j.type 0/1 0/2 ONLINE ONLINE db1 ora.ons ora.ons.type 0/3 0/ ONLINE ONLINE db1 ora.racdb.db ora....se.type 0/2 0/1 ONLINE ONLINE db1 ora....ry.acfs ora....fs.type 0/5 0/ ONLINE ONLINE db1 ora.scan1.vip ora....ip.type 0/0 0/0 ONLINE ONLINE db1