某生产环境Oracle Linux 7.1,Oracle 12C RAC显示节点1的集群资源没有启动,信息如下:

[grid@cs2~]$crsctlstatres-t--------------------------------------------------------------------------------NameTargetStateServerStatedetails--------------------------------------------------------------------------------LocalResources--------------------------------------------------------------------------------ora.ASMNET1LSNR_ASM.lsnrONLINEONLINEcs2STABLEora.CRS.dgONLINEONLINEcs2STABLEora.DATA.dgONLINEONLINEcs2STABLEora.LISTENER.lsnrONLINEONLINEcs2STABLEora.chadONLINEOFFLINEcs2STABLEora.net1.networkONLINEONLINEcs2STABLEora.onsONLINEOFFLINEcs2STABLEora.proxy_advmOFFLINEOFFLINEcs2STABLE--------------------------------------------------------------------------------ClusterResources--------------------------------------------------------------------------------ora.LISTENER_SCAN1.lsnr1ONLINEONLINEcs2STABLEora.LISTENER_SCAN2.lsnr1ONLINEONLINEcs2STABLEora.LISTENER_SCAN3.lsnr1ONLINEONLINEcs2STABLEora.MGMTLSNR1ONLINEONLINEcs2169.254.255.9288.88.88.191,STABLEora.asm1ONLINEOFFLINESTABLE2ONLINEONLINEcs2Started,STABLE3OFFLINEOFFLINESTABLEora.cs.db1ONLINEOFFLINESTABLE2ONLINEOFFLINEInstanceShutdown,STABLEora.cs1.vip1ONLINEINTERMEDIATEcs2FAILEDOVER,STABLEora.cs2.vip1ONLINEONLINEcs2STABLEora.cvu1ONLINEONLINEcs2STABLEora.gns1ONLINEOFFLINESTABLEora.gns.vip1ONLINEOFFLINESTABLEora.mgmtdb1ONLINEOFFLINEInstanceShutdown,STABLEora.qosmserver1ONLINEONLINEcs2STABLEora.scan1.vip1ONLINEONLINEcs2STABLEora.scan2.vip1ONLINEONLINEcs2STABLEora.scan3.vip1ONLINEONLINEcs2STABLE--------------------------------------------------------------------------------

如果查看节点1上的alert.log文件,信息如下:

[root@cs1soft]#tail-100/u01/app/grid/diag/crs/cs1/crs/trace/alert.log2020-01-1420:45:50.226[CVUD(2366CRS-10051:CVUfoundfollowingerrorswithClusterwaresetup:PRVE-3191:(DESCRIPTION=(LOAD_BALANCE=on)(ADDRESS=(PROTOCOL=TCP)(HOST=cs-cluster-scan.cs-cluster.jy.net)(PORT=1521))(CONNECT_DATA=(SERVER=DEDICATED)(SERVICE_NAME=cs)))Listenerrefusedtheconnectionwiththefollowingerror:ORA-12514,TNS:listenerdoesnotcurrentlyknowofservicerequestedinconnectdescriptor2020-02-1714:49:28.563[ORAROOTAGENT(15500)]CRS-5822:Agent'/u01/app/product/12.2.0/crs/bin/orarootagent_root'disconnectedfromserver.Detailsat(:CRSAGF00117:){0:5:63832}in/u01/app/grid/diag/crs/cs1/crs/trace/crsd_orarootagent_root.trc.2020-02-1714:49:28.584[ORAAGENT(15466)]CRS-5822:Agent'/u01/app/product/12.2.0/crs/bin/oraagent_grid'disconnectedfromserver.Detailsat(:CRSAGF00117:){0:1:36}in/u01/app/grid/diag/crs/cs1/crs/trace/crsd_oraagent_grid.trc.2020-02-1714:49:29.607[GPNPD(3293)]CRS-2329:GPNPDonnodecs1shutdown.2020-02-1714:49:30.123[OSYSMOND(6077)]CRS-8504:OracleClusterwareOSYSMONDprocesswithoperatingsystemprocessID6077isexiting2020-02-1714:49:31.377[MDNSD(3254)]CRS-5602:mDNSservicestoppingbyrequest.2020-02-1714:49:31.747[MDNSD(3254)]CRS-8504:OracleClusterwareMDNSDprocesswithoperatingsystemprocessID3254isexiting2020-02-1714:49:46.650[OCTSSD(5883)]CRS-2405:TheClusterTimeSynchronizationServiceonhostcs1isshutdownbyuser2020-02-1714:49:46.651[OCTSSD(5883)]CRS-8504:OracleClusterwareOCTSSDprocesswithoperatingsystemprocessID5883isexiting2020-02-1714:49:47.651[OCSSD(3616)]CRS-1603:CSSDonnodecs1hasbeenshutdown.2020-02-1714:49:47.958[OCSSD(3616)]CRS-1660:TheCSSdaemonshutdownhascompleted2020-02-1714:49:47.959[OCSSD(3616)]CRS-8504:OracleClusterwareOCSSDprocesswithoperatingsystemprocessID3616isexiting2020-02-1714:49:49.726[ORAAGENT(3235)]CRS-5822:Agent'/u01/app/product/12.2.0/crs/bin/oraagent_grid'disconnectedfromserver.Detailsat(:CRSAGF00117:){0:9:818}in/u01/app/grid/diag/crs/cs1/crs/trace/ohasd_oraagent_grid.trc.2020-02-1714:49:49.727[ORAROOTAGENT(3029)]CRS-5822:Agent'/u01/app/product/12.2.0/crs/bin/orarootagent_root'disconnectedfromserver.Detailsat(:CRSAGF00117:){0:1:21}in/u01/app/grid/diag/crs/cs1/crs/trace/ohasd_orarootagent_root.trc.

根据上面的错误信息提示查看crsd_oraagent_grid.trc跟踪文件:

[root@cs1~]#more/u01/app/grid/diag/crs/cs1/crs/trace/crsd_oraagent_grid.trcTracefile/u01/app/grid/diag/crs/cs1/crs/trace/crsd_oraagent_grid.trcOracleDatabase12cClusterwareRelease12.2.0.1.0-ProductionCopyright1996,2016Oracle.Allrightsreserved.***TRACECONTINUEDFROMFILE/u01/app/grid/diag/crs/cs1/crs/trace/crsd_oraagent_grid_910.trc***2020-02-1714:39:22.129:CLSDYNAM:2462013184:[ora.ASMNET1LSNR_ASM.lsnr]{0:1:18}[check]LsnrAgent::regEndpoint110resetregEndPointDone:02020-02-1714:39:22.129:CLSDYNAM:2462013184:[ora.ASMNET1LSNR_ASM.lsnr]{0:1:18}[check]generateEndPointStrings:generateendpoints2020-02-1714:39:22.129:CLSDYNAM:2462013184:[ora.ASMNET1LSNR_ASM.lsnr]{0:1:18}[check]VendorType=02020-02-1714:39:22.129:CLSDYNAM:2462013184:[ora.ASMNET1LSNR_ASM.lsnr]{0:1:18}[check]VendorType=02020-02-1714:39:22.129:CLSDYNAM:2462013184:[ora.ASMNET1LSNR_ASM.lsnr]{0:1:18}[check]VendorType=02020-02-1714:39:22.129:CLSDYNAM:2462013184:[ora.ASMNET1LSNR_ASM.lsnr]{0:1:18}[check]VendorType=02020-02-1714:39:22.129:CLSDYNAM:2462013184:[ora.ASMNET1LSNR_ASM.lsnr]{0:1:18}[check]generateEndPointStrings:NoIPmatchingthesubnet88.88.88.02020-02-1714:39:22.130:CLSDYNAM:2462013184:[ora.ASMNET1LSNR_ASM.lsnr]{0:1:18}[check]OcrContextinitdefaultlevel18146295362020-02-1714:39:22.133:CLSDYNAM:2462013184:[ora.ASMNET1LSNR_ASM.lsnr]{0:1:18}[check]Agent::getDeploymentPlatformIdreturn2020-02-1714:39:22.133:CLSDYNAM:2462013184:[ora.ASMNET1LSNR_ASM.lsnr]{0:1:18}[check]LsnrAgent::generateEndPoints000entry{nonPriv2020-02-1714:39:22.133:CLSDYNAM:2462013184:[ora.ASMNET1LSNR_ASM.lsnr]{0:1:18}[check]LsnrAgent::generateEndPoints000entry{lsnrResName:endpAttr:TCP:1526type:42020-02-1714:39:22.133:CLSDYNAM:2462013184:[ora.ASMNET1LSNR_ASM.lsnr]{0:1:18}[check]LsnrAgent::generateEndPoints040ListenerResName:2020-02-1714:39:22.133:CLSDYNAM:2462013184:[ora.ASMNET1LSNR_ASM.lsnr]{0:1:18}[check]LsnrAgent::removeDuplicates2020-02-1714:39:22.133:CLSDYNAM:2462013184:[ora.ASMNET1LSNR_ASM.lsnr]{0:1:18}[check]LsnrAgent::generateEndPoints999exitnonPriv}2020-02-1714:39:22.133:CLSDYNAM:2462013184:[ora.ASMNET1LSNR_ASM.lsnr]{0:1:18}[check]LsnrAgent::regEndpoint200endpStringsempty2020-02-1714:39:22.134:CLSDYNAM:2462013184:[ora.ASMNET1LSNR_ASM.lsnr]{0:1:18}[check]AsmLsnrAgent::init2020-02-1714:39:22.154:CLSDYNAM:2462013184:[ora.ASMNET1LSNR_ASM.lsnr]{0:1:18}[check]VendorType=02020-02-1714:39:22.154:CLSDYNAM:2462013184:[ora.ASMNET1LSNR_ASM.lsnr]{0:1:18}[check]VendorType=02020-02-1714:39:22.154:CLSDYNAM:2462013184:[ora.ASMNET1LSNR_ASM.lsnr]{0:1:18}[check]VendorType=02020-02-1714:39:22.154:CLSDYNAM:2462013184:[ora.ASMNET1LSNR_ASM.lsnr]{0:1:18}[check]VendorType=02020-02-1714:39:22.154:CLSDYNAM:2462013184:[ora.ASMNET1LSNR_ASM.lsnr]{0:1:18}[check]CannotfindanIPaddressmatchingthesubnet88.88.88.02020-02-1714:39:22.154:CLSDYNAM:2462013184:[ora.ASMNET1LSNR_ASM.lsnr]{0:1:18}[check]OcrContextinitdefaultlevel18146295362020-02-1714:39:22.157:CLSDYNAM:2462013184:[ora.ASMNET1LSNR_ASM.lsnr]{0:1:18}[check]Agent::getDeploymentPlatformIdreturn2020-02-1714:39:22.158:CLSCEVT:2462013184:(:CLSCE0028:)clsce_unsubscribe0x7f725c004510successfullyunsubscribed:9748332020-02-1714:39:22.158:USRTHRD:2462013184:{0:1:18}WorkerThread::removeWorkertotalworkers:22020-02-1714:39:22.158:USRTHRD:2462013184:{0:1:18}Worker::scheduleaddingworker'ReconnSub-LsnrAgentSub-ASMNET1LSNR_ASM'tothread'ReconnectingSubscribers'2020-02-1714:39:22.158:USRTHRD:2462013184:{0:1:18}WorkerThread::addWorkertotalworkers:32020-02-1714:39:22.158:CLSDYNAM:2462013184:[ora.ASMNET1LSNR_ASM.lsnr]{0:1:18}[check]SkippingCSSInitializationforrebootlessrecovery2020-02-1714:39:22.159:CLSCEVT:2478823168:clsce_subscribe0x7f72600d06b0filter='(^CRS_RESOURCE_PROFILE_CHANGE.*RESOURCE_CLASS='(scan_vip|vip)')|(^CRS_RESOURCE_PROFILE_CHANGE.*TYPE='ora\.network\.type')',flags=1,handler=0xc6ad0c,arg=0x7f726016de10

从上面的错误信息中可以找到Cannot find an IP address matching the subnet 88.88.88.0,这说明私有IP丢失了。

如是查看网络IP地址
节点2私有IP正常

[grid@cs2~]$ifconfig-aens160:flags=4163mtu1500inet10.13.13.191netmask255.255.255.0broadcast10.13.13.255inet6fe80::250:56ff:fea0:92afprefixlen64scopeid0x20ether00:50:56:a0:92:aftxqueuelen1000(Ethernet)RXpackets127560233bytes10186683662(9.4GiB)RXerrors0dropped353overruns0frame0TXpackets18954431bytes1554204413(1.4GiB)TXerrors0dropped0overruns0carrier0collisions0ens160:1:flags=4163mtu1500inet10.13.13.130netmask255.255.255.0broadcast10.13.13.255ether00:50:56:a0:92:aftxqueuelen1000(Ethernet)ens160:2:flags=4163mtu1500inet10.13.13.147netmask255.255.255.0broadcast10.13.13.255ether00:50:56:a0:92:aftxqueuelen1000(Ethernet)ens160:3:flags=4163mtu1500inet10.13.13.141netmask255.255.255.0broadcast10.13.13.255ether00:50:56:a0:92:aftxqueuelen1000(Ethernet)ens160:4:flags=4163mtu1500inet10.13.13.138netmask255.255.255.0broadcast10.13.13.255ether00:50:56:a0:92:aftxqueuelen1000(Ethernet)ens160:5:flags=4163mtu1500inet10.13.13.137netmask255.255.255.0broadcast10.13.13.255ether00:50:56:a0:92:aftxqueuelen1000(Ethernet)ens160:6:flags=4163mtu1500inet10.13.13.136netmask255.255.255.0broadcast10.13.13.255ether00:50:56:a0:92:aftxqueuelen1000(Ethernet)ens192:flags=4163mtu1500inet88.88.88.191netmask255.255.255.0broadcast88.88.88.255inet6fe80::250:56ff:fea0:a05cprefixlen64scopeid0x20ether00:50:56:a0:a0:5ctxqueuelen1000(Ethernet)RXpackets294716840bytes215309349272(200.5GiB)RXerrors0dropped355overruns0frame0TXpackets63703465bytes363393728693(338.4GiB)TXerrors0dropped0overruns0carrier0collisions0ens192:1:flags=4163mtu1500inet169.254.255.92netmask255.255.0.0broadcast169.254.255.255ether00:50:56:a0:a0:5ctxqueuelen1000(Ethernet)lo:flags=73mtu65536inet127.0.0.1netmask255.0.0.0inet6::1prefixlen128scopeid0x10looptxqueuelen0(LocalLoopback)RXpackets90029207bytes223833448058(208.4GiB)RXerrors0dropped0overruns0frame0TXpackets90029207bytes223833448058(208.4GiB)TXerrors0dropped0overruns0carrier0collisions0virbr0:flags=4099mtu1500inet192.168.122.1netmask255.255.255.0broadcast192.168.122.255ether52:54:00:d0:6e:0btxqueuelen0(Ethernet)RXpackets0bytes0(0.0B)RXerrors0dropped0overruns0frame0TXpackets0bytes0(0.0B)TXerrors0dropped0overruns0carrier0collisions0virbr0-nic:flags=4098mtu1500ether52:54:00:d0:6e:0btxqueuelen500(Ethernet)RXpackets0bytes0(0.0B)RXerrors0dropped0overruns0frame0TXpackets0bytes0(0.0B)TXerrors0dropped0overruns0carrier0collisions0[grid@cs2~]$cat/etc/hosts127.0.0.1localhost10.13.13.190cs110.13.13.191cs2#10.13.13.192cs1-vip.jy.netcs1-vip#10.13.13.193cs2-vip.jy.netcs2-vip88.88.88.190cs1-priv.jy.netcs1-priv88.88.88.191cs2-priv.jy.netcs2-priv10.13.13.141cs-cluster-scan10.13.13.142cs-cluster-scan10.13.13.143cs-cluster-scan

节点1的私有IP丢失

[root@cs1~]#ifconfig-aens160:flags=4163mtu1500inet10.13.13.190netmask255.255.255.0broadcast10.13.13.255inet6fe80::250:56ff:fea0:4e69prefixlen64scopeid0x20inet62018::2prefixlen64scopeid0x0ether00:50:56:a0:4e:69txqueuelen1000(Ethernet)RXpackets53266410bytes3937179306(3.6GiB)RXerrors0dropped143overruns0frame0TXpackets2988621bytes284679938(271.4MiB)TXerrors0dropped0overruns0carrier0collisions0ens192:flags=4163mtu1500inet6fe80::250:56ff:fea0:17dcprefixlen64scopeid0x20ether00:50:56:a0:17:dctxqueuelen1000(Ethernet)RXpackets70361859bytes5602168856(5.2GiB)RXerrors0dropped141overruns0frame0TXpackets24902418bytes195063342601(181.6GiB)TXerrors0dropped0overruns0carrier0collisions0lo:flags=73mtu65536inet127.0.0.1netmask255.0.0.0inet6::1prefixlen128scopeid0x10looptxqueuelen0(LocalLoopback)RXpackets30235303bytes10958756943(10.2GiB)RXerrors0dropped0overruns0frame0TXpackets30235303bytes10958756943(10.2GiB)TXerrors0dropped0overruns0carrier0collisions0virbr0:flags=4099mtu1500inet192.168.122.1netmask255.255.255.0broadcast192.168.122.255ether52:54:00:d0:6e:0btxqueuelen0(Ethernet)RXpackets0bytes0(0.0B)RXerrors0dropped0overruns0frame0TXpackets0bytes0(0.0B)TXerrors0dropped0overruns0carrier0collisions0virbr0-nic:flags=4098mtu1500ether52:54:00:d0:6e:0btxqueuelen500(Ethernet)RXpackets0bytes0(0.0B)RXerrors0dropped0overruns0frame0TXpackets0bytes0(0.0B)TXerrors0dropped0overruns0carrier0collisions0

设置私有IP

[root@cs1network-scripts]#catifcfg-Ethernet_connection_ens192TYPE=EthernetBOOTPROTO=noneDEFROUTE=yesIPV4_FAILURE_FATAL=noIPV6INIT=noNAME=ens192UUID=8ec45d90-c1fb-4924-84a0-72f58fc9b82fDEVICE=ens192ONBOOT=yesIPADDR=88.88.88.190PREFIX=24NETMASK=255.255.255.0[root@cs1network-scripts]#ifconfig-aens160:flags=4163mtu1500inet10.138.130.190netmask255.255.255.0broadcast10.138.130.255inet6fe80::250:56ff:fea0:4e69prefixlen64scopeid0x20inet62018::2prefixlen64scopeid0x0ether00:50:56:a0:4e:69txqueuelen1000(Ethernet)RXpackets53277081bytes3937945677(3.6GiB)RXerrors0dropped143overruns0frame0TXpackets2989099bytes284806540(271.6MiB)TXerrors0dropped0overruns0carrier0collisions0ens192:flags=4163mtu1500inet88.88.88.190netmask255.255.255.0broadcast88.88.88.255inet6fe80::250:56ff:fea0:17dcprefixlen64scopeid0x20ether00:50:56:a0:17:dctxqueuelen1000(Ethernet)RXpackets70371906bytes5602881459(5.2GiB)RXerrors0dropped141overruns0frame0TXpackets24902428bytes195063343317(181.6GiB)TXerrors0dropped0overruns0carrier0collisions0lo:flags=73mtu65536inet127.0.0.1netmask255.0.0.0inet6::1prefixlen128scopeid0x10looptxqueuelen0(LocalLoopback)RXpackets30236949bytes10959017396(10.2GiB)RXerrors0dropped0overruns0frame0TXpackets30236949bytes10959017396(10.2GiB)TXerrors0dropped0overruns0carrier0collisions0virbr0:flags=4099mtu1500inet192.168.122.1netmask255.255.255.0broadcast192.168.122.255ether52:54:00:d0:6e:0btxqueuelen0(Ethernet)RXpackets0bytes0(0.0B)RXerrors0dropped0overruns0frame0TXpackets0bytes0(0.0B)TXerrors0dropped0overruns0carrier0collisions0virbr0-nic:flags=4098mtu1500ether52:54:00:d0:6e:0btxqueuelen500(Ethernet)RXpackets0bytes0(0.0B)RXerrors0dropped0overruns0frame0TXpackets0bytes0(0.0B)TXerrors0dropped0overruns0carrier0collisions0

[root@cs1 ~]# crsctl stop crs -f
CRS-2791: Starting shutdown of Oracle High Availability Services-managed resources on 'cs2'
CRS-2673: Attempting to stop 'ora.mdnsd' on 'cs2'
CRS-2673: Attempting to stop 'ora.gpnpd' on 'cs2'
CRS-2673: Attempting to stop 'ora.crf' on 'cs2'
CRS-2673: Attempting to stop 'ora.ctssd' on 'cs2'
CRS-2673: Attempting to stop 'ora.evmd' on 'cs2'
CRS-2673: Attempting to stop 'ora.asm' on 'cs2'
CRS-2673: Attempting to stop 'ora.drivers.acfs' on 'cs2'
CRS-2677: Stop of 'ora.drivers.acfs' on 'cs2' succeeded
CRS-2677: Stop of 'ora.ctssd' on 'cs2' succeeded
CRS-2677: Stop of 'ora.crf' on 'cs2' succeeded
CRS-2677: Stop of 'ora.gpnpd' on 'cs2' succeeded
CRS-2677: Stop of 'ora.evmd' on 'cs2' succeeded
CRS-2677: Stop of 'ora.mdnsd' on 'cs2' succeeded
CRS-2677: Stop of 'ora.asm' on 'cs2' succeeded
CRS-2673: Attempting to stop 'ora.cluster_interconnect.haip' on 'cs2'
CRS-2677: Stop of 'ora.cluster_interconnect.haip' on 'cs2' succeeded
CRS-2673: Attempting to stop 'ora.cssd' on 'cs2'
CRS-2677: Stop of 'ora.cssd' on 'cs2' succeeded
CRS-2673: Attempting to stop 'ora.gipcd' on 'cs2'
CRS-2673: Attempting to stop 'ora.driver.afd' on 'cs2'
CRS-2677: Stop of 'ora.driver.afd' on 'cs2' succeeded
CRS-2677: Stop of 'ora.gipcd' on 'cs2' succeeded
CRS-2793: Shutdown of Oracle High Availability Services-managed resources on 'cs2' has completed
CRS-4133: Oracle High Availability Services has been stopped.
[root@cs1 ~]# crsctl start crs
CRS-4123: Oracle High Availability Services has been started.

手动启动节点1的集群软件

[root@cs1~]#crsctlstatres-t--------------------------------------------------------------------------------NameTargetStateServerStatedetails--------------------------------------------------------------------------------LocalResources--------------------------------------------------------------------------------ora.ASMNET1LSNR_ASM.lsnrONLINEONLINEcs1STABLEONLINEONLINEcs2STABLEora.CRS.dgONLINEONLINEcs1STABLEONLINEONLINEcs2STABLEora.DATA.dgONLINEONLINEcs1STABLEONLINEONLINEcs2STABLEora.DN.dgOFFLINEOFFLINEcs1STABLEOFFLINEOFFLINEcs2STABLEora.LISTENER.lsnrONLINEONLINEcs1STABLEONLINEONLINEcs2STABLEora.chadONLINEONLINEcs1STABLEONLINEONLINEcs2STABLEora.net1.networkONLINEONLINEcs1STABLEONLINEONLINEcs2STABLEora.onsONLINEONLINEcs1STABLEONLINEONLINEcs2STABLEora.proxy_advmOFFLINEOFFLINEcs1STABLEOFFLINEOFFLINEcs2STABLE--------------------------------------------------------------------------------ClusterResources--------------------------------------------------------------------------------ora.LISTENER_SCAN1.lsnr1ONLINEONLINEcs1STABLEora.LISTENER_SCAN2.lsnr1ONLINEONLINEcs2STABLEora.LISTENER_SCAN3.lsnr1ONLINEONLINEcs1STABLEora.MGMTLSNR1ONLINEONLINEcs1169.254.207.2488.88.88.190,STABLEora.asm1ONLINEONLINEcs1Started,STABLE2ONLINEONLINEcs2Started,STABLE3OFFLINEOFFLINESTABLEora.cs.db1ONLINEONLINEcs1Open,HOME=/u01/app/oracle/product/12.2.0/db,STABLE2ONLINEONLINEcs2Open,HOME=/u01/app/oracle/product/12.2.0/db,STABLEora.cs1.vip1ONLINEONLINEcs1STABLEora.cs2.vip1ONLINEONLINEcs2STABLEora.cvu1ONLINEONLINEcs1STABLEora.gns1ONLINEONLINEcs2STABLEora.gns.vip1ONLINEONLINEcs2STABLEora.mgmtdb1ONLINEOFFLINEcs1Open,STABLEora.qosmserver1ONLINEONLINEcs1STABLEora.scan1.vip1ONLINEONLINEcs1STABLEora.scan2.vip1ONLINEONLINEcs2STABLEora.scan3.vip1ONLINEONLINEcs1STABLE--------------------------------------------------------------------------------

到此恢复正常状态。