Node declared dead. See error log for details (Arbitration error) (1 reply)

Hello, I ofter encounter this error information:

Time: Wednesday 23 June 2010 - 19:14:47
Status: Temporary error, restart node
Message: Node declared dead. See error log for details (Arbitration error)
Error: 2315
Error data: We(3) have been declared dead by 4 reason: Hearbeat failure(4)
Error object: QMGR (Line: 3555) 0x00000008
Program: ndbmtd
Pid: 6037 thr: 0
Version: mysql-5.1.44 ndb-7.1.3
Trace: /usr/local/mysql/data/ndb_3_trace.log.22 /usr/local/mysql/data/ndb_3_trace.log.22_t1 /usr/local/mysql/data/ndb_3_trace.log.22_t2 /usr/local/mys

please help me. thank you.

my environment:
1 management node
28 sql node
4 data node 32G memory/every data node

my config.ini :
# Options affecting ndbd processes on all data nodes:
[ndbd default]
NoOfReplicas=2 # Number of replicas
#DataMemory=3072M # How much memory to allocate for data storage
DataMemory=20480M
IndexMemory=3413M # How much memory to allocate for index storage
# For DataMemory and IndexMemory, we have used the
# default values. Since the "world" database takes up
# only about 500KB, this should be more than enough for
# this example Cluster setup.
StringMemory=25

ODirect=1
MaxNoOfLocalScans=64

MaxNoOfTables=4096
MaxNoOfOrderedIndexes=2048
MaxNoOfUniqueHashIndexes=512
MaxNoOfAttributes=2000
#MaxNoOfAttributes=24576
MaxNoOfTriggers=14336
MaxNoOfConcurrentOperations=5000000

#MaxAllocate=50M

LockPagesInMainMemory=1

MaxNoOfConcurrentTransactions=16384

NoOfFragmentLogFiles=48

#### New Add ##############
DiskCheckpointSpeedInRestart=100M
FragmentLogFileSize=256M
#TimeBetweenLocalCheckpoints=20
TimeBetweenGlobalCheckpoints=1000
TimeBetweenEpochs=100
InitFragmentLogFiles=SPARSE
MemReportFrequency=30
BackupReportFrequency=10
### Watchdog
TimeBetweenWatchdogCheckInitial=60000
### TransactionInactiveTimeout - should be enabled in Production
TransactionInactiveTimeout=60000
SharedGlobalMemory=384M -- 决定日志、磁盘操作、表空间的元数据和日志文件组、UNDO文件、数据文件的缓冲区总量
LongMessageBuffer=1024M -- 用于节点之间的传递消息的内部缓冲
BatchSizePerLocalScan=512
#############################

#InitFragmentLogFiles=FULL
RedoBuffer=32M
#

TransactionBufferMemory=10M
TimeBetweenLocalCheckpoints=4
TransactionDeadlockDetectionTimeout=10000


DiskPageBufferMemory=256M
DiskCheckpointSpeed=100M

LogLevelStartup=15
LogLevelShutdown=15
LogLevelCheckpoint=8
LogLevelNodeRestart=15
LogLevelError=15

BackupWriteSize=1M
BackupDataBufferSize=16M
BackupLogBufferSize=4M
BackupMemory=20M

UndoIndexBuffer=64M
UndoDataBuffer=256M

StopOnError=0

#NoOfDiskPagesToDiskAfterRestartTUP=40
#NoOfDiskPagesToDiskAfterRestartACC=20
#NoOfDiskPagesToDiskDuringRestartTUP=40
#NoOfDiskPagesToDiskDuringRestartACC=20

MaxNoOfExecutionThreads=8

TotalSendBufferMemory=20M

#HeartbeatIntervalDbDb=15000
#HeartbeatIntervalDbApi=15000

# TCP/IP options:
[tcp default]
#portnumber=1186 # This the default; however, you can use any port that is free
# for all the hosts in the cluster
# Note: It is recommended that you do not specify the port
# number at all and allow the default value to be used instead
SendBufferMemory=20480K
ReceiveBufferMemory=20480K

# Management process options:
[ndb_mgmd]
id=1
hostname=10.192.83.7 # Hostname or IP address of management node
datadir=/var/lib/mysql-cluster # Directory for management node log files

ArbitrationRank=1
ArbitrationDelay=0

# Options for data node "A":
[ndbd]
id=2 # (one [ndbd] section per data node)
hostname=10.192.7.15 # Hostname or IP address
datadir=/usr/local/mysql/data # Directory for this data node's data files
#TotalSendBufferMemory=200M

# Options for data node "B":
[ndbd]
id=3
hostname=10.192.7.16 # Hostname or IP address
datadir=/usr/local/mysql/data # Directory for this data node's data files
#TotalSendBufferMemory=200M

# Options for data node "C":
[ndbd]
id=4
hostname=10.192.136.21 # Hostname or IP address
datadir=/usr/local/mysql/data # Directory for this data node's data files
#TotalSendBufferMemory=200M

# Options for data node "D":
[ndbd]
id=5
hostname=10.192.136.28 # Hostname or IP address
datadir=/usr/local/mysql/data # Directory for this data node's data files
#TotalSendBufferMemory=200M



# SQL node options:

[mysqld]
hostname=10.192.89.6 # Hostname or IP address
# (additional mysqld connections can be
# specified for this node for various
# purposes such as running ndb_restore)
id=18
MaxScanBatchSize=16M
ArbitrationDelay=0

[mysqld]
hostname=10.192.89.60
id=19
MaxScanBatchSize=16M
ArbitrationDelay=0

[mysqld]
MaxScanBatchSize=16M
ArbitrationDelay=0

[mysqld]
MaxScanBatchSize=16M
ArbitrationDelay=0

[mysqld]
MaxScanBatchSize=16M
ArbitrationDelay=0

[mysqld]
MaxScanBatchSize=16M
ArbitrationDelay=0

[mysqld]
MaxScanBatchSize=16M
ArbitrationDelay=0

[mysqld]
MaxScanBatchSize=16M
ArbitrationDelay=0

[mysqld]
MaxScanBatchSize=16M
ArbitrationDelay=0

[mysqld]
MaxScanBatchSize=16M
ArbitrationDelay=0

[mysqld]
MaxScanBatchSize=16M
ArbitrationDelay=0

[mysqld]
MaxScanBatchSize=16M
ArbitrationDelay=0

[mysqld]
MaxScanBatchSize=16M
ArbitrationDelay=0

[mysqld]
MaxScanBatchSize=16M
ArbitrationDelay=0

[mysqld]
MaxScanBatchSize=16M
ArbitrationDelay=0

[mysqld]
MaxScanBatchSize=16M
ArbitrationDelay=0

[mysqld]
MaxScanBatchSize=16M
ArbitrationDelay=0

[mysqld]
MaxScanBatchSize=16M
ArbitrationDelay=0

[mysqld]
MaxScanBatchSize=16M
ArbitrationDelay=0

[mysqld]
MaxScanBatchSize=16M
ArbitrationDelay=0

[mysqld]
MaxScanBatchSize=16M
ArbitrationDelay=0

[mysqld]
MaxScanBatchSize=16M
ArbitrationDelay=0

[mysqld]
MaxScanBatchSize=16M
ArbitrationDelay=0

[mysqld]
MaxScanBatchSize=16M
ArbitrationDelay=0

[mysqld]
MaxScanBatchSize=16M
ArbitrationDelay=0

[mysqld]
MaxScanBatchSize=16M
ArbitrationDelay=0

[mysqld]
MaxScanBatchSize=16M
ArbitrationDelay=0

[mysqld]
MaxScanBatchSize=16M
ArbitrationDelay=0