Mysqld failing after DDL on cluster (no replies)

Cluster,

In 7.2.10, I have experienced a serious mysqld lockup. All queries where throwing:

Got error 4547 'RecordSpecification has overlapping offsets' from NDBCLUSTER

This occurred after I completed some DDL on another mysqld server connected to the same cluster:

ALTER TABLE fl_state
ADD mc_odo_change_t TIMESTAMP NULL DEFAULT NULL AFTER mc_odo;

This inserted a new field into the middle of a table with 227 columns.

This was resolved by killing and restarting the 'mysqld' node.

This table receives about 50 DML/sec from 12 mysqld nodes. Only one of them crashed.

Table schema and metrics below.

I hope this is useful to somebody.

Regards, Ben Clewett.

-----------------------------------------------------

Table Schema:

CREATE TABLE `fl_state` (
`terminal_id` varchar(32) NOT NULL,
`ip_addr` varchar(15) NOT NULL DEFAULT '',
`server_name` varchar(40) NOT NULL DEFAULT '',
`server_pid` int(11) NOT NULL DEFAULT '0',
`version_daemon` varchar(20) NOT NULL DEFAULT '',
`version_software` varchar(20) NOT NULL DEFAULT '',
`version_protocol` varchar(20) NOT NULL DEFAULT '',
`connected` timestamp NULL DEFAULT NULL,
`tally` int(11) NOT NULL DEFAULT '0',
`tacho_type` varchar(20) DEFAULT NULL,
`comp_code` varchar(8) DEFAULT NULL,
`veh_code` varchar(16) DEFAULT NULL,
`veh_rtid` int(11) DEFAULT NULL,
`mem_code` int(11) DEFAULT NULL,
`terminal_type` varchar(20) DEFAULT NULL,
`terminal_type_sub` varchar(20) DEFAULT NULL,
`terminal_type_sub_from_veh` varchar(20) DEFAULT NULL,
`service_code` varchar(16) DEFAULT NULL,
`service_sub_code` varchar(8) DEFAULT NULL,
`entity_type` enum('driven','trailer','member') NOT NULL DEFAULT 'driven',
`site_no` int(11) DEFAULT NULL,
`reset_required` tinyint(1) NOT NULL DEFAULT '0',
`tag_master_veh_rtid` int(11) DEFAULT NULL,
`tag_master_veh_rtid_prev` int(11) DEFAULT NULL,
`tag_master_terminal_id` varchar(32) DEFAULT NULL,
`tag_slave_recorded` tinyint(1) NOT NULL DEFAULT '0',
`tag_slave_count` int(11) NOT NULL DEFAULT '0',
`tag_state_pending` enum('trip','timeout','next_valid','not') NOT NULL DEFAULT 'not',
`tag_state_pending_t` timestamp NULL DEFAULT NULL,
`tag_state_pending_lat` decimal(9,6) DEFAULT NULL,
`tag_state_pending_lon` decimal(9,6) DEFAULT NULL,
`tag_state_pending_km_h` int(11) DEFAULT NULL,
`tag_state_pending_id` int(11) DEFAULT NULL,
`tag_state_timeout_start` timestamp NULL DEFAULT NULL,
`tag_state_timeout_t` int(11) DEFAULT NULL,
`tag_state_pending_trip` enum('wait_primed','wait_next','found_trip','not') NOT NULL DEFAULT 'not',
`tag_slave_last_drop` timestamp NULL DEFAULT NULL,
`tag_slave_last_hookup` timestamp NULL DEFAULT NULL,
`fl_story_rec` bigint(20) unsigned DEFAULT NULL,
`mo_t` timestamp NOT NULL DEFAULT '2000-01-01 00:00:00',
`mo_odo` decimal(11,3) DEFAULT NULL,
`mo_fuel` decimal(10,2) DEFAULT NULL,
`md_t` timestamp NOT NULL DEFAULT '2000-01-01 00:00:00',
`md_t_from` enum('gps','server') NOT NULL DEFAULT 'gps',
`md_valid_fix` tinyint(1) NOT NULL,
`md_ignition` tinyint(1) DEFAULT NULL,
`md_ignition_on_t` timestamp NULL DEFAULT NULL,
`md_ignition_off_t` timestamp NULL DEFAULT NULL,
`md_stationary` tinyint(1) DEFAULT NULL,
`md_lights` tinyint(1) DEFAULT NULL,
`md_battery` tinyint(1) DEFAULT NULL,
`md_panic_active` tinyint(1) DEFAULT NULL,
`md_panic_ack` tinyint(1) DEFAULT NULL,
`md_panic_active_t` timestamp NULL DEFAULT NULL,
`md_panic_cancel_t` timestamp NULL DEFAULT NULL,
`md_panic_ack_t` timestamp NULL DEFAULT NULL,
`md_stored_frame` tinyint(1) DEFAULT NULL,
`md_heading_change` tinyint(1) DEFAULT NULL,
`md_power_save` tinyint(1) DEFAULT NULL,
`md_gps_ariel_fault` enum('good','short','open') DEFAULT NULL,
`md_reset` tinyint(1) DEFAULT NULL,
`md_batt_volts` decimal(4,1) DEFAULT NULL,
`md_batt_percent` decimal(4,1) DEFAULT NULL,
`md_upgrade` tinyint(1) DEFAULT NULL,
`md_engine_by_volts` tinyint(1) DEFAULT NULL,
`md_flag1` tinyint(3) unsigned DEFAULT NULL,
`md_flag2` tinyint(3) unsigned DEFAULT NULL,
`md_tl_rec` bigint(20) unsigned DEFAULT NULL,
`md_server_name` varchar(40) DEFAULT NULL,
`mg_t` timestamp NULL DEFAULT NULL,
`mg_lat` decimal(9,6) DEFAULT NULL,
`mg_lon` decimal(9,6) DEFAULT NULL,
`mg_speed_n_h` decimal(5,1) DEFAULT NULL,
`mg_speed_km_h` decimal(5,1) DEFAULT NULL,
`mg_speed_m_h` decimal(5,1) DEFAULT NULL,
`mg_heading` int(11) DEFAULT NULL,
`mg_tl_rec` bigint(20) unsigned DEFAULT NULL,
`mg_server_name` varchar(40) DEFAULT NULL,
`mz_t` timestamp NULL DEFAULT NULL,
`mz_town` varchar(40) DEFAULT NULL,
`mz_county` varchar(40) DEFAULT NULL,
`mz_country` varchar(3) DEFAULT NULL,
`mz_pcode` varchar(16) DEFAULT NULL,
`mz_gaz_rec` int(11) DEFAULT NULL,
`mc_t` timestamp NULL DEFAULT NULL,
`mc_service_distance` int(11) DEFAULT NULL,
`mc_odo` decimal(11,3) DEFAULT NULL,
`mc_fuel` decimal(10,2) DEFAULT NULL,
`mc_fuel_level` decimal(4,1) DEFAULT NULL,
`mc_temperature` decimal(5,1) DEFAULT NULL,
`mc_cat_level` decimal(4,1) DEFAULT NULL,
`mc_axle_1` decimal(6,1) DEFAULT NULL,
`mc_axle_2` decimal(6,1) DEFAULT NULL,
`mc_axle_3` decimal(6,1) DEFAULT NULL,
`mc_axle_4` decimal(6,1) DEFAULT NULL,
`mc_flag1` tinyint(3) unsigned DEFAULT NULL,
`mc_flag2` tinyint(3) unsigned DEFAULT NULL,
`mc_engine_available` tinyint(1) DEFAULT NULL,
`mc_ending_on` tinyint(1) DEFAULT NULL,
`mc_engine_on` tinyint(1) DEFAULT NULL,
`mc_pto` tinyint(1) DEFAULT NULL,
`mc_from_trip` tinyint(1) DEFAULT NULL,
`mc_tl_rec` bigint(20) unsigned DEFAULT NULL,
`mc_server_name` varchar(40) DEFAULT NULL,
`ms_t` timestamp NULL DEFAULT NULL,
`ms_flag1` tinyint(3) unsigned DEFAULT NULL,
`ms_flag2` tinyint(3) unsigned DEFAULT NULL,
`ms_flag3` tinyint(3) unsigned DEFAULT NULL,
`ms_speed` decimal(5,1) DEFAULT NULL,
`ms_card_1` varchar(16) DEFAULT NULL,
`ms_card_2` varchar(16) DEFAULT NULL,
`ms_card_1_null` enum('not_null','unsupported_device','no_k_line','card_not_inserted','driver_unknown') DEFAULT NULL,
`ms_card_2_null` enum('not_null','unsupported_device','no_k_line','card_not_inserted','driver_unknown') DEFAULT NULL,
`ms_mem_code_1` int(11) DEFAULT NULL,
`ms_mem_code_2` int(11) DEFAULT NULL,
`ms_overspeed` tinyint(1) DEFAULT NULL,
`ms_time_state_1` enum('normal','15m_before_430h','430h','15m_before_900h','900h','15m_before_xxxh','xxxh') DEFAULT NULL,
`ms_time_state_2` enum('normal','15m_before_430h','430h','15m_before_900h','900h','15m_before_xxxh','xxxh') DEFAULT NULL,
`ms_card_1_mode` enum('w','r','d','a') DEFAULT NULL,
`ms_card_2_mode` enum('w','r','d','a') DEFAULT NULL,
`ms_tl_rec` bigint(20) unsigned DEFAULT NULL,
`ms_server_name` varchar(40) DEFAULT NULL,
`mt_t` timestamp NULL DEFAULT NULL,
`mt_trip_id` int(11) DEFAULT NULL,
`mt_source` enum('sdi1','sdi2','fc01') DEFAULT NULL,
`mt_trip_start` timestamp NULL DEFAULT NULL,
`mt_trip_end` timestamp NULL DEFAULT NULL,
`mt_trip_duration` int(11) DEFAULT NULL,
`mt_ignition_on_duration` int(11) DEFAULT NULL,
`mt_engine_on_duration` int(11) DEFAULT NULL,
`mt_motion_duration` int(11) DEFAULT NULL,
`mt_total_trip_duration` int(11) DEFAULT NULL,
`mt_pto_active_nomotion` int(11) DEFAULT NULL,
`mt_pto_active_motion` int(11) DEFAULT NULL,
`mt_trip_start_odometer` decimal(11,3) DEFAULT NULL,
`mt_trip_end_odometer` decimal(11,3) DEFAULT NULL,
`mt_trip_total_odometer` decimal(11,3) DEFAULT NULL,
`mt_trip_start_fuel` decimal(10,1) DEFAULT NULL,
`mt_trip_start_fuel_c` decimal(10,1) DEFAULT NULL,
`mt_trip_end_fuel` decimal(10,1) DEFAULT NULL,
`mt_trip_end_fuel_c` decimal(10,1) DEFAULT NULL,
`mt_trip_total_fuel` decimal(10,1) DEFAULT NULL,
`mt_trip_total_fuel_c` decimal(10,1) DEFAULT NULL,
`mt_trip_idle_fuel` decimal(10,1) DEFAULT NULL,
`mt_trip_idle_fuel_c` decimal(10,1) DEFAULT NULL,
`mt_trip_fuel_l_100km` decimal(6,1) DEFAULT NULL,
`mt_trip_fuel_l_100km_c` decimal(6,1) DEFAULT NULL,
`mt_trip_fuel_l_hr` decimal(6,1) DEFAULT NULL,
`mt_trip_fuel_l_hr_c` decimal(6,1) DEFAULT NULL,
`mt_trip_driving_duration` int(11) DEFAULT NULL,
`mt_trip_working_duration` int(11) DEFAULT NULL,
`mt_trip_rest_duration` int(11) DEFAULT NULL,
`mt_trip_available_duration` int(11) DEFAULT NULL,
`mt_trip_absolute_km_h` decimal(5,2) DEFAULT NULL,
`mt_trip_engine_on_km_h` decimal(5,2) DEFAULT NULL,
`mt_trip_motion_km_h` decimal(5,2) DEFAULT NULL,
`mt_trip_speed_zone1_cc_off_duration` int(11) DEFAULT NULL,
`mt_trip_speed_zone2_cc_off_duration` int(11) DEFAULT NULL,
`mt_trip_speed_zone3_cc_off_duration` int(11) DEFAULT NULL,
`mt_trip_speed_zone4_cc_off_duration` int(11) DEFAULT NULL,
`mt_trip_speed_total_cc_off_duration` int(11) DEFAULT NULL,
`mt_trip_speed_zone1_cc_on_duration` int(11) DEFAULT NULL,
`mt_trip_speed_zone2_cc_on_duration` int(11) DEFAULT NULL,
`mt_trip_speed_zone3_cc_on_duration` int(11) DEFAULT NULL,
`mt_trip_speed_zone4_cc_on_duration` int(11) DEFAULT NULL,
`mt_trip_speed_total_cc_on_duration` int(11) DEFAULT NULL,
`mt_trip_brake_pedal_count` int(11) DEFAULT NULL,
`mt_trip_brake_pedal_duration` int(11) DEFAULT NULL,
`mt_trip_accel_duration` int(11) DEFAULT NULL,
`mt_trip_decel_duration` int(11) DEFAULT NULL,
`mt_trip_accel_max_ms2` decimal(5,2) DEFAULT NULL,
`mt_trip_decel_max_ms2` decimal(5,2) DEFAULT NULL,
`mt_trip_acc_zone1_duration` int(11) DEFAULT NULL,
`mt_trip_acc_zone2_duration` int(11) DEFAULT NULL,
`mt_trip_acc_zone3_duration` int(11) DEFAULT NULL,
`mt_trip_acc_zone4_duration` int(11) DEFAULT NULL,
`mt_trip_acc_zone5_duration` int(11) DEFAULT NULL,
`mt_trip_acc_zone6_duration` int(11) DEFAULT NULL,
`mt_trip_acc_zone7_duration` int(11) DEFAULT NULL,
`mt_trip_throttle_zone1_duration` int(11) DEFAULT NULL,
`mt_trip_throttle_zone2_duration` int(11) DEFAULT NULL,
`mt_trip_throttle_zone3_duration` int(11) DEFAULT NULL,
`mt_trip_throttle_zone4_duration` int(11) DEFAULT NULL,
`mt_trip_engine_zone1_duration` int(11) DEFAULT NULL,
`mt_trip_engine_zone2_duration` int(11) DEFAULT NULL,
`mt_trip_engine_zone3_duration` int(11) DEFAULT NULL,
`mt_message_type` enum('T','U') DEFAULT NULL,
`mt_time_engine_torque_high` int(11) DEFAULT NULL,
`mt_time_in_high_gear` int(11) DEFAULT NULL,
`mt_time_high_engine_load` int(11) DEFAULT NULL,
`mt_time_low_load_high_throttle` int(11) DEFAULT NULL,
`mt_time_kick_down` int(11) DEFAULT NULL,
`mt_mean_engine_load` decimal(4,1) DEFAULT NULL,
`mt_server_name` varchar(40) DEFAULT NULL,
`created` timestamp NOT NULL DEFAULT '2000-01-01 00:00:00',
`updated` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
`ignore_pto` tinyint(1) NOT NULL DEFAULT '0',
`cfg_t` timestamp NULL DEFAULT NULL,
`cfg_apn` varchar(20) DEFAULT NULL,
`cfg_user` varchar(20) DEFAULT NULL,
`cfg_password` varchar(20) DEFAULT NULL,
`cfg_ignition_off_every` varchar(20) DEFAULT NULL,
`cfg_hd_change_rpt` varchar(20) DEFAULT NULL,
`cfg_can_fitted` varchar(20) DEFAULT NULL,
`cfg_extended_can` varchar(20) DEFAULT NULL,
`cfg_trip_every` varchar(20) DEFAULT NULL,
`cfg_tacho_download_enabled` varchar(20) DEFAULT NULL,
`cfg_can_comms_protocol` varchar(20) DEFAULT NULL,
`cfg_network_sel` varchar(20) DEFAULT NULL,
`cfg_ign_rep` varchar(20) DEFAULT NULL,
`cfg_debug` varchar(20) DEFAULT NULL,
`cfg_every` varchar(20) DEFAULT NULL,
`cfg_on_battery_backup_every` varchar(20) DEFAULT NULL,
`cfg_ign_sen` varchar(20) DEFAULT NULL,
`cfg_auto_reset` varchar(20) DEFAULT NULL,
`cfg_red_pwr_mod` varchar(20) DEFAULT NULL,
`cfg_red_pwr_min` varchar(20) DEFAULT NULL,
`cfg_eng_sense_enabled` varchar(20) DEFAULT NULL,
`cfg_eng_on_offset` varchar(20) DEFAULT NULL,
`cfg_panic_enabled` varchar(20) DEFAULT NULL,
`cfg_logging` varchar(20) DEFAULT NULL,
`cfg_rmt_logging` varchar(20) DEFAULT NULL,
`cfg_rmt_log_period` varchar(20) DEFAULT NULL,
`cfg_reset_fuel` varchar(20) DEFAULT NULL,
`cfg_alarm_time` varchar(20) DEFAULT NULL,
`cnf_auto_request` enum('no','yes_new','yes_done') NOT NULL DEFAULT 'no',
`cnf_auto_request_t` timestamp NULL DEFAULT NULL,
PRIMARY KEY (`terminal_id`),
KEY `comp_code` (`comp_code`,`veh_code`),
KEY `veh_rtid` (`veh_rtid`),
KEY `tag_master_terminal_id` (`tag_master_terminal_id`),
KEY `tag_master_veh_rtid` (`tag_master_veh_rtid`),
KEY `cnf_auto_request` (`cnf_auto_request`,`terminal_type_sub`)
) ENGINE=ndbcluster DEFAULT CHARSET=latin1



-----------------------------------------------------

Information Schema:

TABLE_CATALOG: def
TABLE_SCHEMA: ndb
TABLE_NAME: fl_state
TABLE_TYPE: BASE TABLE
ENGINE: ndbcluster
VERSION: 10
ROW_FORMAT: Dynamic
TABLE_ROWS: 6567
AVG_ROW_LENGTH: 844
DATA_LENGTH: 7405568
MAX_DATA_LENGTH: 0
INDEX_LENGTH: 0
DATA_FREE: 0
AUTO_INCREMENT: NULL
CREATE_TIME: NULL
UPDATE_TIME: NULL
CHECK_TIME: NULL
TABLE_COLLATION: latin1_swedish_ci
CHECKSUM: NULL
CREATE_OPTIONS:
TABLE_COMMENT: