InnoDB, phantom transactions… KILLER!

Recently, I had a problem with a client connection (with iis webservices) which blocked other transactions.
The client did nothing but sleep and hold a transaction (a problem probably connected with Gap and Next-Key Locks).
Eventually I made a fresh install of MySQL 5.5 (which has a very useful table in ‘information_schema’) and created a new schema as follows:

create database administrator;
use administrator;
create table INNODB_LOCKS SELECT * FROM information_schema.INNODB_LOCKS LIMIT 0;
create table INNODB_TRX SELECT * FROM information_schema.INNODB_TRX LIMIT 0;
create table INNODB_LOCK_WAITS SELECT * FROM information_schema.INNODB_LOCK_WAITS LIMIT 0;
create table PROCESSLIST SELECT *, 0 `trx_id` FROM information_schema.PROCESSLIST LIMIT 0;

CREATE TABLE IF NOT EXISTS `GHOST_TRX_LOG` (
  `id` int(11) NOT NULL AUTO_INCREMENT,
  `insert_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
  `thread_id` bigint(21) unsigned DEFAULT NULL,
  `thread_user` varchar(16) DEFAULT NULL,
  `thread_host` varchar(64) DEFAULT NULL,
  `thread_info` text,
  `thread_status` varchar(64) DEFAULT NULL,
  `status` tinyint(3) unsigned NOT NULL COMMENT '1 to_validate 2 validated 3 no validation',
  PRIMARY KEY (`id`),
  KEY `ts` (`insert_timestamp`,`thread_user`),
  KEY `user` (`thread_user`),
  KEY `host` (`thread_host`),
  KEY `status` (`status`)
) ENGINE=MyISAM  DEFAULT CHARSET=utf8 AUTO_INCREMENT=1;

Next, I have written an event and added it to the Event Scheduler:

How does it work?

It simply kills all transactions which take more time than set by TRANSACTION_TIME_LIMIT_V (my limit is set to 2 minutes). After that it makes a snapshot of the activities and transactions and puts it into the administrator database. Thanks to this, I can observe and log what is really going on.

I hope this helps anyone!

DROP EVENT IF EXISTS KILL_GHOST_TRX;
 
CREATE definer=root@localhost EVENT KILL_GHOST_TRX
ON SCHEDULE EVERY 1 MINUTE
DO

MAIN_BLOCK: BEGIN

	-- private variables
	DECLARE __done INT DEFAULT 0;
	
	-- public variables
	DECLARE TRANSACTION_TIME_LIMIT_v TIMESTAMP DEFAULT DATE_SUB(CURRENT_TIMESTAMP, INTERVAL 2 MINUTE); 
	DECLARE THREAD_ID_v BIGINT(21) UNSIGNED DEFAULT 0;
	DECLARE THREAD_STATE_v, THREAD_HOST_v VARCHAR(64);
	DECLARE THREAD_INFO_v LONGTEXT;
	DECLARE THREAD_COMMAND_v, THREAD_USER_v, TRX_STATE_v VARCHAR(16);
	DECLARE CURSOR_QTY INT(11) DEFAULT 0;
	DECLARE BLOCKING_TRX_ID_v VARCHAR(18);
	DECLARE p_Kill TINYINT DEFAULT 0;
	
	-- cursors
	DECLARE GHOST_TRX_CURSOR CURSOR FOR SELECT SQL_CALC_FOUND_ROWS blocking_trx_id FROM INFORMATION_SCHEMA.INNODB_LOCK_WAITS GROUP BY 1; 
	
	-- handlers
	DECLARE CONTINUE HANDLER FOR SQLSTATE '02000' SET __done = 1;
	DECLARE CONTINUE HANDLER FOR NOT FOUND 		SET __done = 1;
 
	-- [START]
	-- Is there any blocking? 
	OPEN GHOST_TRX_CURSOR;
	
	SET CURSOR_QTY = (Select FOUND_ROWS());
	
	IF CURSOR_QTY > 0 THEN
		
		BLOCKING_LOOP: LOOP FETCH 
			GHOST_TRX_CURSOR INTO BLOCKING_TRX_ID_v;		
			IF __done = 1 THEN
			  -- nothing to do
			  LEAVE BLOCKING_LOOP;
			END IF;
				
			IF p_Kill = 0 THEN
				PREPARE p_Kill FROM "KILL /* KILL_GHOST_TRX */ ? ;";
			END IF;
	
			SELECT trx.trx_mysql_thread_id	, IFNULL(trx_state, 'none'), 		trx_query, 		`USER`,			`HOST`, 		IFNULL(`STATE`, 'none')
				INTO THREAD_ID_v			, TRX_STATE_v,	THREAD_INFO_v,	`THREAD_USER_v`,	`THREAD_HOST_v`, `THREAD_STATE_v` 
				FROM INFORMATION_SCHEMA.INNODB_TRX trx JOIN INFORMATION_SCHEMA.PROCESSLIST ps ON trx.trx_mysql_thread_id = ps.ID 
					WHERE trx_started < TRANSACTION_TIME_LIMIT_v AND `USER` NOT IN ('system_user', 'root','event_scheduler') LIMIT 1;
			 
			if THREAD_ID_v IS NOT NULL AND THREAD_ID_v > 0 THEN	    
			
				-- Save logs...
				INSERT IGNORE INTO INNODB_TRX			SELECT * FROM information_schema.INNODB_TRX;
				INSERT IGNORE INTO INNODB_LOCKS			SELECT * FROM information_schema.INNODB_LOCKS;
				INSERT IGNORE INTO INNODB_LOCK_WAITS	SELECT * FROM information_schema.INNODB_LOCK_WAITS;
				INSERT IGNORE INTO PROCESSLIST			SELECT *, THREAD_ID_v FROM information_schema.PROCESSLIST;
			
				SET @THREAD_ID = THREAD_ID_v;
			    EXECUTE p_Kill USING @THREAD_ID;
			
			    INSERT INTO GHOST_TRX_LOG	(`thread_id`,	`thread_user`,		`thread_host`, 		`thread_info`,		`thread_status`, `status`) 
			    	VALUES 					(`THREAD_ID_v`,	`THREAD_USER_v`,	`THREAD_HOST_v`,	`THREAD_INFO_v`,	`THREAD_STATE_v`, 1);
			ELSE
			    INSERT INTO GHOST_TRX_LOG	(`thread_id`,	`thread_user`,	`thread_host`, 		`thread_info`,		`thread_status`, `status`) 
			    	VALUES 					(`THREAD_ID_v`,	`THREAD_USER_v`,	`THREAD_HOST_v`,	`THREAD_INFO_v`,	`THREAD_STATE_v`, 3);
			END IF;
			SET __done = 0;
			
		END LOOP;
		CLOSE GHOST_TRX_CURSOR;	
		
		IF p_Kill  THEN
			DEALLOCATE PREPARE p_Kill;
		END IF;
	END IF;	
	-- DELETE FROM GHOST_TRX_LOG WHERE insert_timestamp < NOW() - INTERVAL 31 DAY;
	END MAIN_BLOCK;

  1. No comments yet.

  1. No trackbacks yet.