During some server reboots at our organization over the weekend, several nodes in our non-Prod cluster had some crashes, and in the end, 7 of 84 durable queues in the cluster were gone. All nodes are running RabbitMQ 3.3.3 on Erlang R16B03-1. This is a 3-node cluster, with cluster_partition_handling = autoheal. I'm curious if this is a known issue with autoheal? I tried recreating the issue this morning by recreating the series of events, but was unable to. Our other 2-node non-Prod cluster, with no cluster_partition_handling, was also rebootted, but didn't have any problems. Thanks, Andrew Miller 08:12:01 - Node DIX-D-EVGAPP-02 (a disk node) saw node STR-D-EVGAPP-02 (a memory node) go down (which had been the master node for many queues), and was promoted to master for 75 queues, including those which later would disappear. 08:13:30 - Node DIX-D-EVGAPP-02 saw node STR-D-EVGAPP-02 come back, and began synchronizing back. Most queues synchronized fine, but 7 crashed with errors similar to the following - and each of these queues were gone after this. The third node (another disk node) doesn't have any crashes in its log around this time. =ERROR REPORT==== 27-Jul-2014::08:13:30 === ** Generic server <0.332.0> terminating ** Last message in was {add_on_right,{20,<4967.323.0>}} ** When Server state == {q, {amqqueue, {resource,<<"eventing">>,queue, <<"SalesCenter2.BPClaims_ClosedClaim">>}, true,false,none, [{<<"x-dead-letter-exchange">>,longstr, <<"EnterpriseEventDeadLetter">>}, {<<"x-dead-letter-routing-key">>,longstr, <<"SalesCenter2.BPClaims_ClosedClaim">>}], <0.332.0>, [<8174.310.0>], [<8174.310.0>], [{vhost,<<"eventing">>}, {name,<<"ha-all">>}, {pattern,<<"^.*">>}, {'apply-to',<<"queues">>}, {definition, [{<<"ha-mode">>,<<"all">>}, {<<"ha-sync-mode">>,<<"automatic">>}]}, {priority,0}], [{<8174.311.0>,<8174.310.0>},{<0.333.0>,<0.332.0>}], []}, none,false,rabbit_mirror_queue_master, {state, {resource,<<"eventing">>,queue, <<"SalesCenter2.BPClaims_ClosedClaim">>}, <0.333.0>,<0.4223.0>,rabbit_variable_queue, {vqstate, {0,{[],[]}}, {0,{[],[]}}, {delta,undefined,0,undefined}, {0,{[],[]}}, {0,{[],[]}}, 0, {0,nil}, {0,nil}, {qistate, "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/queues/6GQ2ZHJGDF7X2GH1FS0QPJX2O", {{dict,0,16,16,8,80,48, {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[], []}, {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[], []}}}, []}, undefined,0,65536, #Fun<rabbit_variable_queue.2.51560583>, {0,nil}}, {{client_msstate,msg_store_persistent, <<26,131,31,121,222,176,41,144,168,192,42,159,222, 13,179,107>>, {dict,0,16,16,8,80,48, {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[], []}, {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[], []}}}, {state,311371, "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/msg_store_persistent"}, rabbit_msg_store_ets_index, "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/msg_store_persistent", <0.266.0>,315468,307274,319565,323662}, {client_msstate,msg_store_transient, <<32,49,130,217,86,65,129,214,32,140,24,34,254, 129,171,73>>, {dict,0,16,16,8,80,48, {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[], []}, {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[], []}}}, {state,290886, "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/msg_store_transient"}, rabbit_msg_store_ets_index, "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/msg_store_transient", <0.261.0>,294983,286789,299080,303177}}, true,0,0,0,infinity,0,0,0,0,0, {rates,0.0,0.0,0.0,0.0,{1406,466723,462160}}, {0,nil}, {0,nil}, {0,nil}, {0,nil}, 0,0}, {dict,0,16,16,8,80,48, {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[], []}}}, [], {set,0,16,16,8,80,48, {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[], []}}}}, {state,{queue,[],[],0},{active,1406466722448182,1.0}}, undefined,undefined,undefined,undefined, {state,fine,5000,undefined}, {0,nil}, undefined,undefined,undefined, {state, {dict,0,16,16,8,80,48, {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[], []}}}, delegate}, {resource,<<"eventing">>,exchange, <<"EnterpriseEventDeadLetter">>}, <<"SalesCenter2.BPClaims_ClosedClaim">>,undefined,2, running} ** Reason for termination == ** {function_clause, [{rabbit_amqqueue_process,handle_call, [{add_on_right,{20,<4967.323.0>}}, {<4967.323.0>,#Ref<4967.0.0.2348>}, {q,{amqqueue, {resource,<<"eventing">>,queue, <<"SalesCenter2.BPClaims_ClosedClaim">>}, true,false,none, [{<<"x-dead-letter-exchange">>,longstr, <<"EnterpriseEventDeadLetter">>}, {<<"x-dead-letter-routing-key">>,longstr, <<"SalesCenter2.BPClaims_ClosedClaim">>}], <0.332.0>, [<8174.310.0>], [<8174.310.0>], [{vhost,<<"eventing">>}, {name,<<"ha-all">>}, {pattern,<<"^.*">>}, {'apply-to',<<"queues">>}, {definition, [{<<"ha-mode">>,<<"all">>}, {<<"ha-sync-mode">>,<<"automatic">>}]}, {priority,0}], [{<8174.311.0>,<8174.310.0>},{<0.333.0>,<0.332.0>}], []}, none,false,rabbit_mirror_queue_master, {state, {resource,<<"eventing">>,queue, <<"SalesCenter2.BPClaims_ClosedClaim">>}, <0.333.0>,<0.4223.0>,rabbit_variable_queue, {vqstate, {0,{[],[]}}, {0,{[],[]}}, {delta,undefined,0,undefined}, {0,{[],[]}}, {0,{[],[]}}, 0, {0,nil}, {0,nil}, {qistate, "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/queues/6GQ2ZHJGDF7X2GH1FS0QPJX2O", {{dict,0,16,16,8,80,48, {[],[],[],[],[],[],[],[],[],[],[],[],[],[], [],[]}, {{[],[],[],[],[],[],[],[],[],[],[],[],[],[], [],[]}}}, []}, undefined,0,65536, #Fun<rabbit_variable_queue.2.51560583>, {0,nil}}, {{client_msstate,msg_store_persistent, <<26,131,31,121,222,176,41,144,168,192,42,159,222, 13,179,107>>, {dict,0,16,16,8,80,48, {[],[],[],[],[],[],[],[],[],[],[],[],[],[], [],[]}, {{[],[],[],[],[],[],[],[],[],[],[],[],[],[], [],[]}}}, {state,311371, "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/msg_store_persistent"}, rabbit_msg_store_ets_index, "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/msg_store_persistent", <0.266.0>,315468,307274,319565,323662}, {client_msstate,msg_store_transient, <<32,49,130,217,86,65,129,214,32,140,24,34,254, 129,171,73>>, {dict,0,16,16,8,80,48, {[],[],[],[],[],[],[],[],[],[],[],[],[],[], [],[]}, {{[],[],[],[],[],[],[],[],[],[],[],[],[],[], [],[]}}}, {state,290886, "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/msg_store_transient"}, rabbit_msg_store_ets_index, "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/msg_store_transient", <0.261.0>,294983,286789,299080,303177}}, true,0,0,0,infinity,0,0,0,0,0, {rates,0.0,0.0,0.0,0.0,{1406,466723,462160}}, {0,nil}, {0,nil}, {0,nil}, {0,nil}, 0,0}, {dict,0,16,16,8,80,48, {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}}, [], {set,0,16,16,8,80,48, {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}}}, {state,{queue,[],[],0},{active,1406466722448182,1.0}}, undefined,undefined,undefined,undefined, {state,fine,5000,undefined}, {0,nil}, undefined,undefined,undefined, {state, {dict,0,16,16,8,80,48, {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}}, delegate}, {resource,<<"eventing">>,exchange, <<"EnterpriseEventDeadLetter">>}, <<"SalesCenter2.BPClaims_ClosedClaim">>,undefined,2,running}], []}, {gen_server2,handle_msg,2,[]}, {proc_lib,wake_up,3,[{file,"proc_lib.erl"},{line,249}]}]} 08:13:30 - Node STR-D-EVGAPP-02 also recorded crashes involving the same queues: =ERROR REPORT==== 27-Jul-2014::08:13:30 === ** Generic server <0.323.0> terminating ** Last message in was {'$gen_cast',join} ** When Server state == {state, {20,<0.323.0>}, {{20,<0.323.0>},undefined}, {{20,<0.323.0>},undefined}, {resource,<<"eventing">>,queue, <<"Insight.BPClaims_ModifiedWorkNotAvailable">>}, rabbit_mirror_queue_slave,undefined,-1,undefined, [<0.322.0>], {[],[]}, [],0,undefined, #Fun<rabbit_misc.execute_mnesia_transaction.1>} ** Reason for termination == ** {{function_clause, [{rabbit_amqqueue_process,handle_call, [{add_on_right,{20,<0.323.0>}}, {<0.323.0>,#Ref<0.0.0.2348>}, {q,{amqqueue, {resource,<<"eventing">>,queue, <<"SalesCenter2.BPClaims_ClosedClaim">>}, true,false,none, [{<<"x-dead-letter-exchange">>,longstr, <<"EnterpriseEventDeadLetter">>}, {<<"x-dead-letter-routing-key">>,longstr, <<"SalesCenter2.BPClaims_ClosedClaim">>}], <5297.332.0>, [<5251.310.0>], [<5251.310.0>], [{vhost,<<"eventing">>}, {name,<<"ha-all">>}, {pattern,<<"^.*">>}, {'apply-to',<<"queues">>}, {definition, [{<<"ha-mode">>,<<"all">>}, {<<"ha-sync-mode">>,<<"automatic">>}]}, {priority,0}], [{<5251.311.0>,<5251.310.0>},{<5297.333.0>,<5297.332.0>}], []}, none,false,rabbit_mirror_queue_master, {state, {resource,<<"eventing">>,queue, <<"SalesCenter2.BPClaims_ClosedClaim">>}, <5297.333.0>,<5297.4223.0>,rabbit_variable_queue, {vqstate, {0,{[],[]}}, {0,{[],[]}}, {delta,undefined,0,undefined}, {0,{[],[]}}, {0,{[],[]}}, 0, {0,nil}, {0,nil}, {qistate, "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/queues/6GQ2ZHJGDF7X2GH1FS0QPJX2O", {{dict,0,16,16,8,80,48, {[],[],[],[],[],[],[],[],[],[],[],[],[],[], [],[]}, {{[],[],[],[],[],[],[],[],[],[],[],[],[],[], [],[]}}}, []}, undefined,0,65536, #Fun<rabbit_variable_queue.2.51560583>, {0,nil}}, {{client_msstate,msg_store_persistent, <<26,131,31,121,222,176,41,144,168,192,42,159, 222,13,179,107>>, {dict,0,16,16,8,80,48, {[],[],[],[],[],[],[],[],[],[],[],[],[],[], [],[]}, {{[],[],[],[],[],[],[],[],[],[],[],[],[],[], [],[]}}}, {state,311371, "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/msg_store_persistent"}, rabbit_msg_store_ets_index, "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/msg_store_persistent", <5297.266.0>,315468,307274,319565,323662}, {client_msstate,msg_store_transient, <<32,49,130,217,86,65,129,214,32,140,24,34,254, 129,171,73>>, {dict,0,16,16,8,80,48, {[],[],[],[],[],[],[],[],[],[],[],[],[],[], [],[]}, {{[],[],[],[],[],[],[],[],[],[],[],[],[],[], [],[]}}}, {state,290886, "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/msg_store_transient"}, rabbit_msg_store_ets_index, "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/msg_store_transient", <5297.261.0>,294983,286789,299080,303177}}, true,0,0,0,infinity,0,0,0,0,0, {rates,0.0,0.0,0.0,0.0,{1406,466723,462160}}, {0,nil}, {0,nil}, {0,nil}, {0,nil}, 0,0}, {dict,0,16,16,8,80,48, {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}}, [], {set,0,16,16,8,80,48, {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}}}, {state,{queue,[],[],0},{active,1406466722448182,1.0}}, undefined,undefined,undefined,undefined, {state,fine,5000,undefined}, {0,nil}, undefined,undefined,undefined, {state, {dict,0,16,16,8,80,48, {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}}, delegate}, {resource,<<"eventing">>,exchange, <<"EnterpriseEventDeadLetter">>}, <<"SalesCenter2.BPClaims_ClosedClaim">>,undefined,2,running}], []}, {gen_server2,handle_msg,2,[]}, {proc_lib,wake_up,3,[{file,"proc_lib.erl"},{line,249}]}]}, {gen_server2,call,[<5297.332.0>,{add_on_right,{20,<0.323.0>}},infinity]}} 08:13:31 - And again from node STR-D-EVGAPP-02: =ERROR REPORT==== 27-Jul-2014::08:13:31 === ** Generic server <0.322.0> terminating ** Last message in was {'EXIT',<0.323.0>, {{function_clause, [{rabbit_amqqueue_process,handle_call, [{add_on_right,{20,<0.323.0>}}, {<0.323.0>,#Ref<0.0.0.2348>}, {q, {amqqueue, {resource,<<"eventing">>,queue, <<"SalesCenter2.BPClaims_ClosedClaim">>}, true,false,none, [{<<"x-dead-letter-exchange">>,longstr, <<"EnterpriseEventDeadLetter">>}, {<<"x-dead-letter-routing-key">>,longstr, <<"SalesCenter2.BPClaims_ClosedClaim">>}], <5297.332.0>, [<5251.310.0>], [<5251.310.0>], [{vhost,<<"eventing">>}, {name,<<"ha-all">>}, {pattern,<<"^.*">>}, {'apply-to',<<"queues">>}, {definition, [{<<"ha-mode">>,<<"all">>}, {<<"ha-sync-mode">>,<<"automatic">>}]}, {priority,0}], [{<5251.311.0>,<5251.310.0>}, {<5297.333.0>,<5297.332.0>}], []}, none,false,rabbit_mirror_queue_master, {state, {resource,<<"eventing">>,queue, <<"SalesCenter2.BPClaims_ClosedClaim">>}, <5297.333.0>,<5297.4223.0>, rabbit_variable_queue, {vqstate, {0,{[],[]}}, {0,{[],[]}}, {delta,undefined,0,undefined}, {0,{[],[]}}, {0,{[],[]}}, 0, {0,nil}, {0,nil}, {qistate, "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/queues/6GQ2ZHJGDF7X2GH1FS0QPJX2O", {{dict,0,16,16,8,80,48, {[],[],[],[],[],[],[],[],[],[],[],[],[],[], [],[]}, {{[],[],[],[],[],[],[],[],[],[],[],[],[], [],[],[]}}}, []}, undefined,0,65536, #Fun<rabbit_variable_queue.2.51560583>, {0,nil}}, {{client_msstate,msg_store_persistent, <<26,131,31,121,222,176,41,144,168,192,42, 159,222,13,179,107>>, {dict,0,16,16,8,80,48, {[],[],[],[],[],[],[],[],[],[],[],[],[],[], [],[]}, {{[],[],[],[],[],[],[],[],[],[],[],[],[], [],[],[]}}}, {state,311371, "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/msg_store_persistent"}, rabbit_msg_store_ets_index, "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/msg_store_persistent", <5297.266.0>,315468,307274,319565,323662}, {client_msstate,msg_store_transient, <<32,49,130,217,86,65,129,214,32,140,24,34, 254,129,171,73>>, {dict,0,16,16,8,80,48, {[],[],[],[],[],[],[],[],[],[],[],[],[],[], [],[]}, {{[],[],[],[],[],[],[],[],[],[],[],[],[], [],[],[]}}}, {state,290886, "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/msg_store_transient"}, rabbit_msg_store_ets_index, "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/msg_store_transient", <5297.261.0>,294983,286789,299080,303177}}, true,0,0,0,infinity,0,0,0,0,0, {rates,0.0,0.0,0.0,0.0,{1406,466723,462160}}, {0,nil}, {0,nil}, {0,nil}, {0,nil}, 0,0}, {dict,0,16,16,8,80,48, {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[], []}, {{[],[],[],[],[],[],[],[],[],[],[],[],[],[], [],[]}}}, [], {set,0,16,16,8,80,48, {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[], []}, {{[],[],[],[],[],[],[],[],[],[],[],[],[],[], [],[]}}}}, {state, {queue,[],[],0}, {active,1406466722448182,1.0}}, undefined,undefined,undefined,undefined, {state,fine,5000,undefined}, {0,nil}, undefined,undefined,undefined, {state, {dict,0,16,16,8,80,48, {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[], []}, {{[],[],[],[],[],[],[],[],[],[],[],[],[],[], [],[]}}}, delegate}, {resource,<<"eventing">>,exchange, <<"EnterpriseEventDeadLetter">>}, <<"SalesCenter2.BPClaims_ClosedClaim">>, undefined,2,running}], []}, {gen_server2,handle_msg,2,[]}, {proc_lib,wake_up,3, [{file,"proc_lib.erl"},{line,249}]}]}, {gen_server2,call, [<5297.332.0>, {add_on_right,{20,<0.323.0>}}, infinity]}}} ** When Server state == {state, {amqqueue, {resource,<<"eventing">>,queue, <<"Insight.BPClaims_ModifiedWorkNotAvailable">>}, true,false,none, [{<<"x-dead-letter-exchange">>,longstr, <<"EnterpriseEventDeadLetter">>}, {<<"x-dead-letter-routing-key">>,longstr, <<"Insight.BPClaims_ModifiedWorkNotAvailable">>}], <5251.335.0>, [<5297.331.0>], [<5297.331.0>], [{vhost,<<"eventing">>}, {name,<<"ha-all">>}, {pattern,<<"^.*">>}, {'apply-to',<<"queues">>}, {definition, [{<<"ha-mode">>,<<"all">>}, {<<"ha-sync-mode">>,<<"automatic">>}]}, {priority,0}], [{<5297.332.0>,<5297.331.0>}, {<5251.336.0>,<5251.335.0>}], []}, <0.323.0>,rabbit_variable_queue, {vqstate, {0,{[],[]}}, {0,{[],[]}}, {delta,undefined,0,undefined}, {0,{[],[]}}, {0,{[],[]}}, 0, {0,nil}, {0,nil}, {qistate, "d:/RabbitMQ/db/rabbit@STR-D-EVGAPP-02-mnesia/queues/11YNGR2UMDIGSF4OWRUUJ2HUX", {{dict,0,16,16,8,80,48, {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[], []}}}, []}, undefined,0,65536, #Fun<rabbit_variable_queue.2.51560583>, {0,nil}}, {{client_msstate,msg_store_persistent, <<142,49,10,175,39,218,164,74,2,215,253,147,201, 168,141,2>>, {dict,0,16,16,8,80,48, {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[], []}}}, {state,303177, "d:/RabbitMQ/db/rabbit@STR-D-EVGAPP-02-mnesia/msg_store_persistent"}, rabbit_msg_store_ets_index, "d:/RabbitMQ/db/rabbit@STR-D-EVGAPP-02-mnesia/msg_store_persistent", <0.251.0>,307274,299080,311371,315468}, {client_msstate,msg_store_transient, <<18,152,170,252,55,149,243,37,252,234,131,180,53, 66,14,182>>, {dict,0,16,16,8,80,48, {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[], []}}}, {state,282692, "d:/RabbitMQ/db/rabbit@STR-D-EVGAPP-02-mnesia/msg_store_transient"}, rabbit_msg_store_ets_index, "d:/RabbitMQ/db/rabbit@STR-D-EVGAPP-02-mnesia/msg_store_transient", <0.246.0>,286789,278595,290886,294983}}, true,0,0,0,infinity,0,0,0,0,0, {rates,0.0,0.0,0.0,0.0,{1406,466811,43211}}, {0,nil}, {0,nil}, {0,nil}, {0,nil}, 0,0}, undefined,undefined, {dict,0,16,16,8,80,48, {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}}, {dict,0,16,16,8,80,48, {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}}, {dict,0,16,16,8,80,48, {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}}, {state, {dict,0,16,16,8,80,48, {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[], []}}}, delegate}, undefined} ** Reason for termination == ** {{function_clause, [{rabbit_amqqueue_process,handle_call, [{add_on_right,{20,<0.323.0>}}, {<0.323.0>,#Ref<0.0.0.2348>}, {q,{amqqueue, {resource,<<"eventing">>,queue, <<"SalesCenter2.BPClaims_ClosedClaim">>}, true,false,none, [{<<"x-dead-letter-exchange">>,longstr, <<"EnterpriseEventDeadLetter">>}, {<<"x-dead-letter-routing-key">>,longstr, <<"SalesCenter2.BPClaims_ClosedClaim">>}], <5297.332.0>, [<5251.310.0>], [<5251.310.0>], [{vhost,<<"eventing">>}, {name,<<"ha-all">>}, {pattern,<<"^.*">>}, {'apply-to',<<"queues">>}, {definition, [{<<"ha-mode">>,<<"all">>}, {<<"ha-sync-mode">>,<<"automatic">>}]}, {priority,0}], [{<5251.311.0>,<5251.310.0>},{<5297.333.0>,<5297.332.0>}], []}, none,false,rabbit_mirror_queue_master, {state, {resource,<<"eventing">>,queue, <<"SalesCenter2.BPClaims_ClosedClaim">>}, <5297.333.0>,<5297.4223.0>,rabbit_variable_queue, {vqstate, {0,{[],[]}}, {0,{[],[]}}, {delta,undefined,0,undefined}, {0,{[],[]}}, {0,{[],[]}}, 0, {0,nil}, {0,nil}, {qistate, "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/queues/6GQ2ZHJGDF7X2GH1FS0QPJX2O", {{dict,0,16,16,8,80,48, {[],[],[],[],[],[],[],[],[],[],[],[],[],[], [],[]}, {{[],[],[],[],[],[],[],[],[],[],[],[],[],[], [],[]}}}, []}, undefined,0,65536, #Fun<rabbit_variable_queue.2.51560583>, {0,nil}}, {{client_msstate,msg_store_persistent, <<26,131,31,121,222,176,41,144,168,192,42,159, 222,13,179,107>>, {dict,0,16,16,8,80,48, {[],[],[],[],[],[],[],[],[],[],[],[],[],[], [],[]}, {{[],[],[],[],[],[],[],[],[],[],[],[],[],[], [],[]}}}, {state,311371, "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/msg_store_persistent"}, rabbit_msg_store_ets_index, "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/msg_store_persistent", <5297.266.0>,315468,307274,319565,323662}, {client_msstate,msg_store_transient, <<32,49,130,217,86,65,129,214,32,140,24,34,254, 129,171,73>>, {dict,0,16,16,8,80,48, {[],[],[],[],[],[],[],[],[],[],[],[],[],[], [],[]}, {{[],[],[],[],[],[],[],[],[],[],[],[],[],[], [],[]}}}, {state,290886, "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/msg_store_transient"}, rabbit_msg_store_ets_index, "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/msg_store_transient", <5297.261.0>,294983,286789,299080,303177}}, true,0,0,0,infinity,0,0,0,0,0, {rates,0.0,0.0,0.0,0.0,{1406,466723,462160}}, {0,nil}, {0,nil}, {0,nil}, {0,nil}, 0,0}, {dict,0,16,16,8,80,48, {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}}, [], {set,0,16,16,8,80,48, {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}}}, {state,{queue,[],[],0},{active,1406466722448182,1.0}}, undefined,undefined,undefined,undefined, {state,fine,5000,undefined}, {0,nil}, undefined,undefined,undefined, {state, {dict,0,16,16,8,80,48, {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}}, delegate}, {resource,<<"eventing">>,exchange, <<"EnterpriseEventDeadLetter">>}, <<"SalesCenter2.BPClaims_ClosedClaim">>,undefined,2,running}], []}, {gen_server2,handle_msg,2,[]}, {proc_lib,wake_up,3,[{file,"proc_lib.erl"},{line,249}]}]}, {gen_server2,call,[<5297.332.0>,{add_on_right,{20,<0.323.0>}},infinity]}} This e-mail is confidential. If you are not the intended recipient, you must not disclose or use the information contained in it. If you have received this e-mail in error, please tell us immediately by return e-mail and delete the document. No recipient may use the information in this e-mail in violation of any civil or criminal statute. Sentry disclaims all liability for any unauthorized uses of this e-mail or its contents. Sentry accepts no liability or responsibility for any damage caused by any virus transmitted with this e-mail. _______________________________________________ rabbitmq-discuss mailing list has moved to https://groups.google.com/forum/#!forum/rabbitmq-users, please subscribe to the new list! [hidden email] https://lists.rabbitmq.com/cgi-bin/mailman/listinfo/rabbitmq-discuss |
Thanks for this. This is not a known issue, and not something we've seen
before. In all cases it looks like a queue process crashed because it received a message intended for a GM process. That should never be able to happen. Could you send me full logs from all nodes around the time of the crash? It would be useful to see in what order things happened, if anything led up to this, when nodes restarted for autoheal (if any autoheal happened), etc. Cheers, Simon On 28/07/14 17:40, Miller Andrew wrote: > During some server reboots at our organization over the weekend, several > nodes in our non-Prod cluster had some crashes, and in the end, 7 of 84 > durable queues in the cluster were gone. All nodes are running RabbitMQ > 3.3.3 on Erlang R16B03-1. This is a 3-node cluster, with > cluster_partition_handling = autoheal. I'm curious if this is a known > issue with autoheal? I tried recreating the issue this morning by > recreating the series of events, but was unable to. Our other 2-node > non-Prod cluster, with no cluster_partition_handling, was also > rebootted, but didn't have any problems. > > Thanks, > > Andrew Miller > > 08:12:01 - Node DIX-D-EVGAPP-02 (a disk node) saw node STR-D-EVGAPP-02 > (a memory node) go down (which had been the master node for many > queues), and was promoted to master for 75 queues, including those which > later would disappear. > > 08:13:30 - Node DIX-D-EVGAPP-02 saw node STR-D-EVGAPP-02 come back, and > began synchronizing back. Most queues synchronized fine, but 7 crashed > with errors similar to the following - and each of these queues were > gone after this. The third node (another disk node) doesn't have any > crashes in its log around this time. > > =ERROR REPORT==== 27-Jul-2014::08:13:30 === > > ** Generic server <0.332.0> terminating > > ** Last message in was {add_on_right,{20,<4967.323.0>}} > > ** When Server state == {q, > > {amqqueue, > > {resource,<<"eventing">>,queue, > > <<"SalesCenter2.BPClaims_ClosedClaim">>}, > > true,false,none, > > [{<<"x-dead-letter-exchange">>,longstr, > > <<"EnterpriseEventDeadLetter">>}, > > {<<"x-dead-letter-routing-key">>,longstr, > > <<"SalesCenter2.BPClaims_ClosedClaim">>}], > > <0.332.0>, > > [<8174.310.0>], > > [<8174.310.0>], > > [{vhost,<<"eventing">>}, > > {name,<<"ha-all">>}, > > {pattern,<<"^.*">>}, > > {'apply-to',<<"queues">>}, > > {definition, > > [{<<"ha-mode">>,<<"all">>}, > > {<<"ha-sync-mode">>,<<"automatic">>}]}, > > {priority,0}], > > > [{<8174.311.0>,<8174.310.0>},{<0.333.0>,<0.332.0>}], > > []}, > > none,false,rabbit_mirror_queue_master, > > {state, > > {resource,<<"eventing">>,queue, > > <<"SalesCenter2.BPClaims_ClosedClaim">>}, > > <0.333.0>,<0.4223.0>,rabbit_variable_queue, > > {vqstate, > > {0,{[],[]}}, > > {0,{[],[]}}, > > {delta,undefined,0,undefined}, > > {0,{[],[]}}, > > {0,{[],[]}}, > > 0, > > {0,nil}, > > {0,nil}, > > {qistate, > > > "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/queues/6GQ2ZHJGDF7X2GH1FS0QPJX2O", > > {{dict,0,16,16,8,80,48, > > > {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[], > > []}, > > > {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[], > > []}}}, > > []}, > > undefined,0,65536, > > #Fun<rabbit_variable_queue.2.51560583>, > > {0,nil}}, > > {{client_msstate,msg_store_persistent, > > > <<26,131,31,121,222,176,41,144,168,192,42,159,222, > > 13,179,107>>, > > {dict,0,16,16,8,80,48, > > > {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[], > > []}, > > > {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[], > > []}}}, > > {state,311371, > > > "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/msg_store_persistent"}, > > rabbit_msg_store_ets_index, > > > "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/msg_store_persistent", > > <0.266.0>,315468,307274,319565,323662}, > > {client_msstate,msg_store_transient, > > > <<32,49,130,217,86,65,129,214,32,140,24,34,254, > > 129,171,73>>, > > {dict,0,16,16,8,80,48, > > > {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[], > > []}, > > > {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[], > > []}}}, > > {state,290886, > > > "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/msg_store_transient"}, > > rabbit_msg_store_ets_index, > > > "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/msg_store_transient", > > <0.261.0>,294983,286789,299080,303177}}, > > true,0,0,0,infinity,0,0,0,0,0, > > {rates,0.0,0.0,0.0,0.0,{1406,466723,462160}}, > > {0,nil}, > > {0,nil}, > > {0,nil}, > > {0,nil}, > > 0,0}, > > {dict,0,16,16,8,80,48, > > > {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, > > {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[], > > []}}}, > > [], > > {set,0,16,16,8,80,48, > > > {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, > > {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[], > > []}}}}, > > > {state,{queue,[],[],0},{active,1406466722448182,1.0}}, > > undefined,undefined,undefined,undefined, > > {state,fine,5000,undefined}, > > {0,nil}, > > undefined,undefined,undefined, > > {state, > > {dict,0,16,16,8,80,48, > > > {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, > > {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[], > > []}}}, > > delegate}, > > {resource,<<"eventing">>,exchange, > > <<"EnterpriseEventDeadLetter">>}, > > > <<"SalesCenter2.BPClaims_ClosedClaim">>,undefined,2, > > running} > > ** Reason for termination == > > ** {function_clause, > > [{rabbit_amqqueue_process,handle_call, > > [{add_on_right,{20,<4967.323.0>}}, > > {<4967.323.0>,#Ref<4967.0.0.2348>}, > > {q,{amqqueue, > > {resource,<<"eventing">>,queue, > > <<"SalesCenter2.BPClaims_ClosedClaim">>}, > > true,false,none, > > [{<<"x-dead-letter-exchange">>,longstr, > > <<"EnterpriseEventDeadLetter">>}, > > {<<"x-dead-letter-routing-key">>,longstr, > > <<"SalesCenter2.BPClaims_ClosedClaim">>}], > > <0.332.0>, > > [<8174.310.0>], > > [<8174.310.0>], > > [{vhost,<<"eventing">>}, > > {name,<<"ha-all">>}, > > {pattern,<<"^.*">>}, > > {'apply-to',<<"queues">>}, > > {definition, > > [{<<"ha-mode">>,<<"all">>}, > > {<<"ha-sync-mode">>,<<"automatic">>}]}, > > {priority,0}], > > [{<8174.311.0>,<8174.310.0>},{<0.333.0>,<0.332.0>}], > > []}, > > none,false,rabbit_mirror_queue_master, > > {state, > > {resource,<<"eventing">>,queue, > > <<"SalesCenter2.BPClaims_ClosedClaim">>}, > > <0.333.0>,<0.4223.0>,rabbit_variable_queue, > > {vqstate, > > {0,{[],[]}}, > > {0,{[],[]}}, > > {delta,undefined,0,undefined}, > > {0,{[],[]}}, > > {0,{[],[]}}, > > 0, > > {0,nil}, > > {0,nil}, > > {qistate, > > > "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/queues/6GQ2ZHJGDF7X2GH1FS0QPJX2O", > > {{dict,0,16,16,8,80,48, > > > {[],[],[],[],[],[],[],[],[],[],[],[],[],[], > > [],[]}, > > > {{[],[],[],[],[],[],[],[],[],[],[],[],[],[], > > [],[]}}}, > > []}, > > undefined,0,65536, > > #Fun<rabbit_variable_queue.2.51560583>, > > {0,nil}}, > > {{client_msstate,msg_store_persistent, > > > <<26,131,31,121,222,176,41,144,168,192,42,159,222, > > 13,179,107>>, > > {dict,0,16,16,8,80,48, > > > {[],[],[],[],[],[],[],[],[],[],[],[],[],[], > > [],[]}, > > > {{[],[],[],[],[],[],[],[],[],[],[],[],[],[], > > [],[]}}}, > > {state,311371, > > > "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/msg_store_persistent"}, > > rabbit_msg_store_ets_index, > > > "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/msg_store_persistent", > > <0.266.0>,315468,307274,319565,323662}, > > {client_msstate,msg_store_transient, > > > <<32,49,130,217,86,65,129,214,32,140,24,34,254, > > 129,171,73>>, > > {dict,0,16,16,8,80,48, > > > {[],[],[],[],[],[],[],[],[],[],[],[],[],[], > > [],[]}, > > > {{[],[],[],[],[],[],[],[],[],[],[],[],[],[], > > [],[]}}}, > > {state,290886, > > > "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/msg_store_transient"}, > > rabbit_msg_store_ets_index, > > > "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/msg_store_transient", > > <0.261.0>,294983,286789,299080,303177}}, > > true,0,0,0,infinity,0,0,0,0,0, > > {rates,0.0,0.0,0.0,0.0,{1406,466723,462160}}, > > {0,nil}, > > {0,nil}, > > {0,nil}, > > {0,nil}, > > 0,0}, > > {dict,0,16,16,8,80,48, > > {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, > > > {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}}, > > [], > > {set,0,16,16,8,80,48, > > {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, > > > {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}}}, > > {state,{queue,[],[],0},{active,1406466722448182,1.0}}, > > undefined,undefined,undefined,undefined, > > {state,fine,5000,undefined}, > > {0,nil}, > > undefined,undefined,undefined, > > {state, > > {dict,0,16,16,8,80,48, > > {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, > > > {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}}, > > delegate}, > > {resource,<<"eventing">>,exchange, > > <<"EnterpriseEventDeadLetter">>}, > > > <<"SalesCenter2.BPClaims_ClosedClaim">>,undefined,2,running}], > > []}, > > {gen_server2,handle_msg,2,[]}, > > {proc_lib,wake_up,3,[{file,"proc_lib.erl"},{line,249}]}]} > > 08:13:30 - Node STR-D-EVGAPP-02 also recorded crashes involving the same > queues: > > =ERROR REPORT==== 27-Jul-2014::08:13:30 === > > ** Generic server <0.323.0> terminating > > ** Last message in was {'$gen_cast',join} > > ** When Server state == {state, > > {20,<0.323.0>}, > > {{20,<0.323.0>},undefined}, > > {{20,<0.323.0>},undefined}, > > {resource,<<"eventing">>,queue, > > > <<"Insight.BPClaims_ModifiedWorkNotAvailable">>}, > > > rabbit_mirror_queue_slave,undefined,-1,undefined, > > [<0.322.0>], > > {[],[]}, > > [],0,undefined, > > #Fun<rabbit_misc.execute_mnesia_transaction.1>} > > ** Reason for termination == > > ** {{function_clause, > > [{rabbit_amqqueue_process,handle_call, > > [{add_on_right,{20,<0.323.0>}}, > > {<0.323.0>,#Ref<0.0.0.2348>}, > > {q,{amqqueue, > > {resource,<<"eventing">>,queue, > > <<"SalesCenter2.BPClaims_ClosedClaim">>}, > > true,false,none, > > [{<<"x-dead-letter-exchange">>,longstr, > > <<"EnterpriseEventDeadLetter">>}, > > {<<"x-dead-letter-routing-key">>,longstr, > > <<"SalesCenter2.BPClaims_ClosedClaim">>}], > > <5297.332.0>, > > [<5251.310.0>], > > [<5251.310.0>], > > [{vhost,<<"eventing">>}, > > {name,<<"ha-all">>}, > > {pattern,<<"^.*">>}, > > {'apply-to',<<"queues">>}, > > {definition, > > [{<<"ha-mode">>,<<"all">>}, > > {<<"ha-sync-mode">>,<<"automatic">>}]}, > > {priority,0}], > > > [{<5251.311.0>,<5251.310.0>},{<5297.333.0>,<5297.332.0>}], > > []}, > > none,false,rabbit_mirror_queue_master, > > {state, > > {resource,<<"eventing">>,queue, > > <<"SalesCenter2.BPClaims_ClosedClaim">>}, > > <5297.333.0>,<5297.4223.0>,rabbit_variable_queue, > > {vqstate, > > {0,{[],[]}}, > > {0,{[],[]}}, > > {delta,undefined,0,undefined}, > > {0,{[],[]}}, > > {0,{[],[]}}, > > 0, > > {0,nil}, > > {0,nil}, > > {qistate, > > > "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/queues/6GQ2ZHJGDF7X2GH1FS0QPJX2O", > > {{dict,0,16,16,8,80,48, > > > {[],[],[],[],[],[],[],[],[],[],[],[],[],[], > > [],[]}, > > > {{[],[],[],[],[],[],[],[],[],[],[],[],[],[], > > [],[]}}}, > > []}, > > undefined,0,65536, > > #Fun<rabbit_variable_queue.2.51560583>, > > {0,nil}}, > > {{client_msstate,msg_store_persistent, > > > <<26,131,31,121,222,176,41,144,168,192,42,159, > > 222,13,179,107>>, > > {dict,0,16,16,8,80,48, > > > {[],[],[],[],[],[],[],[],[],[],[],[],[],[], > > [],[]}, > > > {{[],[],[],[],[],[],[],[],[],[],[],[],[],[], > > [],[]}}}, > > {state,311371, > > > "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/msg_store_persistent"}, > > rabbit_msg_store_ets_index, > > > "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/msg_store_persistent", > > <5297.266.0>,315468,307274,319565,323662}, > > {client_msstate,msg_store_transient, > > > <<32,49,130,217,86,65,129,214,32,140,24,34,254, > > 129,171,73>>, > > {dict,0,16,16,8,80,48, > > > {[],[],[],[],[],[],[],[],[],[],[],[],[],[], > > [],[]}, > > > {{[],[],[],[],[],[],[],[],[],[],[],[],[],[], > > [],[]}}}, > > {state,290886, > > > "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/msg_store_transient"}, > > rabbit_msg_store_ets_index, > > > "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/msg_store_transient", > > <5297.261.0>,294983,286789,299080,303177}}, > > true,0,0,0,infinity,0,0,0,0,0, > > {rates,0.0,0.0,0.0,0.0,{1406,466723,462160}}, > > {0,nil}, > > {0,nil}, > > {0,nil}, > > {0,nil}, > > 0,0}, > > {dict,0,16,16,8,80,48, > > {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, > > > {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}}, > > [], > > {set,0,16,16,8,80,48, > > {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, > > > {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}}}, > > {state,{queue,[],[],0},{active,1406466722448182,1.0}}, > > undefined,undefined,undefined,undefined, > > {state,fine,5000,undefined}, > > {0,nil}, > > undefined,undefined,undefined, > > {state, > > {dict,0,16,16,8,80,48, > > {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, > > > {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}}, > > delegate}, > > {resource,<<"eventing">>,exchange, > > <<"EnterpriseEventDeadLetter">>}, > > > <<"SalesCenter2.BPClaims_ClosedClaim">>,undefined,2,running}], > > []}, > > {gen_server2,handle_msg,2,[]}, > > {proc_lib,wake_up,3,[{file,"proc_lib.erl"},{line,249}]}]}, > > > {gen_server2,call,[<5297.332.0>,{add_on_right,{20,<0.323.0>}},infinity]}} > > 08:13:31 - And again from node STR-D-EVGAPP-02: > > =ERROR REPORT==== 27-Jul-2014::08:13:31 === > > ** Generic server <0.322.0> terminating > > ** Last message in was {'EXIT',<0.323.0>, > > {{function_clause, > > [{rabbit_amqqueue_process,handle_call, > > [{add_on_right,{20,<0.323.0>}}, > > {<0.323.0>,#Ref<0.0.0.2348>}, > > {q, > > {amqqueue, > > {resource,<<"eventing">>,queue, > > <<"SalesCenter2.BPClaims_ClosedClaim">>}, > > true,false,none, > > [{<<"x-dead-letter-exchange">>,longstr, > > <<"EnterpriseEventDeadLetter">>}, > > {<<"x-dead-letter-routing-key">>,longstr, > > <<"SalesCenter2.BPClaims_ClosedClaim">>}], > > <5297.332.0>, > > [<5251.310.0>], > > [<5251.310.0>], > > [{vhost,<<"eventing">>}, > > {name,<<"ha-all">>}, > > {pattern,<<"^.*">>}, > > {'apply-to',<<"queues">>}, > > {definition, > > [{<<"ha-mode">>,<<"all">>}, > > {<<"ha-sync-mode">>,<<"automatic">>}]}, > > {priority,0}], > > [{<5251.311.0>,<5251.310.0>}, > > {<5297.333.0>,<5297.332.0>}], > > []}, > > none,false,rabbit_mirror_queue_master, > > {state, > > {resource,<<"eventing">>,queue, > > <<"SalesCenter2.BPClaims_ClosedClaim">>}, > > <5297.333.0>,<5297.4223.0>, > > rabbit_variable_queue, > > {vqstate, > > {0,{[],[]}}, > > {0,{[],[]}}, > > {delta,undefined,0,undefined}, > > {0,{[],[]}}, > > {0,{[],[]}}, > > 0, > > {0,nil}, > > {0,nil}, > > {qistate, > > > "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/queues/6GQ2ZHJGDF7X2GH1FS0QPJX2O", > > {{dict,0,16,16,8,80,48, > > > {[],[],[],[],[],[],[],[],[],[],[],[],[],[], > > [],[]}, > > > {{[],[],[],[],[],[],[],[],[],[],[],[],[], > > [],[],[]}}}, > > []}, > > undefined,0,65536, > > #Fun<rabbit_variable_queue.2.51560583>, > > {0,nil}}, > > {{client_msstate,msg_store_persistent, > > > <<26,131,31,121,222,176,41,144,168,192,42, > > 159,222,13,179,107>>, > > {dict,0,16,16,8,80,48, > > > {[],[],[],[],[],[],[],[],[],[],[],[],[],[], > > [],[]}, > > > {{[],[],[],[],[],[],[],[],[],[],[],[],[], > > [],[],[]}}}, > > {state,311371, > > > "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/msg_store_persistent"}, > > rabbit_msg_store_ets_index, > > > "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/msg_store_persistent", > > > <5297.266.0>,315468,307274,319565,323662}, > > {client_msstate,msg_store_transient, > > > <<32,49,130,217,86,65,129,214,32,140,24,34, > > 254,129,171,73>>, > > {dict,0,16,16,8,80,48, > > > {[],[],[],[],[],[],[],[],[],[],[],[],[],[], > > [],[]}, > > > {{[],[],[],[],[],[],[],[],[],[],[],[],[], > > [],[],[]}}}, > > {state,290886, > > > "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/msg_store_transient"}, > > rabbit_msg_store_ets_index, > > > "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/msg_store_transient", > > > <5297.261.0>,294983,286789,299080,303177}}, > > true,0,0,0,infinity,0,0,0,0,0, > > > {rates,0.0,0.0,0.0,0.0,{1406,466723,462160}}, > > {0,nil}, > > {0,nil}, > > {0,nil}, > > {0,nil}, > > 0,0}, > > {dict,0,16,16,8,80,48, > > > {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[], > > []}, > > > {{[],[],[],[],[],[],[],[],[],[],[],[],[],[], > > [],[]}}}, > > [], > > {set,0,16,16,8,80,48, > > > {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[], > > []}, > > > {{[],[],[],[],[],[],[],[],[],[],[],[],[],[], > > [],[]}}}}, > > {state, > > {queue,[],[],0}, > > {active,1406466722448182,1.0}}, > > undefined,undefined,undefined,undefined, > > {state,fine,5000,undefined}, > > {0,nil}, > > undefined,undefined,undefined, > > {state, > > {dict,0,16,16,8,80,48, > > > {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[], > > []}, > > > {{[],[],[],[],[],[],[],[],[],[],[],[],[],[], > > [],[]}}}, > > delegate}, > > {resource,<<"eventing">>,exchange, > > <<"EnterpriseEventDeadLetter">>}, > > <<"SalesCenter2.BPClaims_ClosedClaim">>, > > undefined,2,running}], > > []}, > > {gen_server2,handle_msg,2,[]}, > > {proc_lib,wake_up,3, > > [{file,"proc_lib.erl"},{line,249}]}]}, > > {gen_server2,call, > > [<5297.332.0>, > > {add_on_right,{20,<0.323.0>}}, > > infinity]}}} > > ** When Server state == {state, > > {amqqueue, > > {resource,<<"eventing">>,queue, > > > <<"Insight.BPClaims_ModifiedWorkNotAvailable">>}, > > true,false,none, > > [{<<"x-dead-letter-exchange">>,longstr, > > <<"EnterpriseEventDeadLetter">>}, > > {<<"x-dead-letter-routing-key">>,longstr, > > > <<"Insight.BPClaims_ModifiedWorkNotAvailable">>}], > > <5251.335.0>, > > [<5297.331.0>], > > [<5297.331.0>], > > [{vhost,<<"eventing">>}, > > {name,<<"ha-all">>}, > > {pattern,<<"^.*">>}, > > {'apply-to',<<"queues">>}, > > {definition, > > [{<<"ha-mode">>,<<"all">>}, > > {<<"ha-sync-mode">>,<<"automatic">>}]}, > > {priority,0}], > > [{<5297.332.0>,<5297.331.0>}, > > {<5251.336.0>,<5251.335.0>}], > > []}, > > <0.323.0>,rabbit_variable_queue, > > {vqstate, > > {0,{[],[]}}, > > {0,{[],[]}}, > > {delta,undefined,0,undefined}, > > {0,{[],[]}}, > > {0,{[],[]}}, > > 0, > > {0,nil}, > > {0,nil}, > > {qistate, > > > "d:/RabbitMQ/db/rabbit@STR-D-EVGAPP-02-mnesia/queues/11YNGR2UMDIGSF4OWRUUJ2HUX", > > {{dict,0,16,16,8,80,48, > > > {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, > > > {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[], > > []}}}, > > []}, > > undefined,0,65536, > > #Fun<rabbit_variable_queue.2.51560583>, > > {0,nil}}, > > {{client_msstate,msg_store_persistent, > > > <<142,49,10,175,39,218,164,74,2,215,253,147,201, > > 168,141,2>>, > > {dict,0,16,16,8,80,48, > > > {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, > > > {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[], > > []}}}, > > {state,303177, > > > "d:/RabbitMQ/db/rabbit@STR-D-EVGAPP-02-mnesia/msg_store_persistent"}, > > rabbit_msg_store_ets_index, > > > "d:/RabbitMQ/db/rabbit@STR-D-EVGAPP-02-mnesia/msg_store_persistent", > > <0.251.0>,307274,299080,311371,315468}, > > {client_msstate,msg_store_transient, > > > <<18,152,170,252,55,149,243,37,252,234,131,180,53, > > 66,14,182>>, > > {dict,0,16,16,8,80,48, > > > {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, > > > {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[], > > []}}}, > > {state,282692, > > > "d:/RabbitMQ/db/rabbit@STR-D-EVGAPP-02-mnesia/msg_store_transient"}, > > rabbit_msg_store_ets_index, > > > "d:/RabbitMQ/db/rabbit@STR-D-EVGAPP-02-mnesia/msg_store_transient", > > <0.246.0>,286789,278595,290886,294983}}, > > true,0,0,0,infinity,0,0,0,0,0, > > {rates,0.0,0.0,0.0,0.0,{1406,466811,43211}}, > > {0,nil}, > > {0,nil}, > > {0,nil}, > > {0,nil}, > > 0,0}, > > undefined,undefined, > > {dict,0,16,16,8,80,48, > > > {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, > > > {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}}, > > {dict,0,16,16,8,80,48, > > > {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, > > > {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}}, > > {dict,0,16,16,8,80,48, > > > {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, > > > {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}}, > > {state, > > {dict,0,16,16,8,80,48, > > > {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, > > {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[], > > []}}}, > > delegate}, > > undefined} > > ** Reason for termination == > > ** {{function_clause, > > [{rabbit_amqqueue_process,handle_call, > > [{add_on_right,{20,<0.323.0>}}, > > {<0.323.0>,#Ref<0.0.0.2348>}, > > {q,{amqqueue, > > {resource,<<"eventing">>,queue, > > <<"SalesCenter2.BPClaims_ClosedClaim">>}, > > true,false,none, > > [{<<"x-dead-letter-exchange">>,longstr, > > <<"EnterpriseEventDeadLetter">>}, > > {<<"x-dead-letter-routing-key">>,longstr, > > <<"SalesCenter2.BPClaims_ClosedClaim">>}], > > <5297.332.0>, > > [<5251.310.0>], > > [<5251.310.0>], > > [{vhost,<<"eventing">>}, > > {name,<<"ha-all">>}, > > {pattern,<<"^.*">>}, > > {'apply-to',<<"queues">>}, > > {definition, > > [{<<"ha-mode">>,<<"all">>}, > > {<<"ha-sync-mode">>,<<"automatic">>}]}, > > {priority,0}], > > > [{<5251.311.0>,<5251.310.0>},{<5297.333.0>,<5297.332.0>}], > > []}, > > none,false,rabbit_mirror_queue_master, > > {state, > > {resource,<<"eventing">>,queue, > > <<"SalesCenter2.BPClaims_ClosedClaim">>}, > > <5297.333.0>,<5297.4223.0>,rabbit_variable_queue, > > {vqstate, > > {0,{[],[]}}, > > {0,{[],[]}}, > > {delta,undefined,0,undefined}, > > {0,{[],[]}}, > > {0,{[],[]}}, > > 0, > > {0,nil}, > > {0,nil}, > > {qistate, > > > "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/queues/6GQ2ZHJGDF7X2GH1FS0QPJX2O", > > {{dict,0,16,16,8,80,48, > > > {[],[],[],[],[],[],[],[],[],[],[],[],[],[], > > [],[]}, > > > {{[],[],[],[],[],[],[],[],[],[],[],[],[],[], > > [],[]}}}, > > []}, > > undefined,0,65536, > > #Fun<rabbit_variable_queue.2.51560583>, > > {0,nil}}, > > {{client_msstate,msg_store_persistent, > > > <<26,131,31,121,222,176,41,144,168,192,42,159, > > 222,13,179,107>>, > > {dict,0,16,16,8,80,48, > > > {[],[],[],[],[],[],[],[],[],[],[],[],[],[], > > [],[]}, > > > {{[],[],[],[],[],[],[],[],[],[],[],[],[],[], > > [],[]}}}, > > {state,311371, > > > "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/msg_store_persistent"}, > > rabbit_msg_store_ets_index, > > > "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/msg_store_persistent", > > <5297.266.0>,315468,307274,319565,323662}, > > {client_msstate,msg_store_transient, > > > <<32,49,130,217,86,65,129,214,32,140,24,34,254, > > 129,171,73>>, > > {dict,0,16,16,8,80,48, > > > {[],[],[],[],[],[],[],[],[],[],[],[],[],[], > > [],[]}, > > > {{[],[],[],[],[],[],[],[],[],[],[],[],[],[], > > [],[]}}}, > > {state,290886, > > > "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/msg_store_transient"}, > > rabbit_msg_store_ets_index, > > > "d:/RabbitMQ/db/rabbit@DIX-D-EVGAPP-02-mnesia/msg_store_transient", > > <5297.261.0>,294983,286789,299080,303177}}, > > true,0,0,0,infinity,0,0,0,0,0, > > {rates,0.0,0.0,0.0,0.0,{1406,466723,462160}}, > > {0,nil}, > > {0,nil}, > > {0,nil}, > > {0,nil}, > > 0,0}, > > {dict,0,16,16,8,80,48, > > {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, > > > {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}}, > > [], > > {set,0,16,16,8,80,48, > > {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, > > > {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}}}, > > {state,{queue,[],[],0},{active,1406466722448182,1.0}}, > > undefined,undefined,undefined,undefined, > > {state,fine,5000,undefined}, > > {0,nil}, > > undefined,undefined,undefined, > > {state, > > {dict,0,16,16,8,80,48, > > {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, > > > {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}}, > > delegate}, > > {resource,<<"eventing">>,exchange, > > <<"EnterpriseEventDeadLetter">>}, > > > <<"SalesCenter2.BPClaims_ClosedClaim">>,undefined,2,running}], > > []}, > > {gen_server2,handle_msg,2,[]}, > > {proc_lib,wake_up,3,[{file,"proc_lib.erl"},{line,249}]}]}, > > > {gen_server2,call,[<5297.332.0>,{add_on_right,{20,<0.323.0>}},infinity]}} > > This e-mail is confidential. If you are not the intended recipient, you > must not disclose or use the information contained in it. If you have > received this e-mail in error, please tell us immediately by return > e-mail and delete the document. No recipient may use the information in > this e-mail in violation of any civil or criminal statute. Sentry > disclaims all liability for any unauthorized uses of this e-mail or its > contents. Sentry accepts no liability or responsibility for any damage > caused by any virus transmitted with this e-mail. > > > > _______________________________________________ > rabbitmq-discuss mailing list has moved to https://groups.google.com/forum/#!forum/rabbitmq-users, > please subscribe to the new list! > > [hidden email] > https://lists.rabbitmq.com/cgi-bin/mailman/listinfo/rabbitmq-discuss > _______________________________________________ rabbitmq-discuss mailing list has moved to https://groups.google.com/forum/#!forum/rabbitmq-users, please subscribe to the new list! [hidden email] https://lists.rabbitmq.com/cgi-bin/mailman/listinfo/rabbitmq-discuss |
Free forum by Nabble | Edit this page |