IDEMPIERE-728 Hazelcast Instance is not active exception on webui. Improve handling of shutdown/crash of hazelcast instance .

org.adempiere.base/src/org/compiere/util/CacheMgt.java

@@ -142,19 +142,23 @@ public class CacheMgt
 		if (service != null) {			
 			ResetCacheCallable callable = new ResetCacheCallable(tableName, recordId);
 			Future<Collection<Integer>> future = service.execute(callable, service.getMembers());
-			int total = 0;
+			if (future != null) {
-			try {
+				int total = 0;
-				Collection<Integer> results = future.get();
+				try {
-				for(Integer i : results) 
+					Collection<Integer> results = future.get();
-				{
+					for(Integer i : results) 
-					total += i.intValue();
+					{
+						total += i.intValue();
+					}
+				} catch (InterruptedException e) {
+					e.printStackTrace();
+				} catch (ExecutionException e) {
+					e.printStackTrace();
 				}
-			} catch (InterruptedException e) {
+				return total;
-				e.printStackTrace();
+			} else {
-			} catch (ExecutionException e) {
+				return resetLocalCache(tableName, recordId);
-				e.printStackTrace();
 			}
-			return total;
 		} else {
 			return resetLocalCache(tableName, recordId);
 		}
@@ -172,7 +176,9 @@ public class CacheMgt
 		IClusterService service = holder.getService();
 		if (service != null) {			
 			CacheNewRecordCallable callable = new CacheNewRecordCallable(tableName, recordId);
-			service.execute(callable, service.getMembers());
+			if (service.execute(callable, service.getMembers()) == null) {
+				localNewRecord(tableName, recordId);
+			}
 		} else {
 			localNewRecord(tableName, recordId);
 		}

org.idempiere.hazelcast.service/src/org/idempiere/hazelcast/service/Activator.java

@@ -17,10 +17,11 @@ import java.io.File;
 import java.io.FileNotFoundException;
 import java.net.MalformedURLException;
 import java.net.URL;
+import java.text.DateFormat;
+import java.util.Date;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
-import java.util.concurrent.atomic.AtomicReference;
 
 import org.compiere.Adempiere;
 import org.compiere.model.ServerStateChangeEvent;
@@ -46,7 +47,7 @@ public class Activator implements BundleActivator {
 	}
 
 	private volatile static HazelcastInstance hazelcastInstance;
-	private static AtomicReference<Future<?>> futureRef = new AtomicReference<Future<?>>();
+	private static Future<?> future;
 
 	/*
 	 * (non-Javadoc)
@@ -68,10 +69,10 @@ public class Activator implements BundleActivator {
 		}
 	}
 
-	private void createHazelCastInstance() {
+	private static synchronized void createHazelCastInstance() {
 		ScheduledThreadPoolExecutor executor = Adempiere.getThreadPoolExecutor();
 		
-		Future<?> future = executor.submit(new Runnable() {			
+		future = executor.submit(new Runnable() {			
 			@Override
 			public void run() {
 				String dataArea = System.getProperty("osgi.install.area");
@@ -92,11 +93,9 @@ public class Activator implements BundleActivator {
 				hazelcastInstance = Hazelcast.newHazelcastInstance(null);		
 			}
 		});
-		futureRef.set(future);
 	}
 
-	public static HazelcastInstance getHazelcastInstance() {
+	public static synchronized HazelcastInstance getHazelcastInstance() {
-		Future<?> future = futureRef.get();
 		if (future != null && !future.isDone()) {
 			try {
 				future.get();
@@ -105,6 +104,26 @@ public class Activator implements BundleActivator {
 			}		
 		}
 				
+		if (hazelcastInstance != null) {
+			if (!hazelcastInstance.getLifecycleService().isRunning()) {
+				System.err.println(DateFormat.getDateTimeInstance().format(new Date()) + " Hazelcast instance is down!");
+				//recreate
+				try {
+					hazelcastInstance = Hazelcast.newHazelcastInstance(null);
+					if (!hazelcastInstance.getLifecycleService().isRunning()) {
+						hazelcastInstance = null;
+						System.err.println(DateFormat.getDateTimeInstance().format(new Date()) + " Failed to re-create Hazelcast instance!");
+					} else {
+						System.err.println(DateFormat.getDateTimeInstance().format(new Date()) + " Hazelcast instance re-created!");
+					}
+				} catch (Throwable t) {
+					t.printStackTrace();
+					hazelcastInstance = null;
+					System.err.println(DateFormat.getDateTimeInstance().format(new Date()) + " Failed to re-create Hazelcast instance!");
+				}				
+			}						
+		}
+		
 		return hazelcastInstance;
 	}
 	
@@ -114,13 +133,13 @@ public class Activator implements BundleActivator {
 	 */
 	public void stop(BundleContext bundleContext) throws Exception {
 		Activator.context = null;
-		Future<?> future = futureRef.get();
+		synchronized (Activator.class) {
-		if (future != null && !future.isDone()) {
+			if (future != null && !future.isDone()) {
-			future.cancel(true); 
+				future.cancel(true); 
-		} else if (hazelcastInstance != null) {
+			} else if (hazelcastInstance != null) {
-			hazelcastInstance.getLifecycleService().shutdown();
+				hazelcastInstance.getLifecycleService().shutdown();
-			hazelcastInstance = null;
+				hazelcastInstance = null;
+			}
 		}
-		futureRef.set(null);
 	}
 }

org.idempiere.hazelcast.service/src/org/idempiere/hazelcast/service/ClusterServiceImpl.java

@@ -23,6 +23,7 @@ import org.idempiere.distributed.IClusterMember;
 import org.idempiere.distributed.IClusterService;
 
 import com.hazelcast.core.DistributedTask;
+import com.hazelcast.core.HazelcastInstance;
 import com.hazelcast.core.Member;
 import com.hazelcast.core.MultiTask;
 
@@ -37,10 +38,13 @@ public class ClusterServiceImpl implements IClusterService {
 	 */
 	@Override
 	public Collection<IClusterMember> getMembers() {
-		Set<Member> members = Activator.getHazelcastInstance().getCluster().getMembers();
+		HazelcastInstance instance = Activator.getHazelcastInstance();		
 		Set<IClusterMember> clusterMembers = new HashSet<IClusterMember>();
-		for(Member member : members) {
+		if (instance != null) {
-			clusterMembers.add(new ClusterMember(member.getUuid(), member.getInetSocketAddress().getAddress(), member.getInetSocketAddress().getPort()));
+			Set<Member> members = instance.getCluster().getMembers();
+			for(Member member : members) {
+				clusterMembers.add(new ClusterMember(member.getUuid(), member.getInetSocketAddress().getAddress(), member.getInetSocketAddress().getPort()));
+			}
 		}
 		return clusterMembers;
 	}
@@ -50,8 +54,13 @@ public class ClusterServiceImpl implements IClusterService {
 	 */
 	@Override
 	public IClusterMember getLocalMember() {
-		Member member = Activator.getHazelcastInstance().getCluster().getLocalMember();
+		HazelcastInstance instance = Activator.getHazelcastInstance();
-		return new ClusterMember(member.getUuid(), member.getInetSocketAddress().getAddress(), member.getInetSocketAddress().getPort());
+		if (instance != null) {
+			Member member = instance.getCluster().getLocalMember();
+			return new ClusterMember(member.getUuid(), member.getInetSocketAddress().getAddress(), member.getInetSocketAddress().getPort());
+		} else {
+			return null;
+		}
 	}
 
 	/* (non-Javadoc)
@@ -59,12 +68,15 @@ public class ClusterServiceImpl implements IClusterService {
 	 */
 	@Override
 	public <V> Future<V> execute(Callable<V> task, IClusterMember clusterMember) {
-		Set<Member> members = Activator.getHazelcastInstance().getCluster().getMembers();
+		HazelcastInstance instance = Activator.getHazelcastInstance();
-		for(Member member : members) {
+		if (instance != null) {
-			if (member.getUuid().equals(clusterMember.getId())) {
+			Set<Member> members = instance.getCluster().getMembers();
-				DistributedTask<V> distributedTask = new DistributedTask<V>(task, member);
+			for(Member member : members) {
-				Activator.getHazelcastInstance().getExecutorService().execute(distributedTask);
+				if (member.getUuid().equals(clusterMember.getId())) {
-				return distributedTask;
+					DistributedTask<V> distributedTask = new DistributedTask<V>(task, member);
+					Activator.getHazelcastInstance().getExecutorService().execute(distributedTask);
+					return distributedTask;
+				}
 			}
 		}
 		return null;
@@ -81,17 +93,20 @@ public class ClusterServiceImpl implements IClusterService {
 		for(IClusterMember clusterMember : clusterMembers) {
 			selectedIds.add(clusterMember.getId());
 		}
-		Set<Member> members = Activator.getHazelcastInstance().getCluster().getMembers();
+		HazelcastInstance instance = Activator.getHazelcastInstance();
-		Set<Member> selectedMembers = new HashSet<Member>();
+		if (instance != null) {
-		for(Member member : members) {
+			Set<Member> members = instance.getCluster().getMembers();
-			if (selectedIds.contains(member.getUuid())) {
+			Set<Member> selectedMembers = new HashSet<Member>();
-				selectedMembers.add(member);
+			for(Member member : members) {
+				if (selectedIds.contains(member.getUuid())) {
+					selectedMembers.add(member);
+				}
+			}
+			if (selectedMembers.size() > 0) {
+				MultiTask<V> multiTask = new MultiTask<V>(task, selectedMembers);
+				Activator.getHazelcastInstance().getExecutorService().execute(multiTask);
+				return multiTask;
 			}
-		}
-		if (selectedMembers.size() > 0) {
-			MultiTask<V> multiTask = new MultiTask<V>(task, selectedMembers);
-			Activator.getHazelcastInstance().getExecutorService().execute(multiTask);
-			return multiTask;
 		}
 		return null;
 	}

org.idempiere.hazelcast.service/src/org/idempiere/hazelcast/service/MessageServiceImpl.java

@@ -16,6 +16,8 @@ package org.idempiere.hazelcast.service;
 import org.idempiere.distributed.IMessageService;
 import org.idempiere.distributed.ITopic;
 
+import com.hazelcast.core.HazelcastInstance;
+
 /**
  * @author hengsin
  *
@@ -30,7 +32,12 @@ public class MessageServiceImpl implements IMessageService {
 
 	@Override
 	public <T> ITopic<T> getTopic(String name) {
-		com.hazelcast.core.ITopic<T> topic = Activator.getHazelcastInstance().getTopic(name);
+		HazelcastInstance instance = Activator.getHazelcastInstance();
-		return new TopicImpl<T>(topic);
+		if (instance != null) {
+			com.hazelcast.core.ITopic<T> topic = instance.getTopic(name);
+			return new TopicImpl<T>(topic);
+		} else {
+			return null;
+		}
 	}
 }