IDEMPIERE-728 Hazelcast Instance is not active exception on webui. Improve handling of shutdown/crash of hazelcast instance .

org.adempiere.base/src/org/compiere/util/CacheMgt.java

@@ -142,6 +142,7 @@ public class CacheMgt
 		if (service != null) {			
 			ResetCacheCallable callable = new ResetCacheCallable(tableName, recordId);
 			Future<Collection<Integer>> future = service.execute(callable, service.getMembers());
+			if (future != null) {
 				int total = 0;
 				try {
 					Collection<Integer> results = future.get();
@@ -158,6 +159,9 @@ public class CacheMgt
 			} else {
 				return resetLocalCache(tableName, recordId);
 			}
+		} else {
+			return resetLocalCache(tableName, recordId);
+		}
 	}
 	
 	/**
@@ -172,7 +176,9 @@ public class CacheMgt
 		IClusterService service = holder.getService();
 		if (service != null) {			
 			CacheNewRecordCallable callable = new CacheNewRecordCallable(tableName, recordId);
-			service.execute(callable, service.getMembers());
+			if (service.execute(callable, service.getMembers()) == null) {
+				localNewRecord(tableName, recordId);
+			}
 		} else {
 			localNewRecord(tableName, recordId);
 		}

org.idempiere.hazelcast.service/src/org/idempiere/hazelcast/service/Activator.java

@@ -17,10 +17,11 @@ import java.io.File;
 import java.io.FileNotFoundException;
 import java.net.MalformedURLException;
 import java.net.URL;
+import java.text.DateFormat;
+import java.util.Date;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
-import java.util.concurrent.atomic.AtomicReference;
 
 import org.compiere.Adempiere;
 import org.compiere.model.ServerStateChangeEvent;
@@ -46,7 +47,7 @@ public class Activator implements BundleActivator {
 	}
 
 	private volatile static HazelcastInstance hazelcastInstance;
-	private static AtomicReference<Future<?>> futureRef = new AtomicReference<Future<?>>();
+	private static Future<?> future;
 
 	/*
 	 * (non-Javadoc)
@@ -68,10 +69,10 @@ public class Activator implements BundleActivator {
 		}
 	}
 
-	private void createHazelCastInstance() {
+	private static synchronized void createHazelCastInstance() {
 		ScheduledThreadPoolExecutor executor = Adempiere.getThreadPoolExecutor();
 		
-		Future<?> future = executor.submit(new Runnable() {			
+		future = executor.submit(new Runnable() {			
 			@Override
 			public void run() {
 				String dataArea = System.getProperty("osgi.install.area");
@@ -92,11 +93,9 @@ public class Activator implements BundleActivator {
 				hazelcastInstance = Hazelcast.newHazelcastInstance(null);		
 			}
 		});
-		futureRef.set(future);
 	}
 
-	public static HazelcastInstance getHazelcastInstance() {
+	public static synchronized HazelcastInstance getHazelcastInstance() {
-		Future<?> future = futureRef.get();
 		if (future != null && !future.isDone()) {
 			try {
 				future.get();
@@ -105,6 +104,26 @@ public class Activator implements BundleActivator {
 			}		
 		}
 				
+		if (hazelcastInstance != null) {
+			if (!hazelcastInstance.getLifecycleService().isRunning()) {
+				System.err.println(DateFormat.getDateTimeInstance().format(new Date()) + " Hazelcast instance is down!");
+				//recreate
+				try {
+					hazelcastInstance = Hazelcast.newHazelcastInstance(null);
+					if (!hazelcastInstance.getLifecycleService().isRunning()) {
+						hazelcastInstance = null;
+						System.err.println(DateFormat.getDateTimeInstance().format(new Date()) + " Failed to re-create Hazelcast instance!");
+					} else {
+						System.err.println(DateFormat.getDateTimeInstance().format(new Date()) + " Hazelcast instance re-created!");
+					}
+				} catch (Throwable t) {
+					t.printStackTrace();
+					hazelcastInstance = null;
+					System.err.println(DateFormat.getDateTimeInstance().format(new Date()) + " Failed to re-create Hazelcast instance!");
+				}				
+			}						
+		}
+		
 		return hazelcastInstance;
 	}
 	
@@ -114,13 +133,13 @@ public class Activator implements BundleActivator {
 	 */
 	public void stop(BundleContext bundleContext) throws Exception {
 		Activator.context = null;
-		Future<?> future = futureRef.get();
+		synchronized (Activator.class) {
 			if (future != null && !future.isDone()) {
 				future.cancel(true); 
 			} else if (hazelcastInstance != null) {
 				hazelcastInstance.getLifecycleService().shutdown();
 				hazelcastInstance = null;
 			}
-		futureRef.set(null);
+		}
 	}
 }

org.idempiere.hazelcast.service/src/org/idempiere/hazelcast/service/ClusterServiceImpl.java

@@ -23,6 +23,7 @@ import org.idempiere.distributed.IClusterMember;
 import org.idempiere.distributed.IClusterService;
 
 import com.hazelcast.core.DistributedTask;
+import com.hazelcast.core.HazelcastInstance;
 import com.hazelcast.core.Member;
 import com.hazelcast.core.MultiTask;
 
@@ -37,11 +38,14 @@ public class ClusterServiceImpl implements IClusterService {
 	 */
 	@Override
 	public Collection<IClusterMember> getMembers() {
-		Set<Member> members = Activator.getHazelcastInstance().getCluster().getMembers();
+		HazelcastInstance instance = Activator.getHazelcastInstance();		
 		Set<IClusterMember> clusterMembers = new HashSet<IClusterMember>();
+		if (instance != null) {
+			Set<Member> members = instance.getCluster().getMembers();
 			for(Member member : members) {
 				clusterMembers.add(new ClusterMember(member.getUuid(), member.getInetSocketAddress().getAddress(), member.getInetSocketAddress().getPort()));
 			}
+		}
 		return clusterMembers;
 	}
 
@@ -50,8 +54,13 @@ public class ClusterServiceImpl implements IClusterService {
 	 */
 	@Override
 	public IClusterMember getLocalMember() {
-		Member member = Activator.getHazelcastInstance().getCluster().getLocalMember();
+		HazelcastInstance instance = Activator.getHazelcastInstance();
+		if (instance != null) {
+			Member member = instance.getCluster().getLocalMember();
 			return new ClusterMember(member.getUuid(), member.getInetSocketAddress().getAddress(), member.getInetSocketAddress().getPort());
+		} else {
+			return null;
+		}
 	}
 
 	/* (non-Javadoc)
@@ -59,7 +68,9 @@ public class ClusterServiceImpl implements IClusterService {
 	 */
 	@Override
 	public <V> Future<V> execute(Callable<V> task, IClusterMember clusterMember) {
-		Set<Member> members = Activator.getHazelcastInstance().getCluster().getMembers();
+		HazelcastInstance instance = Activator.getHazelcastInstance();
+		if (instance != null) {
+			Set<Member> members = instance.getCluster().getMembers();
 			for(Member member : members) {
 				if (member.getUuid().equals(clusterMember.getId())) {
 					DistributedTask<V> distributedTask = new DistributedTask<V>(task, member);
@@ -67,6 +78,7 @@ public class ClusterServiceImpl implements IClusterService {
 					return distributedTask;
 				}
 			}
+		}
 		return null;
 	}
 
@@ -81,7 +93,9 @@ public class ClusterServiceImpl implements IClusterService {
 		for(IClusterMember clusterMember : clusterMembers) {
 			selectedIds.add(clusterMember.getId());
 		}
-		Set<Member> members = Activator.getHazelcastInstance().getCluster().getMembers();
+		HazelcastInstance instance = Activator.getHazelcastInstance();
+		if (instance != null) {
+			Set<Member> members = instance.getCluster().getMembers();
 			Set<Member> selectedMembers = new HashSet<Member>();
 			for(Member member : members) {
 				if (selectedIds.contains(member.getUuid())) {
@@ -93,6 +107,7 @@ public class ClusterServiceImpl implements IClusterService {
 				Activator.getHazelcastInstance().getExecutorService().execute(multiTask);
 				return multiTask;
 			}
+		}
 		return null;
 	}
 

org.idempiere.hazelcast.service/src/org/idempiere/hazelcast/service/MessageServiceImpl.java

@@ -16,6 +16,8 @@ package org.idempiere.hazelcast.service;
 import org.idempiere.distributed.IMessageService;
 import org.idempiere.distributed.ITopic;
 
+import com.hazelcast.core.HazelcastInstance;
+
 /**
  * @author hengsin
  *
@@ -30,7 +32,12 @@ public class MessageServiceImpl implements IMessageService {
 
 	@Override
 	public <T> ITopic<T> getTopic(String name) {
-		com.hazelcast.core.ITopic<T> topic = Activator.getHazelcastInstance().getTopic(name);
+		HazelcastInstance instance = Activator.getHazelcastInstance();
+		if (instance != null) {
+			com.hazelcast.core.ITopic<T> topic = instance.getTopic(name);
 			return new TopicImpl<T>(topic);
+		} else {
+			return null;
+		}
 	}
 }