Implemented connection pool properties file support. Added convert statement cache for PostgreSQL. Enhance connection pool ability to survive under heavy load.

org.adempiere.base-feature/feature.xml

@@ -287,4 +287,10 @@
          version="0.0.0"
          unpack="false"/>
 
+   <plugin
+         id="org.adempiere.install"
+         download-size="0"
+         install-size="0"
+         version="0.0.0"/>
+
 </feature>

org.adempiere.ui.swing-feature/swingclient.product.launch

@@ -21,8 +21,8 @@
 <stringAttribute key="pde.version" value="3.3"/>
 <stringAttribute key="product" value="org.adempiere.ui.swing.client_product"/>
 <stringAttribute key="productFile" value="/org.adempiere.ui.swing-feature/swingclient.product"/>
-<stringAttribute key="selected_target_plugins" value="org.eclipse.osgi.services@default:default,org.eclipse.equinox.util@default:default,org.eclipse.equinox.p2.ql@default:default,com.springsource.javax.mail@default:default,com.springsource.org.apache.activemq@default:default,org.eclipse.ecf.provider.filetransfer.ssl@default:false,org.apache.commons.httpclient@default:default,com.springsource.org.apache.xmlcommons@default:default,org.eclipse.ecf.provider.filetransfer.httpclient@default:default,org.eclipse.equinox.p2.touchpoint.eclipse@default:default,org.eclipse.equinox.common@2:true,org.eclipse.equinox.http.registry@default:default,com.springsource.org.apache.xerces@default:false,org.eclipse.equinox.p2.metadata.repository@default:default,com.springsource.org.apache.xml.resolver@default:default,org.eclipse.core.jobs@default:default,javax.xml@default:default,org.eclipse.osgi@-1:true,com.springsource.org.apache.poi@default:default,com.springsource.org.junit@default:default,org.eclipse.ecf.provider.filetransfer@default:default,org.sat4j.core@default:default,org.eclipse.equinox.simpleconfigurator@1:true,org.eclipse.equinox.preferences@default:default,org.eclipse.equinox.launcher.gtk.linux.x86@default:false,org.eclipse.equinox.frameworkadmin.equinox@default:default,org.eclipse.equinox.p2.artifact.repository@default:default,org.eclipse.core.contenttype@default:default,org.eclipse.equinox.p2.repository@default:default,org.eclipse.equinox.p2.garbagecollector@default:default,org.restlet.ext.net@default:true,org.eclipse.core.runtime@default:true,com.springsource.javax.jms@default:default,com.springsource.javax.xml.rpc@default:default,com.springsource.javax.el@default:default,org.eclipse.equinox.p2.jarprocessor@default:default,com.springsource.net.sf.cglib@default:default,org.eclipse.ecf.filetransfer@default:default,org.eclipse.equinox.frameworkadmin@default:default,org.eclipse.equinox.p2.engine@default:default,org.apache.commons.codec@default:default,com.springsource.javax.ejb@default:default,org.restlet@default:true,org.eclipse.equinox.ds@1:true,org.eclipse.equinox.security@default:default,org.eclipse.equinox.p2.director@default:default,com.springsource.org.apache.xml.serializer@default:default,org.eclipse.ecf.provider.filetransfer.httpclient.ssl@default:false,org.eclipse.equinox.simpleconfigurator.manipulator@default:default,org.apache.ant@default:default,com.springsource.org.apache.commons.collections@default:default,org.apache.commons.logging@default:default,org.eclipse.equinox.launcher@default:default,com.springsource.org.apache.commons.logging@default:default,org.sat4j.pb@default:default,org.eclipse.ecf.ssl@default:false,org.eclipse.ecf.identity@default:default,org.eclipse.ecf@default:default,com.springsource.org.apache.commons.net@default:default,org.eclipse.equinox.registry@default:default,org.eclipse.equinox.p2.console@default:default,com.springsource.javax.servlet@default:default,com.springsource.javax.activation@default:default,org.eclipse.equinox.app@default:default,com.springsource.org.apache.kahadb@default:default,com.springsource.javax.xml.soap@default:default,org.eclipse.equinox.p2.core@default:default,org.eclipse.equinox.p2.metadata@default:default,org.eclipse.equinox.p2.touchpoint.natives@default:default,com.springsource.javax.management.j2ee@default:default"/>
+<stringAttribute key="selected_target_plugins" value="org.restlet@default:true,org.eclipse.core.contenttype@default:default,org.eclipse.equinox.frameworkadmin@default:default,org.apache.commons.codec@default:default,com.springsource.javax.servlet@default:default,org.eclipse.ecf.identity@default:default,org.eclipse.equinox.p2.touchpoint.natives@default:default,org.apache.commons.httpclient@default:default,com.springsource.javax.mail@default:default,org.eclipse.ecf.provider.filetransfer@default:default,com.springsource.org.junit@default:default,org.sat4j.core@default:default,com.springsource.org.apache.poi@default:default,com.springsource.org.apache.xerces@default:false,com.springsource.org.apache.xmlcommons@default:default,org.eclipse.osgi.services@default:default,com.springsource.javax.activation@default:default,org.eclipse.ecf.provider.filetransfer.ssl@default:false,org.eclipse.ecf.provider.filetransfer.httpclient@default:default,org.eclipse.ecf@default:default,com.springsource.javax.xml.rpc@default:default,org.eclipse.equinox.p2.director@default:default,org.eclipse.equinox.registry@default:default,com.springsource.org.apache.activemq@default:default,org.eclipse.equinox.p2.metadata@default:default,org.eclipse.core.runtime@default:true,com.springsource.org.apache.commons.logging@default:default,org.eclipse.equinox.simpleconfigurator.manipulator@default:default,org.eclipse.core.jobs@default:default,com.springsource.org.apache.kahadb@default:default,com.springsource.javax.ejb@default:default,com.springsource.org.apache.xml.resolver@default:default,org.eclipse.equinox.p2.engine@default:default,org.eclipse.equinox.launcher@default:default,org.eclipse.ecf.ssl@default:false,org.restlet.ext.net@default:true,org.eclipse.equinox.p2.garbagecollector@default:default,org.eclipse.equinox.p2.metadata.repository@default:default,com.springsource.org.apache.xml.serializer@default:default,org.eclipse.osgi@-1:true,org.eclipse.ecf.filetransfer@default:default,org.eclipse.equinox.preferences@default:default,com.springsource.net.sf.cglib@default:default,org.eclipse.equinox.p2.ql@default:default,org.eclipse.equinox.p2.jarprocessor@default:default,org.eclipse.equinox.security@default:default,com.springsource.javax.el@default:default,org.apache.commons.logging@default:default,com.springsource.org.apache.commons.net@default:default,org.eclipse.equinox.p2.artifact.repository@default:default,javax.xml@default:default,org.eclipse.equinox.app@default:default,com.springsource.javax.management.j2ee@default:default,org.eclipse.equinox.p2.console@default:default,com.springsource.org.apache.commons.collections@default:default,org.sat4j.pb@default:default,com.springsource.javax.jms@default:default,org.eclipse.equinox.http.registry@default:default,org.eclipse.ecf.provider.filetransfer.httpclient.ssl@default:false,org.eclipse.equinox.common@2:true,com.springsource.javax.xml.soap@default:default,org.apache.ant@default:default,org.eclipse.equinox.util@default:default,org.eclipse.equinox.p2.touchpoint.eclipse@default:default,org.eclipse.equinox.launcher.gtk.linux.x86@default:false,org.eclipse.equinox.frameworkadmin.equinox@default:default,org.eclipse.equinox.p2.repository@default:default,org.eclipse.equinox.simpleconfigurator@1:true,org.eclipse.equinox.ds@1:true,org.eclipse.equinox.p2.core@default:default"/>
-<stringAttribute key="selected_workspace_plugins" value="org.adempiere.base@default:default,org.adempiere.extend@default:false,org.adempiere.ui@default:default,org.adempiere.report.jasper.swing@default:default,org.adempiere.plugin.utils@default:default,org.adempiere.report.jasper@default:default,org.adempiere.report.jasper.library@default:default,org.adempiere.pipo@default:default,org.apache.ecs@default:default,org.adempiere.replication@default:default,org.adempiere.payment.processor@default:default,org.adempiere.base.callout@default:default,org.adempiere.pipo.handlers@default:default,org.compiere.db.oracle.provider@default:default,org.adempiere.ui.swing@default:default,org.compiere.db.postgresql.provider@default:default,org.adempiere.base.process@default:default"/>
+<stringAttribute key="selected_workspace_plugins" value="org.adempiere.report.jasper.library@default:default,org.adempiere.replication@default:default,org.apache.ecs@default:default,org.adempiere.plugin.utils@default:default,org.adempiere.ui@default:default,org.adempiere.pipo.handlers@default:default,org.compiere.db.postgresql.provider@default:default,org.adempiere.report.jasper.swing@default:default,org.compiere.db.oracle.provider@default:default,org.adempiere.ui.swing@default:default,org.adempiere.extend@default:false,org.adempiere.install@default:default,org.adempiere.base.process@default:default,org.adempiere.payment.processor@default:default,org.adempiere.pipo@default:default,org.adempiere.report.jasper@default:default,org.adempiere.base@default:default,org.adempiere.base.callout@default:default"/>
 <booleanAttribute key="show_selected_only" value="false"/>
 <booleanAttribute key="tracing" value="false"/>
 <booleanAttribute key="useCustomFeatures" value="false"/>

org.compiere.db.oracle.provider/META-INF/MANIFEST.MF

@@ -10,4 +10,7 @@ Bundle-ClassPath: .,
  ojdbc6.jar,
  c3p0-0.9.1.2.jar,
  c3p0-oracle-thin-extras-0.9.1.2.jar
+Import-Package: org.osgi.framework
+Bundle-ActivationPolicy: lazy
+Bundle-Activator: org.adempiere.db.oracle.OracleBundleActivator
 

org.compiere.db.oracle.provider/META-INF/pool/client.properties

@@ -0,0 +1,16 @@
+#timeout
+IdleConnectionTestPeriod=1200
+AcquireRetryAttempts=2
+MaxIdleTimeExcessConnections=1200
+MaxIdleTime=1200
+#UnreturnedConnectionTimeout=1800
+
+#size
+MaxPoolSize=15
+InitialPoolSize=1
+MinPoolSize=1
+
+#flag
+TestConnectionOnCheckin=false
+TestConnectionOnCheckout=false
+#CheckoutTimeout=60;

org.compiere.db.oracle.provider/META-INF/pool/server.properties

@@ -0,0 +1,16 @@
+#timeout
+IdleConnectionTestPeriod=1200
+AcquireRetryAttempts=2
+MaxIdleTimeExcessConnections=1200
+MaxIdleTime=1200
+#UnreturnedConnectionTimeout=1800
+
+#size
+MaxPoolSize=150
+InitialPoolSize=10
+MinPoolSize=5
+
+#flag
+TestConnectionOnCheckin=false
+TestConnectionOnCheckout=false
+#CheckoutTimeout=60;

org.compiere.db.oracle.provider/src/org/adempiere/db/oracle/OracleBundleActivator.java

@@ -0,0 +1,38 @@
+/******************************************************************************
+ * Product: Adempiere ERP & CRM Smart Business Solution                       *
+ * Copyright (C) 2010 Heng Sin Low                							  *
+ * This program is free software; you can redistribute it and/or modify it    *
+ * under the terms version 2 of the GNU General Public License as published   *
+ * by the Free Software Foundation. This program is distributed in the hope   *
+ * that it will be useful, but WITHOUT ANY WARRANTY; without even the implied *
+ * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.           *
+ * See the GNU General Public License for more details.                       *
+ * You should have received a copy of the GNU General Public License along    *
+ * with this program; if not, write to the Free Software Foundation, Inc.,    *
+ * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA.                     *
+ *****************************************************************************/
+package org.adempiere.db.oracle;
+
+import org.osgi.framework.BundleActivator;
+import org.osgi.framework.BundleContext;
+
+/**
+ *
+ * @author hengsin
+ *
+ */
+public class OracleBundleActivator implements BundleActivator {
+
+	public static BundleContext bundleContext = null;
+
+	@Override
+	public void start(BundleContext context) throws Exception {
+		bundleContext = context;
+	}
+
+	@Override
+	public void stop(BundleContext context) throws Exception {
+		bundleContext = null;
+	}
+
+}

org.compiere.db.oracle.provider/src/org/compiere/db/DB_Oracle.java

@@ -16,7 +16,9 @@
  *****************************************************************************/
 package org.compiere.db;
 
+import java.io.IOException;
 import java.math.BigDecimal;
+import java.net.URL;
 import java.sql.Connection;
 import java.sql.DatabaseMetaData;
 import java.sql.Driver;
@@ -25,12 +27,15 @@ import java.sql.PreparedStatement;
 import java.sql.SQLException;
 import java.sql.Statement;
 import java.sql.Timestamp;
+import java.util.Properties;
+import java.util.Random;
 import java.util.logging.Level;
 
 import javax.sql.DataSource;
 
 import oracle.jdbc.OracleDriver;
 
+import org.adempiere.db.oracle.OracleBundleActivator;
 import org.adempiere.exceptions.DBException;
 import org.compiere.Adempiere;
 import org.compiere.dbPort.Convert;
@@ -40,6 +45,7 @@ import org.compiere.util.DB;
 import org.compiere.util.DisplayType;
 import org.compiere.util.Ini;
 import org.compiere.util.Language;
+import org.compiere.util.Trx;
 import org.compiere.util.Util;
 
 import com.mchange.v2.c3p0.ComboPooledDataSource;
@@ -57,6 +63,7 @@ import com.mchange.v2.c3p0.ComboPooledDataSource;
  */
 public class DB_Oracle implements AdempiereDatabase
 {
+
     /**
      *  Oracle Database
      */
@@ -97,8 +104,6 @@ public class DB_Oracle implements AdempiereDatabase
     /** Connection String           */
     private String                  m_connectionURL;
 
-    /** Statement Cache (50)        */
-    private static final String     MAX_STATEMENTS = "200";
     /** Data Source                 */
     private ComboPooledDataSource   m_ds = null;
 
@@ -113,6 +118,7 @@ public class DB_Oracle implements AdempiereDatabase
 
     private static int              m_maxbusyconnections = 0;
 
+    private Random rand = new Random();
 
     /**
      *  Get Database Name
@@ -333,6 +339,7 @@ public class DB_Oracle implements AdempiereDatabase
             sb.append(" , # Busy Connections: ").append(m_ds.getNumBusyConnections());
             sb.append(" , # Idle Connections: ").append(m_ds.getNumIdleConnections());
             sb.append(" , # Orphaned Connections: ").append(m_ds.getNumUnclosedOrphanedConnections());
+            sb.append(" , # Active Transactions: ").append(Trx.getActiveTransactions().length);
         }
         catch (Exception e)
         {}
@@ -546,6 +553,24 @@ public class DB_Oracle implements AdempiereDatabase
         if (m_ds != null)
             return m_ds;
 
+        URL url = Ini.isClient()
+        	? OracleBundleActivator.bundleContext.getBundle().getEntry("META-INF/pool/client.properties")
+        	: OracleBundleActivator.bundleContext.getBundle().getEntry("META-INF/pool/server.properties");
+        Properties poolProperties = new Properties();
+        try {
+			poolProperties.load(url.openStream());
+		} catch (IOException e) {
+			throw new DBException(e);
+		}
+
+		int idleConnectionTestPeriod = getIntProperty(poolProperties, "IdleConnectionTestPeriod", 1200);
+		int acquireRetryAttempts = getIntProperty(poolProperties, "AcquireRetryAttempts", 2);
+		int maxIdleTimeExcessConnections = getIntProperty(poolProperties, "MaxIdleTimeExcessConnections", 1200);
+		int maxIdleTime = getIntProperty(poolProperties, "MaxIdleTime", 1200);
+		int unreturnedConnectionTimeout = getIntProperty(poolProperties, "UnreturnedConnectionTimeout", 0);
+		boolean testConnectionOnCheckin = getBooleanProperty(poolProperties, "TestConnectionOnCheckin", false);
+		boolean testConnectionOnCheckout = getBooleanProperty(poolProperties, "TestConnectionOnCheckout", false);
+		int checkoutTimeout = getIntProperty(poolProperties, "CheckoutTimeout", 0);
         try
         {
             System.setProperty("com.mchange.v2.log.MLog", "com.mchange.v2.log.FallbackMLog");
@@ -558,34 +583,42 @@ public class DB_Oracle implements AdempiereDatabase
             cpds.setUser(connection.getDbUid());
             cpds.setPassword(connection.getDbPwd());
             cpds.setPreferredTestQuery(DEFAULT_CONN_TEST_SQL);
-            cpds.setIdleConnectionTestPeriod(1200);
+            cpds.setIdleConnectionTestPeriod(idleConnectionTestPeriod);
-            cpds.setAcquireRetryAttempts(2);
+            cpds.setAcquireRetryAttempts(acquireRetryAttempts);
-            //cpds.setTestConnectionOnCheckin(true);
+            cpds.setTestConnectionOnCheckin(testConnectionOnCheckin);
-            //cpds.setTestConnectionOnCheckout(true);
+            cpds.setTestConnectionOnCheckout(testConnectionOnCheckout);
-            //cpds.setCheckoutTimeout(60);
+            if (checkoutTimeout > 0)
+            	cpds.setCheckoutTimeout(checkoutTimeout);
 
+            cpds.setMaxIdleTimeExcessConnections(maxIdleTimeExcessConnections);
+            cpds.setMaxIdleTime(maxIdleTime);
             if (Ini.isClient())
             {
-                cpds.setInitialPoolSize(1);
+            	int maxPoolSize = getIntProperty(poolProperties, "MaxPoolSize", 15);
-                cpds.setMinPoolSize(1);
+            	int initialPoolSize = getIntProperty(poolProperties, "InitialPoolSize", 1);
-                cpds.setMaxPoolSize(15);
+            	int minPoolSize = getIntProperty(poolProperties, "MinPoolSize", 1);
-                cpds.setMaxIdleTimeExcessConnections(1200);
+                cpds.setInitialPoolSize(initialPoolSize);
-                cpds.setMaxIdleTime(900);
+                cpds.setMinPoolSize(minPoolSize);
-                m_maxbusyconnections = 10;
+                cpds.setMaxPoolSize(maxPoolSize);
+                m_maxbusyconnections = (int) (maxPoolSize * 0.9);
             }
             else
             {
-                cpds.setInitialPoolSize(10);
+            	int maxPoolSize = getIntProperty(poolProperties, "MaxPoolSize", 400);
-                cpds.setMinPoolSize(5);
+            	int initialPoolSize = getIntProperty(poolProperties, "InitialPoolSize", 10);
-                cpds.setMaxPoolSize(150);
+            	int minPoolSize = getIntProperty(poolProperties, "MinPoolSize", 5);
-                cpds.setMaxIdleTimeExcessConnections(1200);
+                cpds.setInitialPoolSize(initialPoolSize);
-                cpds.setMaxIdleTime(1200);
+                cpds.setMinPoolSize(minPoolSize);
-                m_maxbusyconnections = 120;
+                cpds.setMaxPoolSize(maxPoolSize);
+                m_maxbusyconnections = (int) (maxPoolSize * 0.9);
             }
 
-            //the following sometimes kill active connection!
+            if (unreturnedConnectionTimeout > 0)
-            //cpds.setUnreturnedConnectionTimeout(1200);
+            {
-            //cpds.setDebugUnreturnedConnectionStackTraces(true);
+	            //the following sometimes kill active connection!
+	            cpds.setUnreturnedConnectionTimeout(1200);
+	            cpds.setDebugUnreturnedConnectionStackTraces(true);
+            }
 
             m_ds = cpds;
         }
@@ -600,7 +633,6 @@ public class DB_Oracle implements AdempiereDatabase
         return m_ds;
     }   //  getDataSource
 
-
     /**
      *  Get Cached Connection
      *  @param connection info
@@ -623,14 +655,27 @@ public class DB_Oracle implements AdempiereDatabase
             //
             try
             {
-                conn = (Connection)m_ds.getConnection();
+            	int numConnections = m_ds.getNumBusyConnections();
+        		if(numConnections >= m_maxbusyconnections && m_maxbusyconnections > 0)
+        		{
+        			//system is under heavy load, wait between 20 to 40 seconds
+        			int randomNum = rand.nextInt(40 - 20 + 1) + 20;
+        			Thread.sleep(randomNum * 1000);
+        		}
+        		conn = m_ds.getConnection();
+        		if (conn == null) {
+        			//try again after 10 to 30 seconds
+        			int randomNum = rand.nextInt(30 - 10 + 1) + 10;
+        			Thread.sleep(randomNum * 1000);
+        			conn = m_ds.getConnection();
+        		}
+
                 if (conn != null)
                 {
                     if (conn.getTransactionIsolation() != transactionIsolation)
                         conn.setTransactionIsolation(transactionIsolation);
                     if (conn.getAutoCommit() != autoCommit)
                         conn.setAutoCommit(autoCommit);
-//                      conn.setDefaultRowPrefetch(20);     //  10 default - reduces round trips
                 }
             }
             catch (Exception e)
@@ -668,18 +713,29 @@ public class DB_Oracle implements AdempiereDatabase
         try
         {
         	if (conn != null) {
+        		boolean trace = "true".equalsIgnoreCase(System.getProperty("org.adempiere.db.traceStatus"));
         		int numConnections = m_ds.getNumBusyConnections();
-	            if(numConnections >= m_maxbusyconnections && m_maxbusyconnections > 0)
+        		if (numConnections > 1)
-	            {
+        		{
-	                log.warning(getStatus());
+	    			if (trace)
-	                //hengsin: make a best effort to reclaim leak connection
+	    			{
-	                Runtime.getRuntime().runFinalization();
+	    				log.warning(getStatus());
-	            }
+	    			}
+	    			if(numConnections >= m_maxbusyconnections && m_maxbusyconnections > 0)
+		            {
+	    				if (!trace)
+	    					log.warning(getStatus());
+		                //hengsin: make a best effort to reclaim leak connection
+		                Runtime.getRuntime().runFinalization();
+		            }
+        		}
+        	} else {
+        		//don't use log.severe here as it will try to access db again
+        		System.err.println("Failed to acquire new connection. Status=" + getStatus());
         	}
         }
         catch (Exception ex)
         {
-
         }
         if (exception != null)
             throw exception;
@@ -1118,4 +1174,29 @@ public class DB_Oracle implements AdempiereDatabase
 		return false;
 	}
 
+	private int getIntProperty(Properties properties, String key, int defaultValue)
+	{
+		int i = defaultValue;
+		try
+		{
+			String s = properties.getProperty(key);
+			if (s != null && s.trim().length() > 0)
+				i = Integer.parseInt(s);
+		}
+		catch (Exception e) {}
+		return i;
+	}
+
+	private boolean getBooleanProperty(Properties properties, String key, boolean defaultValue)
+	{
+		boolean b = defaultValue;
+		try
+		{
+			String s = properties.getProperty(key);
+			if (s != null && s.trim().length() > 0)
+				b = Boolean.valueOf(s);
+		}
+		catch (Exception e) {}
+		return b;
+	}
 }   //  DB_Oracle

org.compiere.db.postgresql.provider/META-INF/MANIFEST.MF

@@ -10,4 +10,7 @@ Bundle-ClassPath: .,
 Require-Bundle: org.adempiere.base;bundle-version="1.0.0",
  org.adempiere.install;bundle-version="1.0.0"
 Import-Package: junit.framework;version="3.8.2",
- org.junit;version="4.8.1"
+ org.junit;version="4.8.1",
+ org.osgi.framework
+Bundle-ActivationPolicy: lazy
+Bundle-Activator: org.adempiere.db.postgresql.PostgreSQLBundleActivator

org.compiere.db.postgresql.provider/META-INF/pool/client.properties

@@ -0,0 +1,16 @@
+#timeout
+IdleConnectionTestPeriod=1200
+AcquireRetryAttempts=2
+MaxIdleTimeExcessConnections=1200
+MaxIdleTime=1200
+#UnreturnedConnectionTimeout=1800
+
+#size
+MaxPoolSize=15
+InitialPoolSize=1
+MinPoolSize=1
+
+#flag
+TestConnectionOnCheckin=false
+TestConnectionOnCheckout=false
+#CheckoutTimeout=60;

org.compiere.db.postgresql.provider/META-INF/pool/server.properties

@@ -0,0 +1,16 @@
+#timeout
+IdleConnectionTestPeriod=1200
+AcquireRetryAttempts=2
+MaxIdleTimeExcessConnections=1200
+MaxIdleTime=1200
+#UnreturnedConnectionTimeout=1800
+
+#size
+MaxPoolSize=150
+InitialPoolSize=10
+MinPoolSize=5
+
+#flag
+TestConnectionOnCheckin=false
+TestConnectionOnCheckout=false
+#CheckoutTimeout=60;

org.compiere.db.postgresql.provider/src/org/adempiere/db/postgresql/PostgreSQLBundleActivator.java

@@ -0,0 +1,38 @@
+/******************************************************************************
+ * Product: Adempiere ERP & CRM Smart Business Solution                       *
+ * Copyright (C) 2010 Heng Sin Low                							  *
+ * This program is free software; you can redistribute it and/or modify it    *
+ * under the terms version 2 of the GNU General Public License as published   *
+ * by the Free Software Foundation. This program is distributed in the hope   *
+ * that it will be useful, but WITHOUT ANY WARRANTY; without even the implied *
+ * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.           *
+ * See the GNU General Public License for more details.                       *
+ * You should have received a copy of the GNU General Public License along    *
+ * with this program; if not, write to the Free Software Foundation, Inc.,    *
+ * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA.                     *
+ *****************************************************************************/
+package org.adempiere.db.postgresql;
+
+import org.osgi.framework.BundleActivator;
+import org.osgi.framework.BundleContext;
+
+/**
+ *
+ * @author hengsin
+ *
+ */
+public class PostgreSQLBundleActivator implements BundleActivator {
+
+	public static BundleContext bundleContext = null;
+
+	@Override
+	public void start(BundleContext context) throws Exception {
+		bundleContext = context;
+	}
+
+	@Override
+	public void stop(BundleContext context) throws Exception {
+		bundleContext = null;
+	}
+
+}

org.compiere.db.postgresql.provider/src/org/compiere/db/DB_PostgreSQL.java

@@ -18,25 +18,33 @@
  *****************************************************************************/
 package org.compiere.db;
 
+import java.io.IOException;
 import java.math.BigDecimal;
+import java.net.URL;
 import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
 import java.sql.Timestamp;
+import java.util.Properties;
+import java.util.Random;
 import java.util.logging.Level;
 
 import javax.sql.ConnectionPoolDataSource;
 import javax.sql.DataSource;
 import javax.sql.RowSet;
 
+import org.adempiere.db.postgresql.PostgreSQLBundleActivator;
+import org.adempiere.exceptions.DBException;
 import org.compiere.dbPort.Convert;
 import org.compiere.dbPort.Convert_PostgreSQL;
+import org.compiere.util.CCache;
 import org.compiere.util.CLogger;
 import org.compiere.util.DB;
 import org.compiere.util.DisplayType;
 import org.compiere.util.Ini;
+import org.compiere.util.Trx;
 
 import com.mchange.v2.c3p0.ComboPooledDataSource;
 
@@ -84,7 +92,8 @@ public class DB_PostgreSQL implements AdempiereDatabase
 	/** Cached Database Name	*/
 	private String			m_dbName = null;
 
-    private String				m_userName = null;
+    @SuppressWarnings("unused")
+	private String				m_userName = null;
 
     /** Connection String       	*/
 	private String          		m_connectionURL;
@@ -96,6 +105,10 @@ public class DB_PostgreSQL implements AdempiereDatabase
 
     public static final String NATIVE_MARKER = "NATIVE_"+Database.DB_POSTGRESQL+"_KEYWORK";
 
+    private CCache<String, String> convertCache = new CCache<String, String>("SQLConvertCache", 100, 0);
+
+    private Random rand = new Random();
+
 	/**
 	 *  Get Database Name
 	 *  @return database short name
@@ -271,6 +284,7 @@ public class DB_PostgreSQL implements AdempiereDatabase
             sb.append(" , # Busy Connections: ").append(m_ds.getNumBusyConnections());
             sb.append(" , # Idle Connections: ").append(m_ds.getNumIdleConnections());
             sb.append(" , # Orphaned Connections: ").append(m_ds.getNumUnclosedOrphanedConnections());
+            sb.append(" , # Active Transactions: ").append(Trx.getActiveTransactions().length);
         }
         catch (Exception e)
         {}
@@ -286,23 +300,19 @@ public class DB_PostgreSQL implements AdempiereDatabase
 	 */
 	public String convertStatement (String oraStatement)
 	{
+		String cache = convertCache.get(oraStatement);
+		if (cache != null) {
+			Convert.logMigrationScript(oraStatement, cache);
+			return cache;
+		}
+
 		String retValue[] = m_convert.convert(oraStatement);
 
         //begin vpj-cd e-evolution 03/14/2005
-		if (retValue.length == 0 )
+		if (retValue == null || retValue.length == 0 )
 			return  oraStatement;
         //end vpj-cd e-evolution 03/14/2005
 
-		if (retValue == null)
-        //begin vpj-cd 24/06/2005 e-evolution	
-		{	
-			log.log(Level.SEVERE,("DB_PostgreSQL.convertStatement - Not Converted (" + oraStatement + ") - "
-					+ m_convert.getConversionError()));
-			throw new IllegalArgumentException
-				("DB_PostgreSQL.convertStatement - Not Converted (" + oraStatement + ") - "
-					+ m_convert.getConversionError());
-		}
-		//		end vpj-cd 24/06/2005 e-evolution
 		if (retValue.length != 1)
 			//begin vpj-cd 24/06/2005 e-evolution
 			{
@@ -313,6 +323,9 @@ public class DB_PostgreSQL implements AdempiereDatabase
 					+ " (" + oraStatement + ") - " + m_convert.getConversionError());
 			}
 			//end vpj-cd 24/06/2005 e-evolution
+
+		convertCache.put(oraStatement, retValue[0]);
+
 		//  Diagnostics (show changed, but not if AD_Error
 		if (log.isLoggable(Level.FINE))
 		{
@@ -507,29 +520,89 @@ public class DB_PostgreSQL implements AdempiereDatabase
 		boolean autoCommit, int transactionIsolation)
 		throws Exception
 	{
-		if (m_ds == null)
+		Connection conn = null;
-			getDataSource(connection);
+        Exception exception = null;
-		//
+        try
-		Connection conn = m_ds.getConnection();
+        {
-		if (conn != null) {
+            if (m_ds == null)
-			//
+                getDataSource(connection);
-			conn.setAutoCommit(autoCommit);
-			conn.setTransactionIsolation(transactionIsolation);
 
-			try
+            //
-	        {
+            try
-                int numConnections = m_ds.getNumBusyConnections();
+            {
-	            if(numConnections >= m_maxbusyconnections && m_maxbusyconnections > 0)
+            	int numConnections = m_ds.getNumBusyConnections();
-	            {
+        		if(numConnections >= m_maxbusyconnections && m_maxbusyconnections > 0)
-	                log.warning(getStatus());
+        		{
-	                //hengsin: make a best effort to reclaim leak connection
+        			//system is under heavy load, wait between 20 to 40 seconds
-	                Runtime.getRuntime().runFinalization();
+        			int randomNum = rand.nextInt(40 - 20 + 1) + 20;
-	            }
+        			Thread.sleep(randomNum * 1000);
-	        }
+        		}
-	        catch (Exception ex)
+        		conn = m_ds.getConnection();
-	        {}
+        		if (conn == null) {
-		}
+        			//try again after 10 to 30 seconds
-		return conn;
+        			int randomNum = rand.nextInt(30 - 10 + 1) + 10;
+        			Thread.sleep(randomNum * 1000);
+        			conn = m_ds.getConnection();
+        		}
+
+                if (conn != null)
+                {
+                    if (conn.getTransactionIsolation() != transactionIsolation)
+                        conn.setTransactionIsolation(transactionIsolation);
+                    if (conn.getAutoCommit() != autoCommit)
+                        conn.setAutoCommit(autoCommit);
+                }
+            }
+            catch (Exception e)
+            {
+                exception = e;
+                conn = null;
+            }
+
+            if (conn == null && exception != null)
+            {
+            	//log might cause infinite loop since it will try to acquire database connection again
+            	/*
+                log.log(Level.SEVERE, exception.toString());
+                log.fine(toString()); */
+            	System.err.println(exception.toString());
+            }
+        }
+        catch (Exception e)
+        {
+            exception = e;
+        }
+
+        try
+        {
+        	if (conn != null) {
+        		boolean trace = "true".equalsIgnoreCase(System.getProperty("org.adempiere.db.traceStatus"));
+        		int numConnections = m_ds.getNumBusyConnections();
+        		if (numConnections > 1)
+        		{
+	    			if (trace)
+	    			{
+	    				log.warning(getStatus());
+	    			}
+	    			if(numConnections >= m_maxbusyconnections && m_maxbusyconnections > 0)
+		            {
+	    				if (!trace)
+	    					log.warning(getStatus());
+		                //hengsin: make a best effort to reclaim leak connection
+		                Runtime.getRuntime().runFinalization();
+		            }
+        		}
+        	} else {
+        		//don't use log.severe here as it will try to access db again
+        		System.err.println("Failed to acquire new connection. Status=" + getStatus());
+        	}
+        }
+        catch (Exception ex)
+        {
+        }
+        if (exception != null)
+            throw exception;
+        return conn;
 	}	//	getCachedConnection
 
 
@@ -543,6 +616,25 @@ public class DB_PostgreSQL implements AdempiereDatabase
 		if (m_ds != null)
 			return m_ds;
 
+		URL url = Ini.isClient()
+    	? PostgreSQLBundleActivator.bundleContext.getBundle().getEntry("META-INF/pool/client.properties")
+    	: PostgreSQLBundleActivator.bundleContext.getBundle().getEntry("META-INF/pool/server.properties");
+		Properties poolProperties = new Properties();
+		try {
+			poolProperties.load(url.openStream());
+		} catch (IOException e) {
+			throw new DBException(e);
+		}
+
+		int idleConnectionTestPeriod = getIntProperty(poolProperties, "IdleConnectionTestPeriod", 1200);
+		int acquireRetryAttempts = getIntProperty(poolProperties, "AcquireRetryAttempts", 2);
+		int maxIdleTimeExcessConnections = getIntProperty(poolProperties, "MaxIdleTimeExcessConnections", 1200);
+		int maxIdleTime = getIntProperty(poolProperties, "MaxIdleTime", 1200);
+		int unreturnedConnectionTimeout = getIntProperty(poolProperties, "UnreturnedConnectionTimeout", 0);
+		boolean testConnectionOnCheckin = getBooleanProperty(poolProperties, "TestConnectionOnCheckin", false);
+		boolean testConnectionOnCheckout = getBooleanProperty(poolProperties, "TestConnectionOnCheckout", false);
+		int checkoutTimeout = getIntProperty(poolProperties, "CheckoutTimeout", 0);
+
         try
         {
             System.setProperty("com.mchange.v2.log.MLog", "com.mchange.v2.log.FallbackMLog");
@@ -555,34 +647,44 @@ public class DB_PostgreSQL implements AdempiereDatabase
             cpds.setUser(connection.getDbUid());
             cpds.setPassword(connection.getDbPwd());
             cpds.setPreferredTestQuery(DEFAULT_CONN_TEST_SQL);
-            cpds.setIdleConnectionTestPeriod(1200);
+            cpds.setIdleConnectionTestPeriod(idleConnectionTestPeriod);
-            //cpds.setTestConnectionOnCheckin(true);
+            cpds.setMaxIdleTimeExcessConnections(maxIdleTimeExcessConnections);
-            //cpds.setTestConnectionOnCheckout(true);
+            cpds.setMaxIdleTime(maxIdleTime);
-            cpds.setAcquireRetryAttempts(2);
+            cpds.setTestConnectionOnCheckin(testConnectionOnCheckin);
-            //cpds.setCheckoutTimeout(60);
+            cpds.setTestConnectionOnCheckout(testConnectionOnCheckout);
+            cpds.setAcquireRetryAttempts(acquireRetryAttempts);
+            if (checkoutTimeout > 0)
+            	cpds.setCheckoutTimeout(checkoutTimeout);
 
             if (Ini.isClient())
             {
-                cpds.setInitialPoolSize(1);
+            	int maxPoolSize = getIntProperty(poolProperties, "MaxPoolSize", 15);
-                cpds.setMinPoolSize(1);
+            	int initialPoolSize = getIntProperty(poolProperties, "InitialPoolSize", 1);
-                cpds.setMaxPoolSize(15);
+            	int minPoolSize = getIntProperty(poolProperties, "MinPoolSize", 1);
-                cpds.setMaxIdleTimeExcessConnections(1200);
+                cpds.setInitialPoolSize(initialPoolSize);
-                cpds.setMaxIdleTime(900);
+                cpds.setMinPoolSize(minPoolSize);
-                m_maxbusyconnections = 10;
+                cpds.setMaxPoolSize(maxPoolSize);
+
+                m_maxbusyconnections = (int) (maxPoolSize * 0.9);
             }
             else
             {
-                cpds.setInitialPoolSize(10);
+            	int maxPoolSize = getIntProperty(poolProperties, "MaxPoolSize", 400);
-                cpds.setMinPoolSize(5);
+            	int initialPoolSize = getIntProperty(poolProperties, "InitialPoolSize", 10);
-                cpds.setMaxPoolSize(150);
+            	int minPoolSize = getIntProperty(poolProperties, "MinPoolSize", 5);
-                cpds.setMaxIdleTimeExcessConnections(1200);
+                cpds.setInitialPoolSize(initialPoolSize);
-                cpds.setMaxIdleTime(1200);
+                cpds.setInitialPoolSize(initialPoolSize);
-                m_maxbusyconnections = 120;
+                cpds.setMinPoolSize(minPoolSize);
+                cpds.setMaxPoolSize(maxPoolSize);
+                m_maxbusyconnections = (int) (maxPoolSize * 0.9);
             }
 
-            //the following sometimes kill active connection!
+            if (unreturnedConnectionTimeout > 0)
-            //cpds.setUnreturnedConnectionTimeout(1200);
+            {
-            //cpds.setDebugUnreturnedConnectionStackTraces(true);
+	            //the following sometimes kill active connection!
+	            cpds.setUnreturnedConnectionTimeout(1200);
+	            cpds.setDebugUnreturnedConnectionStackTraces(true);
+            }
 
             m_ds = cpds;
         }
@@ -686,7 +788,7 @@ public class DB_PostgreSQL implements AdempiereDatabase
 		//jz temp, modify later from user.constraints
 	}
 
-        	/**
+    /**
 	 *  Check if DBMS support the sql statement
 	 *  @sql SQL statement
 	 *  @return true: yes
@@ -732,42 +834,6 @@ public class DB_PostgreSQL implements AdempiereDatabase
 		}
 	}
 
-	/**
-	 * 	Test
-	 *	@param args ignored
-	 */
-	public static void main(String[] args)
-	{
-		DB_PostgreSQL postgresql = new DB_PostgreSQL();
-		//
-		String databaseName = "adempiere";
-		String uid = "adempiere";
-		String pwd = "adempiere";
-		String jdbcURL = postgresql.getConnectionURL("vpj", DEFAULT_PORT, databaseName, uid);
-		System.out.println(jdbcURL);
-		try
-		{
-			postgresql.getDriver();
-			Connection conn = DriverManager.getConnection (jdbcURL, uid, pwd);                        
-			
-			//CachedRowSetImpl crs = null;
-			//crs = new CachedRowSetImpl();
-			//crs.setSyncProvider("com.sun.rowset.providers.RIOptimisticProvider");
-			//crs.setConcurrency(ResultSet.CONCUR_READ_ONLY);
-			//crs.setType(ResultSet.TYPE_SCROLL_INSENSITIVE);
-			//crs.setCommand("SELECT * FROM AD_Client");
-			//
-			//crs.execute(conn);
-			//
-			conn.close();
-			conn = null;
-		}
-		catch (Exception ex)
-		{
-			ex.printStackTrace();
-		}
-	}	//	main
-
 	public int getNextID(String name) {
 
 		int m_sequence_id = DB.getSQLValue(null, "SELECT nextval('"+name.toLowerCase()+"')");
@@ -824,4 +890,30 @@ public class DB_PostgreSQL implements AdempiereDatabase
 	public boolean isPagingSupported() {
 		return true;
 	}
+
+	private int getIntProperty(Properties properties, String key, int defaultValue)
+	{
+		int i = defaultValue;
+		try
+		{
+			String s = properties.getProperty(key);
+			if (s != null && s.trim().length() > 0)
+				i = Integer.parseInt(s);
+		}
+		catch (Exception e) {}
+		return i;
+	}
+
+	private boolean getBooleanProperty(Properties properties, String key, boolean defaultValue)
+	{
+		boolean b = defaultValue;
+		try
+		{
+			String s = properties.getProperty(key);
+			if (s != null && s.trim().length() > 0)
+				b = Boolean.valueOf(s);
+		}
+		catch (Exception e) {}
+		return b;
+	}
 }   //  DB_PostgreSQL