IDEMPIERE-2631 Process Indicator in CVS Import process

migration/i2.1z/oracle/201505200950_IDEMPIERE-2631.sql

@@ -0,0 +1,13 @@
+SET SQLBLANKLINES ON
+SET DEFINE OFF
+
+-- IDEMPIERE-2631 Process Indicator in CVS Import process
+-- May 19, 2015 12:54:20 PM CEST
+INSERT INTO AD_Message (MsgType,MsgText,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Message_ID,Value,EntityType,AD_Message_UU) VALUES ('I','Processing... {0} of {1} ({2}%)',0,0,'Y',TO_DATE('2015-05-19 12:54:19','YYYY-MM-DD HH24:MI:SS'),0,TO_DATE('2015-05-19 12:54:19','YYYY-MM-DD HH24:MI:SS'),0,200351,'PercentProcessingProgress','D','ca3d703d-8777-451e-8b78-f05c65e6e99c')
+;
+
+INSERT INTO AD_Message (MsgType,MsgText,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Message_ID,Value,EntityType,AD_Message_UU) VALUES ('I','Checking for errors... {0} preprocessed.',0,0,'Y',TO_DATE('2015-05-20 10:42:30','YYYY-MM-DD HH24:MI:SS'),0,TO_DATE('2015-05-20 10:42:30','YYYY-MM-DD HH24:MI:SS'),0,200352,'PreProcessingCVSProgress','D','86c3ae89-0e50-49cf-b5c0-128bb4a91fd8')
+;
+
+SELECT register_migration_script('201505200950_IDEMPIERE-2631.sql') FROM dual
+;

migration/i2.1z/postgresql/201505200950_IDEMPIERE-2631.sql

@@ -0,0 +1,11 @@
+-- IDEMPIERE-2631 Process Indicator in CVS Import process
+-- May 19, 2015 12:54:20 PM CEST
+INSERT INTO AD_Message (MsgType,MsgText,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Message_ID,Value,EntityType,AD_Message_UU) VALUES ('I','Processing... {0} of {1} ({2}%)',0,0,'Y',TO_TIMESTAMP('2015-05-19 12:54:19','YYYY-MM-DD HH24:MI:SS'),0,TO_TIMESTAMP('2015-05-19 12:54:19','YYYY-MM-DD HH24:MI:SS'),0,200351,'PercentProcessingProgress','D','ca3d703d-8777-451e-8b78-f05c65e6e99c')
+;
+
+INSERT INTO AD_Message (MsgType,MsgText,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Message_ID,Value,EntityType,AD_Message_UU) VALUES ('I','Checking for errors... {0} preprocessed.',0,0,'Y',TO_TIMESTAMP('2015-05-20 10:42:30','YYYY-MM-DD HH24:MI:SS'),0,TO_TIMESTAMP('2015-05-20 10:42:30','YYYY-MM-DD HH24:MI:SS'),0,200352,'PreProcessingCVSProgress','D','86c3ae89-0e50-49cf-b5c0-128bb4a91fd8')
+;
+
+SELECT register_migration_script('201505200950_IDEMPIERE-2631.sql') FROM dual
+;
+

org.adempiere.base.process/src/org/idempiere/process/ImportCSVProcess.java

@@ -133,7 +133,7 @@ public class ImportCSVProcess extends SvrProcess {
 	protected void importFile(String filePath, IGridTabImporter csvImporter, GridTab activeTab, List<GridTab> childTabs) throws Exception {
 		m_file_istream = new FileInputStream(filePath);
 
-		File outFile = csvImporter.fileImport(activeTab, childTabs, m_file_istream, Charset.forName(m_importTemplate.getCharacterSet()), p_ImportMode);
+		File outFile = csvImporter.fileImport(activeTab, childTabs, m_file_istream, Charset.forName(m_importTemplate.getCharacterSet()), p_ImportMode, processUI);
 		// TODO: Potential improvement - traverse the outFile and call addLog with the results
 
 		if (processUI != null)

org.adempiere.base/src/org/adempiere/base/IGridTabImporter.java

@@ -19,6 +19,7 @@ import java.io.InputStream;
 import java.nio.charset.Charset;
 import java.util.List;
 
+import org.adempiere.util.IProcessUI;
 import org.compiere.model.GridTab;
 
 /**
@@ -36,6 +37,18 @@ public interface IGridTabImporter {
 	 * @param charset
 	 */
 	public File fileImport(GridTab gridTab, List<GridTab> childs, InputStream filestream, Charset charset, String importMode);
+	
+	/**
+	 * export gridTab data to file
+	 * @param gridTab
+	 * @param childs
+	 * @param filestream
+	 * @param charset
+	 * @param importMode
+	 * @param processUI
+	 * @return
+	 */
+	public File fileImport(GridTab gridTab, List<GridTab> childs, InputStream filestream, Charset charset, String importMode, IProcessUI processUI);
 
 	/**
 	 * @return file extension

org.adempiere.base/src/org/adempiere/impexp/GridTabCSVImporter.java

@@ -14,8 +14,8 @@
  *****************************************************************************/
 package org.adempiere.impexp;
 
-import static org.compiere.model.SystemIDs.REFERENCE_PAYMENTRULE;
 import static org.compiere.model.SystemIDs.REFERENCE_DOCUMENTACTION;
+import static org.compiere.model.SystemIDs.REFERENCE_PAYMENTRULE;
 
 import java.io.File;
 import java.io.IOException;
@@ -30,6 +30,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Calendar;
 import java.util.Comparator;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -39,6 +40,7 @@ import java.util.logging.Level;
 
 import org.adempiere.base.IGridTabImporter;
 import org.adempiere.exceptions.AdempiereException;
+import org.adempiere.util.IProcessUI;
 import org.adempiere.util.ProcessUtil;
 import org.compiere.model.GridField;
 import org.compiere.model.GridTab;
@@ -93,6 +95,10 @@ public class GridTabCSVImporter implements IGridTabImporter
 	private static CLogger log = CLogger.getCLogger(GridTabCSVImporter.class);
 	
 	public File fileImport(GridTab gridTab, List<GridTab> childs, InputStream filestream, Charset charset , String importMode) {		
+		return fileImport(gridTab, childs, filestream, charset, importMode, null);
+	}
+	@Override
+	public File fileImport(GridTab gridTab, List<GridTab> childs, InputStream filestream, Charset charset, String importMode, IProcessUI processUI) {
 		ICsvMapReader mapReader = null;
 		File errFile = null;
 		File logFile = null;
@@ -105,7 +111,7 @@ public class GridTabCSVImporter implements IGridTabImporter
 		PO masterRecord = null;
         
 		if(!gridTab.isInsertRecord() && isInsertMode())
-        	throw new AdempiereException("Insert record disabled for Tab");
+			throwAdempiereException("Insert record disabled for Tab");
 	
 		try {
 			String errFileName = FileUtil.getTempMailName("Import_" + gridTab.getTableName(), "_err.csv");
@@ -123,8 +129,9 @@ public class GridTabCSVImporter implements IGridTabImporter
 			for(int idx = 0; idx < header.size(); idx++) {
 				String headName = header.get(idx);
 				
-				if (headName==null)
+				if (headName==null) {
-					throw new AdempiereException("Header column cannot be empty, Col: " + (idx + 1));
+					throwAdempiereException("Header column cannot be empty, Col: " + (idx + 1));
+				}
 				
 				if (headName.equals(ERROR_HEADER) || headName.equals(LOG_HEADER)){
 					header.set(idx, null);
@@ -133,7 +140,7 @@ public class GridTabCSVImporter implements IGridTabImporter
 				}
 				if (headName.indexOf(">") > 0) {
 					if(idx==0){
-					   throw new AdempiereException(Msg.getMsg(Env.getCtx(),"WrongHeader", new Object[] {headName}));
+						throwAdempiereException(Msg.getMsg(Env.getCtx(),"WrongHeader", new Object[] {headName}));
 				    }else if (headName.contains(MTable.getTableName(Env.getCtx(), MLocation.Table_ID)) && locationFields==null){ 
 				       locationFields = getSpecialMColumn(header,MTable.getTableName(Env.getCtx(), MLocation.Table_ID),idx);
 					   for(GridField sField:locationFields){
@@ -151,7 +158,7 @@ public class GridTabCSVImporter implements IGridTabImporter
 					GridField field 	= gridTab.getField(columnName);
 					
 					if (field == null)
-						throw new AdempiereException(Msg.getMsg(Env.getCtx(), "FieldNotFound" , new Object[] {columnName}) );
+						throwAdempiereException(Msg.getMsg(Env.getCtx(), "FieldNotFound" , new Object[] {columnName}) );
 					else if(isKeyColumn && !isThereKey)
 						isThereKey =true;
 					else if (!isThereDocAction &&
@@ -164,7 +171,7 @@ public class GridTabCSVImporter implements IGridTabImporter
 			}	
 			
 			if(isUpdateOrMergeMode() && !isThereKey)
-			    throw new AdempiereException(gridTab.getTableName()+": "+Msg.getMsg(Env.getCtx(), "NoKeyFound"));
+				throwAdempiereException(gridTab.getTableName()+": "+Msg.getMsg(Env.getCtx(), "NoKeyFound"));
 			
 			tabMapIndexes.put(gridTab,indxDetail-1);
 			String  childTableName   = null;
@@ -182,7 +189,7 @@ public class GridTabCSVImporter implements IGridTabImporter
 		    		   if(currentDetailTab!=null){ 
 		    			 //check out key per Tab   
 		   		    	 if(isUpdateOrMergeMode() && !isThereKey){
-		 				    throw new AdempiereException(currentDetailTab.getTableName()+": "+Msg.getMsg(Env.getCtx(), "NoKeyFound"));
+		   		    		throwAdempiereException(currentDetailTab.getTableName()+": "+Msg.getMsg(Env.getCtx(), "NoKeyFound"));
 		   		    	 }else{
 		   		    	    tabMapIndexes.put(currentDetailTab,idx-1); 	
 			    			isThereKey =false; 
@@ -198,7 +205,7 @@ public class GridTabCSVImporter implements IGridTabImporter
 		    	   }
 		    	   
 				   if(currentDetailTab == null) 
-					  throw new AdempiereException(Msg.getMsg(Env.getCtx(),"NoChildTab",new Object[] {childTableName}));
+					   throwAdempiereException(Msg.getMsg(Env.getCtx(),"NoChildTab",new Object[] {childTableName}));
 		    	   
 				   String columnName = detailName;
 				   if (columnName.contains(MTable.getTableName(Env.getCtx(), MLocation.Table_ID)) && locationFields==null){
@@ -215,20 +222,20 @@ public class GridTabCSVImporter implements IGridTabImporter
 					   GridField field = currentDetailTab.getField(columnName);
 					  
 					   if(field == null)
-						  throw new AdempiereException(Msg.getMsg(Env.getCtx(), "FieldNotFound",new Object[] {detailName}));
+						   throwAdempiereException(Msg.getMsg(Env.getCtx(), "FieldNotFound",new Object[] {detailName}));
 					   else if(isKeyColumn && !isThereKey)
 						  isThereKey =true;
 					
 					   readProcArray.add(getProccesorFromColumn(field));  
 				   }				   
 		    	}else
-		    	   throw new AdempiereException(Msg.getMsg(Env.getCtx(),"WrongDetailName",new Object[] {" col("+idx+") ",detailName}));
+		    		throwAdempiereException(Msg.getMsg(Env.getCtx(),"WrongDetailName",new Object[] {" col("+idx+") ",detailName}));
 		    	
 		    }
 		    
 		    if(currentDetailTab!=null){
 		    	if(isUpdateOrMergeMode() && !isThereKey)
-				   throw new AdempiereException(currentDetailTab.getTableName()+": "+Msg.getMsg(Env.getCtx(), "NoKeyFound"));
+		    		throwAdempiereException(currentDetailTab.getTableName()+": "+Msg.getMsg(Env.getCtx(), "NoKeyFound"));
 
 			    tabMapIndexes.put(currentDetailTab,header.size()-1); 	   
 		    }
@@ -254,7 +261,12 @@ public class GridTabCSVImporter implements IGridTabImporter
 			List<Map<String, Object>> data = new ArrayList<Map<String, Object>>();
 			List<String> rawData = new ArrayList<String>();
 			// pre-process to check for errors
+			long lastOutput = new Date().getTime();
 			while (true) {
+				if( processUI != null && new Date().getTime()-lastOutput > 1000 /* one second */){
+					processUI.statusUpdate(refreshImportStatus(data.size(), 0));
+					lastOutput = new Date().getTime();
+				}
 				Map<String, Object> map = null;
 				boolean isLineError = false; 
 				StringBuilder errMsg = new StringBuilder();
@@ -315,7 +327,13 @@ public class GridTabCSVImporter implements IGridTabImporter
 				Trx trx = null;
 				String trxName= null;
 				List<String>  rowsTmpResult = new ArrayList<String>();
+				lastOutput = new Date().getTime();
+
 				for (int idx = 0; idx < data.size(); idx++) {
+					if( processUI != null && new Date().getTime()-lastOutput > 1000 /* one second */){
+						processUI.statusUpdate(refreshImportStatus(idx + 1, data.size() + 1));
+						lastOutput = new Date().getTime();
+					}
 					String rawLine = rawData.get(idx);
 					String logMsg = null;
 					StringBuilder rowResult = new StringBuilder();
@@ -371,7 +389,7 @@ public class GridTabCSVImporter implements IGridTabImporter
 										  
 								    	  rowsTmpResult.set(0,rowsTmpResult.get(0).replace(quoteChar + "\n",docResult + quoteChar + "\n")); 
 								      }else {
-								      	 throw new AdempiereException("No Process found for document action.");	  
+								    	  throwAdempiereException("No Process found for document action.");	  
 								      }
 									  
 									  if(isError){
@@ -539,7 +557,7 @@ public class GridTabCSVImporter implements IGridTabImporter
 					rawLine = rawLine + delimiter + quoteChar + rowResult.toString().replaceAll(delimiter, "") + quoteChar + "\n";
 					rowsTmpResult.add(rawLine);
 				}
-					
+
 				if(trx!=null){
 				   if(error){
 					  trx.rollback();	 
@@ -562,24 +580,24 @@ public class GridTabCSVImporter implements IGridTabImporter
 						     
 						     rowsTmpResult.set(0,rowsTmpResult.get(0).replace(quoteChar + "\n",docResult + quoteChar + "\n"));    
 						 }else {
-						    throw new AdempiereException("No Process found for document action.");	  
+							 throwAdempiereException("No Process found for document action.");	  
 						 }
 						 
 						 if(isError){
-							trx.rollback();	 
+							trx.rollback();
 							for(String row:rowsTmpResult){
 								row = row.replaceAll("Updated","RolledBack");
 								row = row.replaceAll("Inserted","RolledBack");
 							    logFileW.write(row);
-							}   
+							}
 						 }else{
-							trx.commit();   
+							trx.commit();
-							for(String row:rowsTmpResult)						   
+							for(String row:rowsTmpResult)
 								logFileW.write(row);
 						 }
 					  }else {
 					     trx.commit();  
-						 for(String row:rowsTmpResult)						   
+						 for(String row:rowsTmpResult)
 							 logFileW.write(row);
 					  }
 					}   
@@ -622,6 +640,26 @@ public class GridTabCSVImporter implements IGridTabImporter
 		else
 			return errFile;
 	}
+	
+	private void throwAdempiereException(String msg){
+	    throw new AdempiereException(msg);
+	}
+	
+	private String refreshImportStatus(int currentRecord, int total){
+		int percent = currentRecord * 100;
+		if (total > 0)
+			percent = percent / total;
+		else
+			percent = 0;
+		
+		if( percent == 0 ){
+			Object[] args = new Object[] {currentRecord};
+			return Msg.getMsg(Env.getCtx(), "PreProcessingCVSProgress", args);
+		}else{
+			Object[] args = new Object[] {currentRecord, total, percent};
+			return Msg.getMsg(Env.getCtx(), "PercentProcessingProgress", args);
+		}
+	}
 
 	private String processDocAction(PO document, int AD_Process_ID){
 		int AD_Workflow_ID = MProcess.get(Env.getCtx(),AD_Process_ID).getAD_Workflow_ID();