From d4a6a974377232251ae22c37d0a919ff1d87ce82 Mon Sep 17 00:00:00 2001 From: Diego Ruiz Date: Wed, 20 May 2015 11:03:56 +0200 Subject: [PATCH] IDEMPIERE-2631 Process Indicator in CVS Import process --- .../oracle/201505200950_IDEMPIERE-2631.sql | 13 ++++ .../201505200950_IDEMPIERE-2631.sql | 11 +++ .../idempiere/process/ImportCSVProcess.java | 2 +- .../org/adempiere/base/IGridTabImporter.java | 13 ++++ .../adempiere/impexp/GridTabCSVImporter.java | 78 ++++++++++++++----- 5 files changed, 96 insertions(+), 21 deletions(-) create mode 100644 migration/i2.1z/oracle/201505200950_IDEMPIERE-2631.sql create mode 100644 migration/i2.1z/postgresql/201505200950_IDEMPIERE-2631.sql diff --git a/migration/i2.1z/oracle/201505200950_IDEMPIERE-2631.sql b/migration/i2.1z/oracle/201505200950_IDEMPIERE-2631.sql new file mode 100644 index 0000000000..d1bf0962f6 --- /dev/null +++ b/migration/i2.1z/oracle/201505200950_IDEMPIERE-2631.sql @@ -0,0 +1,13 @@ +SET SQLBLANKLINES ON +SET DEFINE OFF + +-- IDEMPIERE-2631 Process Indicator in CVS Import process +-- May 19, 2015 12:54:20 PM CEST +INSERT INTO AD_Message (MsgType,MsgText,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Message_ID,Value,EntityType,AD_Message_UU) VALUES ('I','Processing... {0} of {1} ({2}%)',0,0,'Y',TO_DATE('2015-05-19 12:54:19','YYYY-MM-DD HH24:MI:SS'),0,TO_DATE('2015-05-19 12:54:19','YYYY-MM-DD HH24:MI:SS'),0,200351,'PercentProcessingProgress','D','ca3d703d-8777-451e-8b78-f05c65e6e99c') +; + +INSERT INTO AD_Message (MsgType,MsgText,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Message_ID,Value,EntityType,AD_Message_UU) VALUES ('I','Checking for errors... {0} preprocessed.',0,0,'Y',TO_DATE('2015-05-20 10:42:30','YYYY-MM-DD HH24:MI:SS'),0,TO_DATE('2015-05-20 10:42:30','YYYY-MM-DD HH24:MI:SS'),0,200352,'PreProcessingCVSProgress','D','86c3ae89-0e50-49cf-b5c0-128bb4a91fd8') +; + +SELECT register_migration_script('201505200950_IDEMPIERE-2631.sql') FROM dual +; diff --git a/migration/i2.1z/postgresql/201505200950_IDEMPIERE-2631.sql b/migration/i2.1z/postgresql/201505200950_IDEMPIERE-2631.sql new file mode 100644 index 0000000000..419f85f842 --- /dev/null +++ b/migration/i2.1z/postgresql/201505200950_IDEMPIERE-2631.sql @@ -0,0 +1,11 @@ +-- IDEMPIERE-2631 Process Indicator in CVS Import process +-- May 19, 2015 12:54:20 PM CEST +INSERT INTO AD_Message (MsgType,MsgText,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Message_ID,Value,EntityType,AD_Message_UU) VALUES ('I','Processing... {0} of {1} ({2}%)',0,0,'Y',TO_TIMESTAMP('2015-05-19 12:54:19','YYYY-MM-DD HH24:MI:SS'),0,TO_TIMESTAMP('2015-05-19 12:54:19','YYYY-MM-DD HH24:MI:SS'),0,200351,'PercentProcessingProgress','D','ca3d703d-8777-451e-8b78-f05c65e6e99c') +; + +INSERT INTO AD_Message (MsgType,MsgText,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Message_ID,Value,EntityType,AD_Message_UU) VALUES ('I','Checking for errors... {0} preprocessed.',0,0,'Y',TO_TIMESTAMP('2015-05-20 10:42:30','YYYY-MM-DD HH24:MI:SS'),0,TO_TIMESTAMP('2015-05-20 10:42:30','YYYY-MM-DD HH24:MI:SS'),0,200352,'PreProcessingCVSProgress','D','86c3ae89-0e50-49cf-b5c0-128bb4a91fd8') +; + +SELECT register_migration_script('201505200950_IDEMPIERE-2631.sql') FROM dual +; + diff --git a/org.adempiere.base.process/src/org/idempiere/process/ImportCSVProcess.java b/org.adempiere.base.process/src/org/idempiere/process/ImportCSVProcess.java index 22b4af1791..0d9e5a9569 100644 --- a/org.adempiere.base.process/src/org/idempiere/process/ImportCSVProcess.java +++ b/org.adempiere.base.process/src/org/idempiere/process/ImportCSVProcess.java @@ -133,7 +133,7 @@ public class ImportCSVProcess extends SvrProcess { protected void importFile(String filePath, IGridTabImporter csvImporter, GridTab activeTab, List childTabs) throws Exception { m_file_istream = new FileInputStream(filePath); - File outFile = csvImporter.fileImport(activeTab, childTabs, m_file_istream, Charset.forName(m_importTemplate.getCharacterSet()), p_ImportMode); + File outFile = csvImporter.fileImport(activeTab, childTabs, m_file_istream, Charset.forName(m_importTemplate.getCharacterSet()), p_ImportMode, processUI); // TODO: Potential improvement - traverse the outFile and call addLog with the results if (processUI != null) diff --git a/org.adempiere.base/src/org/adempiere/base/IGridTabImporter.java b/org.adempiere.base/src/org/adempiere/base/IGridTabImporter.java index 58fdeaca4c..0849e01798 100644 --- a/org.adempiere.base/src/org/adempiere/base/IGridTabImporter.java +++ b/org.adempiere.base/src/org/adempiere/base/IGridTabImporter.java @@ -19,6 +19,7 @@ import java.io.InputStream; import java.nio.charset.Charset; import java.util.List; +import org.adempiere.util.IProcessUI; import org.compiere.model.GridTab; /** @@ -36,6 +37,18 @@ public interface IGridTabImporter { * @param charset */ public File fileImport(GridTab gridTab, List childs, InputStream filestream, Charset charset, String importMode); + + /** + * export gridTab data to file + * @param gridTab + * @param childs + * @param filestream + * @param charset + * @param importMode + * @param processUI + * @return + */ + public File fileImport(GridTab gridTab, List childs, InputStream filestream, Charset charset, String importMode, IProcessUI processUI); /** * @return file extension diff --git a/org.adempiere.base/src/org/adempiere/impexp/GridTabCSVImporter.java b/org.adempiere.base/src/org/adempiere/impexp/GridTabCSVImporter.java index c7c7c23920..c1002d01b5 100644 --- a/org.adempiere.base/src/org/adempiere/impexp/GridTabCSVImporter.java +++ b/org.adempiere.base/src/org/adempiere/impexp/GridTabCSVImporter.java @@ -14,8 +14,8 @@ *****************************************************************************/ package org.adempiere.impexp; -import static org.compiere.model.SystemIDs.REFERENCE_PAYMENTRULE; import static org.compiere.model.SystemIDs.REFERENCE_DOCUMENTACTION; +import static org.compiere.model.SystemIDs.REFERENCE_PAYMENTRULE; import java.io.File; import java.io.IOException; @@ -30,6 +30,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; import java.util.Comparator; +import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -39,6 +40,7 @@ import java.util.logging.Level; import org.adempiere.base.IGridTabImporter; import org.adempiere.exceptions.AdempiereException; +import org.adempiere.util.IProcessUI; import org.adempiere.util.ProcessUtil; import org.compiere.model.GridField; import org.compiere.model.GridTab; @@ -93,6 +95,10 @@ public class GridTabCSVImporter implements IGridTabImporter private static CLogger log = CLogger.getCLogger(GridTabCSVImporter.class); public File fileImport(GridTab gridTab, List childs, InputStream filestream, Charset charset , String importMode) { + return fileImport(gridTab, childs, filestream, charset, importMode, null); + } + @Override + public File fileImport(GridTab gridTab, List childs, InputStream filestream, Charset charset, String importMode, IProcessUI processUI) { ICsvMapReader mapReader = null; File errFile = null; File logFile = null; @@ -105,7 +111,7 @@ public class GridTabCSVImporter implements IGridTabImporter PO masterRecord = null; if(!gridTab.isInsertRecord() && isInsertMode()) - throw new AdempiereException("Insert record disabled for Tab"); + throwAdempiereException("Insert record disabled for Tab"); try { String errFileName = FileUtil.getTempMailName("Import_" + gridTab.getTableName(), "_err.csv"); @@ -123,8 +129,9 @@ public class GridTabCSVImporter implements IGridTabImporter for(int idx = 0; idx < header.size(); idx++) { String headName = header.get(idx); - if (headName==null) - throw new AdempiereException("Header column cannot be empty, Col: " + (idx + 1)); + if (headName==null) { + throwAdempiereException("Header column cannot be empty, Col: " + (idx + 1)); + } if (headName.equals(ERROR_HEADER) || headName.equals(LOG_HEADER)){ header.set(idx, null); @@ -133,7 +140,7 @@ public class GridTabCSVImporter implements IGridTabImporter } if (headName.indexOf(">") > 0) { if(idx==0){ - throw new AdempiereException(Msg.getMsg(Env.getCtx(),"WrongHeader", new Object[] {headName})); + throwAdempiereException(Msg.getMsg(Env.getCtx(),"WrongHeader", new Object[] {headName})); }else if (headName.contains(MTable.getTableName(Env.getCtx(), MLocation.Table_ID)) && locationFields==null){ locationFields = getSpecialMColumn(header,MTable.getTableName(Env.getCtx(), MLocation.Table_ID),idx); for(GridField sField:locationFields){ @@ -151,7 +158,7 @@ public class GridTabCSVImporter implements IGridTabImporter GridField field = gridTab.getField(columnName); if (field == null) - throw new AdempiereException(Msg.getMsg(Env.getCtx(), "FieldNotFound" , new Object[] {columnName}) ); + throwAdempiereException(Msg.getMsg(Env.getCtx(), "FieldNotFound" , new Object[] {columnName}) ); else if(isKeyColumn && !isThereKey) isThereKey =true; else if (!isThereDocAction && @@ -164,7 +171,7 @@ public class GridTabCSVImporter implements IGridTabImporter } if(isUpdateOrMergeMode() && !isThereKey) - throw new AdempiereException(gridTab.getTableName()+": "+Msg.getMsg(Env.getCtx(), "NoKeyFound")); + throwAdempiereException(gridTab.getTableName()+": "+Msg.getMsg(Env.getCtx(), "NoKeyFound")); tabMapIndexes.put(gridTab,indxDetail-1); String childTableName = null; @@ -182,7 +189,7 @@ public class GridTabCSVImporter implements IGridTabImporter if(currentDetailTab!=null){ //check out key per Tab if(isUpdateOrMergeMode() && !isThereKey){ - throw new AdempiereException(currentDetailTab.getTableName()+": "+Msg.getMsg(Env.getCtx(), "NoKeyFound")); + throwAdempiereException(currentDetailTab.getTableName()+": "+Msg.getMsg(Env.getCtx(), "NoKeyFound")); }else{ tabMapIndexes.put(currentDetailTab,idx-1); isThereKey =false; @@ -198,7 +205,7 @@ public class GridTabCSVImporter implements IGridTabImporter } if(currentDetailTab == null) - throw new AdempiereException(Msg.getMsg(Env.getCtx(),"NoChildTab",new Object[] {childTableName})); + throwAdempiereException(Msg.getMsg(Env.getCtx(),"NoChildTab",new Object[] {childTableName})); String columnName = detailName; if (columnName.contains(MTable.getTableName(Env.getCtx(), MLocation.Table_ID)) && locationFields==null){ @@ -215,20 +222,20 @@ public class GridTabCSVImporter implements IGridTabImporter GridField field = currentDetailTab.getField(columnName); if(field == null) - throw new AdempiereException(Msg.getMsg(Env.getCtx(), "FieldNotFound",new Object[] {detailName})); + throwAdempiereException(Msg.getMsg(Env.getCtx(), "FieldNotFound",new Object[] {detailName})); else if(isKeyColumn && !isThereKey) isThereKey =true; readProcArray.add(getProccesorFromColumn(field)); } }else - throw new AdempiereException(Msg.getMsg(Env.getCtx(),"WrongDetailName",new Object[] {" col("+idx+") ",detailName})); + throwAdempiereException(Msg.getMsg(Env.getCtx(),"WrongDetailName",new Object[] {" col("+idx+") ",detailName})); } if(currentDetailTab!=null){ if(isUpdateOrMergeMode() && !isThereKey) - throw new AdempiereException(currentDetailTab.getTableName()+": "+Msg.getMsg(Env.getCtx(), "NoKeyFound")); + throwAdempiereException(currentDetailTab.getTableName()+": "+Msg.getMsg(Env.getCtx(), "NoKeyFound")); tabMapIndexes.put(currentDetailTab,header.size()-1); } @@ -254,7 +261,12 @@ public class GridTabCSVImporter implements IGridTabImporter List> data = new ArrayList>(); List rawData = new ArrayList(); // pre-process to check for errors + long lastOutput = new Date().getTime(); while (true) { + if( processUI != null && new Date().getTime()-lastOutput > 1000 /* one second */){ + processUI.statusUpdate(refreshImportStatus(data.size(), 0)); + lastOutput = new Date().getTime(); + } Map map = null; boolean isLineError = false; StringBuilder errMsg = new StringBuilder(); @@ -315,7 +327,13 @@ public class GridTabCSVImporter implements IGridTabImporter Trx trx = null; String trxName= null; List rowsTmpResult = new ArrayList(); + lastOutput = new Date().getTime(); + for (int idx = 0; idx < data.size(); idx++) { + if( processUI != null && new Date().getTime()-lastOutput > 1000 /* one second */){ + processUI.statusUpdate(refreshImportStatus(idx + 1, data.size() + 1)); + lastOutput = new Date().getTime(); + } String rawLine = rawData.get(idx); String logMsg = null; StringBuilder rowResult = new StringBuilder(); @@ -371,7 +389,7 @@ public class GridTabCSVImporter implements IGridTabImporter rowsTmpResult.set(0,rowsTmpResult.get(0).replace(quoteChar + "\n",docResult + quoteChar + "\n")); }else { - throw new AdempiereException("No Process found for document action."); + throwAdempiereException("No Process found for document action."); } if(isError){ @@ -539,7 +557,7 @@ public class GridTabCSVImporter implements IGridTabImporter rawLine = rawLine + delimiter + quoteChar + rowResult.toString().replaceAll(delimiter, "") + quoteChar + "\n"; rowsTmpResult.add(rawLine); } - + if(trx!=null){ if(error){ trx.rollback(); @@ -562,24 +580,24 @@ public class GridTabCSVImporter implements IGridTabImporter rowsTmpResult.set(0,rowsTmpResult.get(0).replace(quoteChar + "\n",docResult + quoteChar + "\n")); }else { - throw new AdempiereException("No Process found for document action."); + throwAdempiereException("No Process found for document action."); } if(isError){ - trx.rollback(); + trx.rollback(); for(String row:rowsTmpResult){ row = row.replaceAll("Updated","RolledBack"); row = row.replaceAll("Inserted","RolledBack"); logFileW.write(row); - } + } }else{ - trx.commit(); - for(String row:rowsTmpResult) + trx.commit(); + for(String row:rowsTmpResult) logFileW.write(row); } }else { trx.commit(); - for(String row:rowsTmpResult) + for(String row:rowsTmpResult) logFileW.write(row); } } @@ -622,6 +640,26 @@ public class GridTabCSVImporter implements IGridTabImporter else return errFile; } + + private void throwAdempiereException(String msg){ + throw new AdempiereException(msg); + } + + private String refreshImportStatus(int currentRecord, int total){ + int percent = currentRecord * 100; + if (total > 0) + percent = percent / total; + else + percent = 0; + + if( percent == 0 ){ + Object[] args = new Object[] {currentRecord}; + return Msg.getMsg(Env.getCtx(), "PreProcessingCVSProgress", args); + }else{ + Object[] args = new Object[] {currentRecord, total, percent}; + return Msg.getMsg(Env.getCtx(), "PercentProcessingProgress", args); + } + } private String processDocAction(PO document, int AD_Process_ID){ int AD_Workflow_ID = MProcess.get(Env.getCtx(),AD_Process_ID).getAD_Workflow_ID();