midsuit
/
core-jgi
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

IDEMPIERE-6040 Improvements for CSV import template (#2292) 

- improve beforeSave validation of CSVHeader using SuperCSV library
- add validation for the CSVAliasHeader
This commit is contained in:
Carlos Ruiz 2024-04-05 03:56:30 +02:00
parent b7485b679e
commit 4af5caf944
3 changed files with 87 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
migration/iD11/oracle/202404041545_IDEMPIERE-6040.sql Normal file
View File

@ -0,0 +1,10 @@
-- IDEMPIERE-6040 Improvements for CSV import template
SELECT register_migration_script('202404041545_IDEMPIERE-6040.sql') FROM dual;
SET SQLBLANKLINES ON
SET DEFINE OFF
-- Apr 4, 2024, 3:45:17 PM CEST
INSERT INTO AD_Message (MsgType,MsgText,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Message_ID,Value,EntityType,AD_Message_UU) VALUES ('E','CSV Alias Header is not valid, it must have the same number of columns as the CSV Header',0,0,'Y',TO_TIMESTAMP('2024-04-04 15:45:16','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-04-04 15:45:16','YYYY-MM-DD HH24:MI:SS'),100,200885,'CSVAliasHeaderNotValid','D','c0f8a304-5ff2-4503-95a0-13d61aa391a3')
;

7
migration/iD11/postgresql/202404041545_IDEMPIERE-6040.sql Normal file
View File

@ -0,0 +1,7 @@
-- IDEMPIERE-6040 Improvements for CSV import template
SELECT register_migration_script('202404041545_IDEMPIERE-6040.sql') FROM dual;
-- Apr 4, 2024, 3:45:17 PM CEST
INSERT INTO AD_Message (MsgType,MsgText,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Message_ID,Value,EntityType,AD_Message_UU) VALUES ('E','CSV Alias Header is not valid, it must have the same number of columns as the CSV Header',0,0,'Y',TO_TIMESTAMP('2024-04-04 15:45:16','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-04-04 15:45:16','YYYY-MM-DD HH24:MI:SS'),100,200885,'CSVAliasHeaderNotValid','D','c0f8a304-5ff2-4503-95a0-13d61aa391a3')
;

75
org.adempiere.base/src/org/compiere/model/MImportTemplate.java
View File

@ -15,6 +15,7 @@ package org.compiere.model;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
@ -26,6 +27,7 @@ import java.sql.ResultSet;
import java.text.DecimalFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.Properties;
@ -45,7 +47,11 @@ import org.compiere.util.DB;
import org.compiere.util.DisplayType;
import org.compiere.util.Env;
import org.compiere.util.Msg;
import org.compiere.util.Util;
import org.idempiere.cache.ImmutablePOSupport;
import org.supercsv.io.CsvMapReader;
import org.supercsv.io.ICsvMapReader;
import org.supercsv.prefs.CsvPreference;
/**
* Import Template Model
@ -136,8 +142,14 @@ public class MImportTemplate extends X_AD_ImportTemplate implements ImmutablePOS
log.saveError("Error", Msg.parseTranslation(getCtx(), "@Invalid@ @CharacterSet@"));
return false;
}
if (is_new() || is_ValueChanged(COLUMNNAME_CSVHeader) || is_ValueChanged(COLUMNNAME_AD_Tab_ID))
calculateColumnTypes(); // this throws an Exception if there are wrong columns in the CSV Header
if ( is_new()
|| is_ValueChanged(COLUMNNAME_CSVHeader)
|| is_ValueChanged(COLUMNNAME_CSVAliasHeader)
|| is_ValueChanged(COLUMNNAME_CharacterSet)
|| is_ValueChanged(COLUMNNAME_SeparatorChar)
|| is_ValueChanged(COLUMNNAME_QuoteChar)
|| is_ValueChanged(COLUMNNAME_AD_Tab_ID))
calculateAndValidateColumnTypes(); // this throws an Exception if there are wrong columns in the CSV Header
return super.beforeSave(newRecord);
}
@ -308,7 +320,7 @@ public class MImportTemplate extends X_AD_ImportTemplate implements ImmutablePOS
throw new AdempiereException("Wrong template type -> " + getImportTemplateType());
}
List<Integer> colTypes = calculateColumnTypes();
List<Integer> colTypes = calculateAndValidateColumnTypes();
Sheet sheet = workbook.getSheetAt(0); // First sheet
@ -444,9 +456,62 @@ public class MImportTemplate extends X_AD_ImportTemplate implements ImmutablePOS
* Any column can end with /K (can be ignored)
* @return List of expected DisplayType for every column
*/
private List<Integer> calculateColumnTypes() {
private List<Integer> calculateAndValidateColumnTypes() {
List<Integer> retValue = new ArrayList<Integer>();
String[] csvHeaders = getCSVHeader().split(getSeparatorChar());
String delimiterChar = getSeparatorChar();
String quoteChar = getQuoteChar();
CsvPreference csvpref = new CsvPreference.Builder(quoteChar.charAt(0), delimiterChar.charAt(0), "\r\n" /* ignored */).build();
InputStream is = null;
List<String>csvHeaders = null;
ICsvMapReader mapReader = null;
try {
is = new ByteArrayInputStream( getCSVHeader().getBytes(getCharacterSet()));
InputStreamReader reader = new InputStreamReader(is);
mapReader = new CsvMapReader(reader, csvpref);
csvHeaders = Arrays.asList(mapReader.getHeader(true));
} catch (IOException e) {
throw new AdempiereException(e);
} finally {
if (mapReader != null) {
try {
mapReader.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
if (csvHeaders == null || csvHeaders.size() == 0) {
throwCSVHeaderNotFound("");
}
// Validate that alias has the same number of columns as the header
if (! Util.isEmpty(getCSVAliasHeader())) {
InputStream isa = null;
List<String>csvAliasHeaders = null;
ICsvMapReader mapReaderAlias = null;
try {
isa = new ByteArrayInputStream( getCSVAliasHeader().getBytes(getCharacterSet()));
InputStreamReader reader = new InputStreamReader(isa);
mapReaderAlias = new CsvMapReader(reader, csvpref);
csvAliasHeaders = Arrays.asList(mapReaderAlias.getHeader(true));
} catch (IOException e) {
throw new AdempiereException(e);
} finally {
if (mapReaderAlias != null) {
try {
mapReaderAlias.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
if (csvAliasHeaders == null || csvAliasHeaders.size() != csvHeaders.size()) {
throw new AdempiereException(Msg.getMsg(getCtx(), "CSVAliasHeaderNotValid"));
}
}
// Validate existence of each column and obtain the data type
MTab mainTab = MTab.get(getAD_Tab_ID());
MTable mainTable = MTable.get(mainTab.getAD_Table_ID());
for (String csvHeader : csvHeaders) {