From 4af5caf9449a225c19937f42a21abf1f5ea1efb6 Mon Sep 17 00:00:00 2001 From: Carlos Ruiz Date: Fri, 5 Apr 2024 03:56:30 +0200 Subject: [PATCH] IDEMPIERE-6040 Improvements for CSV import template (#2292) - improve beforeSave validation of CSVHeader using SuperCSV library - add validation for the CSVAliasHeader --- .../oracle/202404041545_IDEMPIERE-6040.sql | 10 +++ .../202404041545_IDEMPIERE-6040.sql | 7 ++ .../org/compiere/model/MImportTemplate.java | 75 +++++++++++++++++-- 3 files changed, 87 insertions(+), 5 deletions(-) create mode 100644 migration/iD11/oracle/202404041545_IDEMPIERE-6040.sql create mode 100644 migration/iD11/postgresql/202404041545_IDEMPIERE-6040.sql diff --git a/migration/iD11/oracle/202404041545_IDEMPIERE-6040.sql b/migration/iD11/oracle/202404041545_IDEMPIERE-6040.sql new file mode 100644 index 0000000000..3d3a4e35d2 --- /dev/null +++ b/migration/iD11/oracle/202404041545_IDEMPIERE-6040.sql @@ -0,0 +1,10 @@ +-- IDEMPIERE-6040 Improvements for CSV import template +SELECT register_migration_script('202404041545_IDEMPIERE-6040.sql') FROM dual; + +SET SQLBLANKLINES ON +SET DEFINE OFF + +-- Apr 4, 2024, 3:45:17 PM CEST +INSERT INTO AD_Message (MsgType,MsgText,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Message_ID,Value,EntityType,AD_Message_UU) VALUES ('E','CSV Alias Header is not valid, it must have the same number of columns as the CSV Header',0,0,'Y',TO_TIMESTAMP('2024-04-04 15:45:16','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-04-04 15:45:16','YYYY-MM-DD HH24:MI:SS'),100,200885,'CSVAliasHeaderNotValid','D','c0f8a304-5ff2-4503-95a0-13d61aa391a3') +; + diff --git a/migration/iD11/postgresql/202404041545_IDEMPIERE-6040.sql b/migration/iD11/postgresql/202404041545_IDEMPIERE-6040.sql new file mode 100644 index 0000000000..22f2cb7eb0 --- /dev/null +++ b/migration/iD11/postgresql/202404041545_IDEMPIERE-6040.sql @@ -0,0 +1,7 @@ +-- IDEMPIERE-6040 Improvements for CSV import template +SELECT register_migration_script('202404041545_IDEMPIERE-6040.sql') FROM dual; + +-- Apr 4, 2024, 3:45:17 PM CEST +INSERT INTO AD_Message (MsgType,MsgText,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Message_ID,Value,EntityType,AD_Message_UU) VALUES ('E','CSV Alias Header is not valid, it must have the same number of columns as the CSV Header',0,0,'Y',TO_TIMESTAMP('2024-04-04 15:45:16','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-04-04 15:45:16','YYYY-MM-DD HH24:MI:SS'),100,200885,'CSVAliasHeaderNotValid','D','c0f8a304-5ff2-4503-95a0-13d61aa391a3') +; + diff --git a/org.adempiere.base/src/org/compiere/model/MImportTemplate.java b/org.adempiere.base/src/org/compiere/model/MImportTemplate.java index 0247ce3c2f..d0579c5f4a 100644 --- a/org.adempiere.base/src/org/compiere/model/MImportTemplate.java +++ b/org.adempiere.base/src/org/compiere/model/MImportTemplate.java @@ -15,6 +15,7 @@ package org.compiere.model; import java.io.BufferedReader; import java.io.BufferedWriter; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileWriter; @@ -26,6 +27,7 @@ import java.sql.ResultSet; import java.text.DecimalFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; import java.util.List; import java.util.Properties; @@ -45,7 +47,11 @@ import org.compiere.util.DB; import org.compiere.util.DisplayType; import org.compiere.util.Env; import org.compiere.util.Msg; +import org.compiere.util.Util; import org.idempiere.cache.ImmutablePOSupport; +import org.supercsv.io.CsvMapReader; +import org.supercsv.io.ICsvMapReader; +import org.supercsv.prefs.CsvPreference; /** * Import Template Model @@ -136,8 +142,14 @@ public class MImportTemplate extends X_AD_ImportTemplate implements ImmutablePOS log.saveError("Error", Msg.parseTranslation(getCtx(), "@Invalid@ @CharacterSet@")); return false; } - if (is_new() || is_ValueChanged(COLUMNNAME_CSVHeader) || is_ValueChanged(COLUMNNAME_AD_Tab_ID)) - calculateColumnTypes(); // this throws an Exception if there are wrong columns in the CSV Header + if ( is_new() + || is_ValueChanged(COLUMNNAME_CSVHeader) + || is_ValueChanged(COLUMNNAME_CSVAliasHeader) + || is_ValueChanged(COLUMNNAME_CharacterSet) + || is_ValueChanged(COLUMNNAME_SeparatorChar) + || is_ValueChanged(COLUMNNAME_QuoteChar) + || is_ValueChanged(COLUMNNAME_AD_Tab_ID)) + calculateAndValidateColumnTypes(); // this throws an Exception if there are wrong columns in the CSV Header return super.beforeSave(newRecord); } @@ -308,7 +320,7 @@ public class MImportTemplate extends X_AD_ImportTemplate implements ImmutablePOS throw new AdempiereException("Wrong template type -> " + getImportTemplateType()); } - List colTypes = calculateColumnTypes(); + List colTypes = calculateAndValidateColumnTypes(); Sheet sheet = workbook.getSheetAt(0); // First sheet @@ -444,9 +456,62 @@ public class MImportTemplate extends X_AD_ImportTemplate implements ImmutablePOS * Any column can end with /K (can be ignored) * @return List of expected DisplayType for every column */ - private List calculateColumnTypes() { + private List calculateAndValidateColumnTypes() { List retValue = new ArrayList(); - String[] csvHeaders = getCSVHeader().split(getSeparatorChar()); + + String delimiterChar = getSeparatorChar(); + String quoteChar = getQuoteChar(); + CsvPreference csvpref = new CsvPreference.Builder(quoteChar.charAt(0), delimiterChar.charAt(0), "\r\n" /* ignored */).build(); + InputStream is = null; + ListcsvHeaders = null; + ICsvMapReader mapReader = null; + try { + is = new ByteArrayInputStream( getCSVHeader().getBytes(getCharacterSet())); + InputStreamReader reader = new InputStreamReader(is); + mapReader = new CsvMapReader(reader, csvpref); + csvHeaders = Arrays.asList(mapReader.getHeader(true)); + } catch (IOException e) { + throw new AdempiereException(e); + } finally { + if (mapReader != null) { + try { + mapReader.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + if (csvHeaders == null || csvHeaders.size() == 0) { + throwCSVHeaderNotFound(""); + } + + // Validate that alias has the same number of columns as the header + if (! Util.isEmpty(getCSVAliasHeader())) { + InputStream isa = null; + ListcsvAliasHeaders = null; + ICsvMapReader mapReaderAlias = null; + try { + isa = new ByteArrayInputStream( getCSVAliasHeader().getBytes(getCharacterSet())); + InputStreamReader reader = new InputStreamReader(isa); + mapReaderAlias = new CsvMapReader(reader, csvpref); + csvAliasHeaders = Arrays.asList(mapReaderAlias.getHeader(true)); + } catch (IOException e) { + throw new AdempiereException(e); + } finally { + if (mapReaderAlias != null) { + try { + mapReaderAlias.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + if (csvAliasHeaders == null || csvAliasHeaders.size() != csvHeaders.size()) { + throw new AdempiereException(Msg.getMsg(getCtx(), "CSVAliasHeaderNotValid")); + } + } + + // Validate existence of each column and obtain the data type MTab mainTab = MTab.get(getAD_Tab_ID()); MTable mainTable = MTable.get(mainTab.getAD_Table_ID()); for (String csvHeader : csvHeaders) {