From d16538ae386a707ea9471ef97e99de0fdc015498 Mon Sep 17 00:00:00 2001 From: Carlos Ruiz Date: Mon, 1 Apr 2024 05:51:47 +0200 Subject: [PATCH] IDEMPIERE-6040 Improvements for CSV import template (#2279) * IDEMPIERE-6040 Improvements for CSV import template - Delimiter (Field Separator) configurable - Quote delimiter configurable - Fix issue when the UTF-8 file comes with BOM character * - add support for preprocessing excel files * - convert Excel to CSV in a proper format according to the column types --- .../oracle/202403262247_IDEMPIERE-6040.sql | 110 +++++++ .../202403262247_IDEMPIERE-6040.sql | 107 +++++++ .../idempiere/process/ImportCSVProcess.java | 2 +- .../org/adempiere/base/IGridTabImporter.java | 14 + .../adempiere/impexp/GridTabCSVImporter.java | 36 ++- .../compiere/model/I_AD_ImportTemplate.java | 27 ++ .../org/compiere/model/MImportTemplate.java | 282 +++++++++++++++++- .../compiere/model/X_AD_ImportTemplate.java | 80 ++++- .../webui/panel/action/CSVImportAction.java | 2 +- 9 files changed, 632 insertions(+), 28 deletions(-) create mode 100644 migration/iD11/oracle/202403262247_IDEMPIERE-6040.sql create mode 100644 migration/iD11/postgresql/202403262247_IDEMPIERE-6040.sql diff --git a/migration/iD11/oracle/202403262247_IDEMPIERE-6040.sql b/migration/iD11/oracle/202403262247_IDEMPIERE-6040.sql new file mode 100644 index 0000000000..4935dc4956 --- /dev/null +++ b/migration/iD11/oracle/202403262247_IDEMPIERE-6040.sql @@ -0,0 +1,110 @@ +-- IDEMPIERE-6040 Improvements for CSV import template +SELECT register_migration_script('202403262247_IDEMPIERE-6040.sql') FROM dual; + +SET SQLBLANKLINES ON +SET DEFINE OFF + +-- Mar 26, 2024, 10:47:25 PM CET +INSERT INTO AD_Column (AD_Column_ID,Version,Name,AD_Table_ID,ColumnName,DefaultValue,FieldLength,IsKey,IsParent,IsMandatory,IsTranslated,IsIdentifier,SeqNo,IsEncrypted,AD_Reference_ID,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Element_ID,IsUpdateable,IsSelectionColumn,EntityType,IsSyncDatabase,IsAlwaysUpdateable,IsAutocomplete,IsAllowLogging,AD_Column_UU,IsAllowCopy,SeqNoSelection,IsToolbarButton,IsSecure,IsHtml,IsPartitionKey) VALUES (216584,0,'Separator Character',200153,'SeparatorChar',',',1,'N','N','Y','N','N',0,'N',10,0,0,'Y',TO_TIMESTAMP('2024-03-26 22:47:24','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-26 22:47:24','YYYY-MM-DD HH24:MI:SS'),100,54158,'Y','N','D','N','N','N','Y','3083364e-7dac-4ee2-8fa9-7ffe047b0461','Y',0,'N','N','N','N') +; + +-- Mar 26, 2024, 10:47:28 PM CET +ALTER TABLE AD_ImportTemplate ADD SeparatorChar VARCHAR2(1 CHAR) DEFAULT ',' NOT NULL +; + +-- Mar 26, 2024, 10:48:07 PM CET +INSERT INTO AD_Element (AD_Element_ID,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,ColumnName,Name,Description,Help,PrintName,EntityType,AD_Element_UU) VALUES (203928,0,0,'Y',TO_TIMESTAMP('2024-03-26 22:47:56','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-26 22:47:56','YYYY-MM-DD HH24:MI:SS'),100,'QuoteChar','Quote Char',NULL,NULL,'Quote Char','D','9bdf8508-f83b-46bc-baca-c27d5b1fac5e') +; + +-- Mar 26, 2024, 10:48:24 PM CET +INSERT INTO AD_Column (AD_Column_ID,Version,Name,AD_Table_ID,ColumnName,DefaultValue,FieldLength,IsKey,IsParent,IsMandatory,IsTranslated,IsIdentifier,SeqNo,IsEncrypted,AD_Reference_ID,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Element_ID,IsUpdateable,IsSelectionColumn,EntityType,IsSyncDatabase,IsAlwaysUpdateable,IsAutocomplete,IsAllowLogging,AD_Column_UU,IsAllowCopy,SeqNoSelection,IsToolbarButton,IsSecure,IsHtml,IsPartitionKey) VALUES (216585,0,'Quote Char',200153,'QuoteChar','"',1,'N','N','Y','N','N',0,'N',10,0,0,'Y',TO_TIMESTAMP('2024-03-26 22:48:23','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-26 22:48:23','YYYY-MM-DD HH24:MI:SS'),100,203928,'Y','N','D','N','N','N','Y','9b51ff6c-2c9c-4f1b-ad66-a8fa4e0b1f1b','Y',0,'N','N','N','N') +; + +-- Mar 26, 2024, 10:48:27 PM CET +ALTER TABLE AD_ImportTemplate ADD QuoteChar VARCHAR2(1 CHAR) DEFAULT '"' NOT NULL +; + +-- Mar 26, 2024, 10:48:44 PM CET +INSERT INTO AD_Field (AD_Field_ID,Name,AD_Tab_ID,AD_Column_ID,IsDisplayed,DisplayLength,SeqNo,IsSameLine,IsHeading,IsFieldOnly,IsEncrypted,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,IsReadOnly,IsCentrallyMaintained,EntityType,AD_Field_UU,IsDisplayedGrid,SeqNoGrid,ColumnSpan) VALUES (208474,'Separator Character',200167,216584,'Y',1,110,'N','N','N','N',0,0,'Y',TO_TIMESTAMP('2024-03-26 22:48:44','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-26 22:48:44','YYYY-MM-DD HH24:MI:SS'),100,'N','Y','D','ada7dc92-28b4-4887-a577-d49226450b6c','Y',110,2) +; + +-- Mar 26, 2024, 10:48:45 PM CET +INSERT INTO AD_Field (AD_Field_ID,Name,AD_Tab_ID,AD_Column_ID,IsDisplayed,DisplayLength,SeqNo,IsSameLine,IsHeading,IsFieldOnly,IsEncrypted,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,IsReadOnly,IsCentrallyMaintained,EntityType,AD_Field_UU,IsDisplayedGrid,SeqNoGrid,ColumnSpan) VALUES (208475,'Quote Char',200167,216585,'Y',1,120,'N','N','N','N',0,0,'Y',TO_TIMESTAMP('2024-03-26 22:48:44','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-26 22:48:44','YYYY-MM-DD HH24:MI:SS'),100,'N','Y','D','6f9a0de7-2df9-46b2-a491-a450f61d594e','Y',120,2) +; + +-- Mar 26, 2024, 10:49:29 PM CET +UPDATE AD_Field SET SeqNo=70, ColumnSpan=1,Updated=TO_TIMESTAMP('2024-03-26 22:49:29','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=203455 +; + +-- Mar 26, 2024, 10:49:29 PM CET +UPDATE AD_Field SET IsDisplayed='Y', SeqNo=80, XPosition=3, ColumnSpan=1,Updated=TO_TIMESTAMP('2024-03-26 22:49:29','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=208474 +; + +-- Mar 26, 2024, 10:49:29 PM CET +UPDATE AD_Field SET IsDisplayed='Y', SeqNo=90, XPosition=5, ColumnSpan=1,Updated=TO_TIMESTAMP('2024-03-26 22:49:29','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=208475 +; + +-- Mar 26, 2024, 10:49:29 PM CET +UPDATE AD_Field SET SeqNo=100,Updated=TO_TIMESTAMP('2024-03-26 22:49:29','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=203456 +; + +-- Mar 26, 2024, 10:49:29 PM CET +UPDATE AD_Field SET SeqNo=110,Updated=TO_TIMESTAMP('2024-03-26 22:49:29','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=203457 +; + +-- Mar 26, 2024, 10:49:29 PM CET +UPDATE AD_Field SET SeqNo=120,Updated=TO_TIMESTAMP('2024-03-26 22:49:29','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=203458 +; + +-- Mar 26, 2024, 10:53:52 PM CET +UPDATE AD_Field SET DefaultValue='@SQL=SELECT '','' FROM Dual',Updated=TO_TIMESTAMP('2024-03-26 22:53:52','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=208474 +; + +-- Mar 27, 2024, 12:23:08 PM CET +INSERT INTO AD_Reference (AD_Reference_ID,Name,ValidationType,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,EntityType,IsOrderByValue,AD_Reference_UU,ShowInactive) VALUES (200268,'ImportTemplateType','L',0,0,'Y',TO_TIMESTAMP('2024-03-27 12:23:08','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:23:08','YYYY-MM-DD HH24:MI:SS'),100,'D','N','7926c8b4-2e48-4dd3-9054-8e26245c8128','N') +; + +-- Mar 27, 2024, 12:23:20 PM CET +INSERT INTO AD_Ref_List (AD_Ref_List_ID,Name,AD_Reference_ID,Value,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,EntityType,AD_Ref_List_UU) VALUES (200704,'CSV',200268,'CSV',0,0,'Y',TO_TIMESTAMP('2024-03-27 12:23:20','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:23:20','YYYY-MM-DD HH24:MI:SS'),100,'D','7259e211-5477-4f1b-b2a3-1ce67f919749') +; + +-- Mar 27, 2024, 12:23:26 PM CET +INSERT INTO AD_Ref_List (AD_Ref_List_ID,Name,AD_Reference_ID,Value,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,EntityType,AD_Ref_List_UU) VALUES (200705,'XLS',200268,'XLS',0,0,'Y',TO_TIMESTAMP('2024-03-27 12:23:26','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:23:26','YYYY-MM-DD HH24:MI:SS'),100,'D','4c979a06-6550-4763-9cd2-066c0d897b9c') +; + +-- Mar 27, 2024, 12:23:34 PM CET +INSERT INTO AD_Ref_List (AD_Ref_List_ID,Name,AD_Reference_ID,Value,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,EntityType,AD_Ref_List_UU) VALUES (200706,'XLSX',200268,'XLSX',0,0,'Y',TO_TIMESTAMP('2024-03-27 12:23:33','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:23:33','YYYY-MM-DD HH24:MI:SS'),100,'D','0f46a993-00ac-4572-935c-0590629d092b') +; + +-- Mar 27, 2024, 12:23:54 PM CET +INSERT INTO AD_Element (AD_Element_ID,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,ColumnName,Name,Description,Help,PrintName,EntityType,AD_Element_UU) VALUES (203929,0,0,'Y',TO_TIMESTAMP('2024-03-27 12:23:42','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:23:42','YYYY-MM-DD HH24:MI:SS'),100,'ImportTemplateType','Import Template Type',NULL,NULL,'Import Template Type','D','6855e8ac-5c66-428f-84e9-dd81002aa03e') +; + +-- Mar 27, 2024, 12:24:26 PM CET +INSERT INTO AD_Column (AD_Column_ID,Version,Name,AD_Table_ID,ColumnName,DefaultValue,FieldLength,IsKey,IsParent,IsMandatory,IsTranslated,IsIdentifier,SeqNo,IsEncrypted,AD_Reference_ID,AD_Reference_Value_ID,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Element_ID,IsUpdateable,IsSelectionColumn,EntityType,IsSyncDatabase,IsAlwaysUpdateable,IsAutocomplete,IsAllowLogging,AD_Column_UU,IsAllowCopy,SeqNoSelection,IsToolbarButton,IsSecure,IsHtml,IsPartitionKey) VALUES (216586,0,'Import Template Type',200153,'ImportTemplateType','CSV',4,'N','N','Y','N','N',0,'N',17,200268,0,0,'Y',TO_TIMESTAMP('2024-03-27 12:24:26','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:24:26','YYYY-MM-DD HH24:MI:SS'),100,203929,'Y','N','D','N','N','N','Y','4a5dce57-eb09-4bbc-b2d7-1e09d49eb7f3','Y',0,'N','N','N','N') +; + +-- Mar 27, 2024, 12:24:30 PM CET +ALTER TABLE AD_ImportTemplate ADD ImportTemplateType VARCHAR2(4 CHAR) DEFAULT 'CSV' NOT NULL +; + +-- Mar 27, 2024, 12:24:41 PM CET +INSERT INTO AD_Field (AD_Field_ID,Name,AD_Tab_ID,AD_Column_ID,IsDisplayed,DisplayLength,SeqNo,IsSameLine,IsHeading,IsFieldOnly,IsEncrypted,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,IsReadOnly,IsCentrallyMaintained,EntityType,AD_Field_UU,IsDisplayedGrid,SeqNoGrid,ColumnSpan) VALUES (208476,'Import Template Type',200167,216586,'Y',4,130,'N','N','N','N',0,0,'Y',TO_TIMESTAMP('2024-03-27 12:24:40','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:24:40','YYYY-MM-DD HH24:MI:SS'),100,'N','Y','D','88a997fd-241c-429c-a5fe-9f64d71c2a3a','Y',130,2) +; + +-- Mar 27, 2024, 12:25:34 PM CET +UPDATE AD_Field SET IsDisplayed='Y', SeqNo=120, XPosition=1, ColumnSpan=1,Updated=TO_TIMESTAMP('2024-03-27 12:25:34','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=208476 +; + +-- Mar 27, 2024, 12:25:34 PM CET +UPDATE AD_Field SET IsDisplayed='Y', SeqNo=130, XPosition=5,Updated=TO_TIMESTAMP('2024-03-27 12:25:34','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=203458 +; + +-- Mar 27, 2024, 12:26:14 PM CET +INSERT INTO AD_Message (MsgType,MsgText,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Message_ID,Value,EntityType,AD_Message_UU) VALUES ('E','There was an error converting the Excel file to CSV',0,0,'Y',TO_TIMESTAMP('2024-03-27 12:26:13','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:26:13','YYYY-MM-DD HH24:MI:SS'),100,200878,'ErrorConvertingXlsToCsv','D','e36c4d43-93f8-4b45-bfcc-ec4e7fba72cd') +; + +-- Mar 27, 2024, 9:44:59 PM CET +INSERT INTO AD_Message (MsgType,MsgText,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Message_ID,Value,EntityType,AD_Message_UU) VALUES ('E','CSV Header Column not found -> {0}',0,0,'Y',TO_TIMESTAMP('2024-03-27 21:44:59','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 21:44:59','YYYY-MM-DD HH24:MI:SS'),100,200879,'CSVHeaderColumnNotFound','D','f3fdb148-1a0a-4c62-b8fa-55ab40e0e434') +; + diff --git a/migration/iD11/postgresql/202403262247_IDEMPIERE-6040.sql b/migration/iD11/postgresql/202403262247_IDEMPIERE-6040.sql new file mode 100644 index 0000000000..e9e551d322 --- /dev/null +++ b/migration/iD11/postgresql/202403262247_IDEMPIERE-6040.sql @@ -0,0 +1,107 @@ +-- IDEMPIERE-6040 Improvements for CSV import template +SELECT register_migration_script('202403262247_IDEMPIERE-6040.sql') FROM dual; + +-- Mar 26, 2024, 10:47:25 PM CET +INSERT INTO AD_Column (AD_Column_ID,Version,Name,AD_Table_ID,ColumnName,DefaultValue,FieldLength,IsKey,IsParent,IsMandatory,IsTranslated,IsIdentifier,SeqNo,IsEncrypted,AD_Reference_ID,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Element_ID,IsUpdateable,IsSelectionColumn,EntityType,IsSyncDatabase,IsAlwaysUpdateable,IsAutocomplete,IsAllowLogging,AD_Column_UU,IsAllowCopy,SeqNoSelection,IsToolbarButton,IsSecure,IsHtml,IsPartitionKey) VALUES (216584,0,'Separator Character',200153,'SeparatorChar',',',1,'N','N','Y','N','N',0,'N',10,0,0,'Y',TO_TIMESTAMP('2024-03-26 22:47:24','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-26 22:47:24','YYYY-MM-DD HH24:MI:SS'),100,54158,'Y','N','D','N','N','N','Y','3083364e-7dac-4ee2-8fa9-7ffe047b0461','Y',0,'N','N','N','N') +; + +-- Mar 26, 2024, 10:47:28 PM CET +ALTER TABLE AD_ImportTemplate ADD COLUMN SeparatorChar VARCHAR(1) DEFAULT ',' NOT NULL +; + +-- Mar 26, 2024, 10:48:07 PM CET +INSERT INTO AD_Element (AD_Element_ID,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,ColumnName,Name,Description,Help,PrintName,EntityType,AD_Element_UU) VALUES (203928,0,0,'Y',TO_TIMESTAMP('2024-03-26 22:47:56','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-26 22:47:56','YYYY-MM-DD HH24:MI:SS'),100,'QuoteChar','Quote Char',NULL,NULL,'Quote Char','D','9bdf8508-f83b-46bc-baca-c27d5b1fac5e') +; + +-- Mar 26, 2024, 10:48:24 PM CET +INSERT INTO AD_Column (AD_Column_ID,Version,Name,AD_Table_ID,ColumnName,DefaultValue,FieldLength,IsKey,IsParent,IsMandatory,IsTranslated,IsIdentifier,SeqNo,IsEncrypted,AD_Reference_ID,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Element_ID,IsUpdateable,IsSelectionColumn,EntityType,IsSyncDatabase,IsAlwaysUpdateable,IsAutocomplete,IsAllowLogging,AD_Column_UU,IsAllowCopy,SeqNoSelection,IsToolbarButton,IsSecure,IsHtml,IsPartitionKey) VALUES (216585,0,'Quote Char',200153,'QuoteChar','"',1,'N','N','Y','N','N',0,'N',10,0,0,'Y',TO_TIMESTAMP('2024-03-26 22:48:23','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-26 22:48:23','YYYY-MM-DD HH24:MI:SS'),100,203928,'Y','N','D','N','N','N','Y','9b51ff6c-2c9c-4f1b-ad66-a8fa4e0b1f1b','Y',0,'N','N','N','N') +; + +-- Mar 26, 2024, 10:48:27 PM CET +ALTER TABLE AD_ImportTemplate ADD COLUMN QuoteChar VARCHAR(1) DEFAULT '"' NOT NULL +; + +-- Mar 26, 2024, 10:48:44 PM CET +INSERT INTO AD_Field (AD_Field_ID,Name,AD_Tab_ID,AD_Column_ID,IsDisplayed,DisplayLength,SeqNo,IsSameLine,IsHeading,IsFieldOnly,IsEncrypted,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,IsReadOnly,IsCentrallyMaintained,EntityType,AD_Field_UU,IsDisplayedGrid,SeqNoGrid,ColumnSpan) VALUES (208474,'Separator Character',200167,216584,'Y',1,110,'N','N','N','N',0,0,'Y',TO_TIMESTAMP('2024-03-26 22:48:44','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-26 22:48:44','YYYY-MM-DD HH24:MI:SS'),100,'N','Y','D','ada7dc92-28b4-4887-a577-d49226450b6c','Y',110,2) +; + +-- Mar 26, 2024, 10:48:45 PM CET +INSERT INTO AD_Field (AD_Field_ID,Name,AD_Tab_ID,AD_Column_ID,IsDisplayed,DisplayLength,SeqNo,IsSameLine,IsHeading,IsFieldOnly,IsEncrypted,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,IsReadOnly,IsCentrallyMaintained,EntityType,AD_Field_UU,IsDisplayedGrid,SeqNoGrid,ColumnSpan) VALUES (208475,'Quote Char',200167,216585,'Y',1,120,'N','N','N','N',0,0,'Y',TO_TIMESTAMP('2024-03-26 22:48:44','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-26 22:48:44','YYYY-MM-DD HH24:MI:SS'),100,'N','Y','D','6f9a0de7-2df9-46b2-a491-a450f61d594e','Y',120,2) +; + +-- Mar 26, 2024, 10:49:29 PM CET +UPDATE AD_Field SET SeqNo=70, ColumnSpan=1,Updated=TO_TIMESTAMP('2024-03-26 22:49:29','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=203455 +; + +-- Mar 26, 2024, 10:49:29 PM CET +UPDATE AD_Field SET IsDisplayed='Y', SeqNo=80, XPosition=3, ColumnSpan=1,Updated=TO_TIMESTAMP('2024-03-26 22:49:29','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=208474 +; + +-- Mar 26, 2024, 10:49:29 PM CET +UPDATE AD_Field SET IsDisplayed='Y', SeqNo=90, XPosition=5, ColumnSpan=1,Updated=TO_TIMESTAMP('2024-03-26 22:49:29','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=208475 +; + +-- Mar 26, 2024, 10:49:29 PM CET +UPDATE AD_Field SET SeqNo=100,Updated=TO_TIMESTAMP('2024-03-26 22:49:29','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=203456 +; + +-- Mar 26, 2024, 10:49:29 PM CET +UPDATE AD_Field SET SeqNo=110,Updated=TO_TIMESTAMP('2024-03-26 22:49:29','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=203457 +; + +-- Mar 26, 2024, 10:49:29 PM CET +UPDATE AD_Field SET SeqNo=120,Updated=TO_TIMESTAMP('2024-03-26 22:49:29','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=203458 +; + +-- Mar 26, 2024, 10:53:52 PM CET +UPDATE AD_Field SET DefaultValue='@SQL=SELECT '','' FROM Dual',Updated=TO_TIMESTAMP('2024-03-26 22:53:52','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=208474 +; + +-- Mar 27, 2024, 12:23:08 PM CET +INSERT INTO AD_Reference (AD_Reference_ID,Name,ValidationType,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,EntityType,IsOrderByValue,AD_Reference_UU,ShowInactive) VALUES (200268,'ImportTemplateType','L',0,0,'Y',TO_TIMESTAMP('2024-03-27 12:23:08','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:23:08','YYYY-MM-DD HH24:MI:SS'),100,'D','N','7926c8b4-2e48-4dd3-9054-8e26245c8128','N') +; + +-- Mar 27, 2024, 12:23:20 PM CET +INSERT INTO AD_Ref_List (AD_Ref_List_ID,Name,AD_Reference_ID,Value,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,EntityType,AD_Ref_List_UU) VALUES (200704,'CSV',200268,'CSV',0,0,'Y',TO_TIMESTAMP('2024-03-27 12:23:20','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:23:20','YYYY-MM-DD HH24:MI:SS'),100,'D','7259e211-5477-4f1b-b2a3-1ce67f919749') +; + +-- Mar 27, 2024, 12:23:26 PM CET +INSERT INTO AD_Ref_List (AD_Ref_List_ID,Name,AD_Reference_ID,Value,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,EntityType,AD_Ref_List_UU) VALUES (200705,'XLS',200268,'XLS',0,0,'Y',TO_TIMESTAMP('2024-03-27 12:23:26','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:23:26','YYYY-MM-DD HH24:MI:SS'),100,'D','4c979a06-6550-4763-9cd2-066c0d897b9c') +; + +-- Mar 27, 2024, 12:23:34 PM CET +INSERT INTO AD_Ref_List (AD_Ref_List_ID,Name,AD_Reference_ID,Value,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,EntityType,AD_Ref_List_UU) VALUES (200706,'XLSX',200268,'XLSX',0,0,'Y',TO_TIMESTAMP('2024-03-27 12:23:33','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:23:33','YYYY-MM-DD HH24:MI:SS'),100,'D','0f46a993-00ac-4572-935c-0590629d092b') +; + +-- Mar 27, 2024, 12:23:54 PM CET +INSERT INTO AD_Element (AD_Element_ID,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,ColumnName,Name,Description,Help,PrintName,EntityType,AD_Element_UU) VALUES (203929,0,0,'Y',TO_TIMESTAMP('2024-03-27 12:23:42','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:23:42','YYYY-MM-DD HH24:MI:SS'),100,'ImportTemplateType','Import Template Type',NULL,NULL,'Import Template Type','D','6855e8ac-5c66-428f-84e9-dd81002aa03e') +; + +-- Mar 27, 2024, 12:24:26 PM CET +INSERT INTO AD_Column (AD_Column_ID,Version,Name,AD_Table_ID,ColumnName,DefaultValue,FieldLength,IsKey,IsParent,IsMandatory,IsTranslated,IsIdentifier,SeqNo,IsEncrypted,AD_Reference_ID,AD_Reference_Value_ID,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Element_ID,IsUpdateable,IsSelectionColumn,EntityType,IsSyncDatabase,IsAlwaysUpdateable,IsAutocomplete,IsAllowLogging,AD_Column_UU,IsAllowCopy,SeqNoSelection,IsToolbarButton,IsSecure,IsHtml,IsPartitionKey) VALUES (216586,0,'Import Template Type',200153,'ImportTemplateType','CSV',4,'N','N','Y','N','N',0,'N',17,200268,0,0,'Y',TO_TIMESTAMP('2024-03-27 12:24:26','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:24:26','YYYY-MM-DD HH24:MI:SS'),100,203929,'Y','N','D','N','N','N','Y','4a5dce57-eb09-4bbc-b2d7-1e09d49eb7f3','Y',0,'N','N','N','N') +; + +-- Mar 27, 2024, 12:24:30 PM CET +ALTER TABLE AD_ImportTemplate ADD COLUMN ImportTemplateType VARCHAR(4) DEFAULT 'CSV' NOT NULL +; + +-- Mar 27, 2024, 12:24:41 PM CET +INSERT INTO AD_Field (AD_Field_ID,Name,AD_Tab_ID,AD_Column_ID,IsDisplayed,DisplayLength,SeqNo,IsSameLine,IsHeading,IsFieldOnly,IsEncrypted,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,IsReadOnly,IsCentrallyMaintained,EntityType,AD_Field_UU,IsDisplayedGrid,SeqNoGrid,ColumnSpan) VALUES (208476,'Import Template Type',200167,216586,'Y',4,130,'N','N','N','N',0,0,'Y',TO_TIMESTAMP('2024-03-27 12:24:40','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:24:40','YYYY-MM-DD HH24:MI:SS'),100,'N','Y','D','88a997fd-241c-429c-a5fe-9f64d71c2a3a','Y',130,2) +; + +-- Mar 27, 2024, 12:25:34 PM CET +UPDATE AD_Field SET IsDisplayed='Y', SeqNo=120, XPosition=1, ColumnSpan=1,Updated=TO_TIMESTAMP('2024-03-27 12:25:34','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=208476 +; + +-- Mar 27, 2024, 12:25:34 PM CET +UPDATE AD_Field SET IsDisplayed='Y', SeqNo=130, XPosition=5,Updated=TO_TIMESTAMP('2024-03-27 12:25:34','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=203458 +; + +-- Mar 27, 2024, 12:26:14 PM CET +INSERT INTO AD_Message (MsgType,MsgText,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Message_ID,Value,EntityType,AD_Message_UU) VALUES ('E','There was an error converting the Excel file to CSV',0,0,'Y',TO_TIMESTAMP('2024-03-27 12:26:13','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:26:13','YYYY-MM-DD HH24:MI:SS'),100,200878,'ErrorConvertingXlsToCsv','D','e36c4d43-93f8-4b45-bfcc-ec4e7fba72cd') +; + +-- Mar 27, 2024, 9:44:59 PM CET +INSERT INTO AD_Message (MsgType,MsgText,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Message_ID,Value,EntityType,AD_Message_UU) VALUES ('E','CSV Header Column not found -> {0}',0,0,'Y',TO_TIMESTAMP('2024-03-27 21:44:59','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 21:44:59','YYYY-MM-DD HH24:MI:SS'),100,200879,'CSVHeaderColumnNotFound','D','f3fdb148-1a0a-4c62-b8fa-55ab40e0e434') +; + diff --git a/org.adempiere.base.process/src/org/idempiere/process/ImportCSVProcess.java b/org.adempiere.base.process/src/org/idempiere/process/ImportCSVProcess.java index 2ca4b50fc4..cc615409f1 100644 --- a/org.adempiere.base.process/src/org/idempiere/process/ImportCSVProcess.java +++ b/org.adempiere.base.process/src/org/idempiere/process/ImportCSVProcess.java @@ -147,7 +147,7 @@ public class ImportCSVProcess extends SvrProcess implements DataStatusListener { m_file_istream = new FileInputStream(filePath); m_file_istream = m_importTemplate.validateFile(m_file_istream); - File outFile = csvImporter.fileImport(activeTab, childTabs, m_file_istream, Charset.forName(m_importTemplate.getCharacterSet()), p_ImportMode, processUI); + File outFile = csvImporter.fileImport(activeTab, childTabs, m_file_istream, Charset.forName(m_importTemplate.getCharacterSet()), p_ImportMode, m_importTemplate.getSeparatorChar(), m_importTemplate.getQuoteChar(), processUI); // TODO: Potential improvement - traverse the outFile and call addLog with the results if (processUI != null) diff --git a/org.adempiere.base/src/org/adempiere/base/IGridTabImporter.java b/org.adempiere.base/src/org/adempiere/base/IGridTabImporter.java index 2827e513d7..b65457b80c 100644 --- a/org.adempiere.base/src/org/adempiere/base/IGridTabImporter.java +++ b/org.adempiere.base/src/org/adempiere/base/IGridTabImporter.java @@ -49,6 +49,20 @@ public interface IGridTabImporter { */ public File fileImport(GridTab gridTab, List childs, InputStream filestream, Charset charset, String importMode, IProcessUI processUI); + /** + * Import data from filestream to gridTab + * @param gridTab + * @param childs + * @param filestream + * @param charset + * @param importMode + * @param delimiterChar + * @param quoteChar + * @param processUI + * @return File for import log + */ + public File fileImport(GridTab gridTab, List childs, InputStream filestream, Charset charset, String importMode, String delimiterChar, String quoteChar, IProcessUI processUI); + /** * @return file extension */ diff --git a/org.adempiere.base/src/org/adempiere/impexp/GridTabCSVImporter.java b/org.adempiere.base/src/org/adempiere/impexp/GridTabCSVImporter.java index 59ece79133..3d0e22d5f7 100644 --- a/org.adempiere.base/src/org/adempiere/impexp/GridTabCSVImporter.java +++ b/org.adempiere.base/src/org/adempiere/impexp/GridTabCSVImporter.java @@ -116,11 +116,10 @@ public class GridTabCSVImporter implements IGridTabImporter private File logFile; private PrintWriter errFileW; private PrintWriter logFileW; - - private CsvPreference csvpref = CsvPreference.STANDARD_PREFERENCE; - private String delimiter = String.valueOf((char) csvpref.getDelimiterChar()); - private String quoteChar = String.valueOf((char) csvpref.getQuoteChar()); - + + private String delimiterChar = ","; + private String quoteChar = "\""; + //Trx private Trx trx; private String trxName; @@ -131,11 +130,16 @@ public class GridTabCSVImporter implements IGridTabImporter @Override public File fileImport(GridTab gridTab, List childs, InputStream filestream, Charset charset , String importMode) { - return fileImport(gridTab, childs, filestream, charset, importMode, null); + return fileImport(gridTab, childs, filestream, charset, importMode, null, null, null); }//fileImport @Override public File fileImport(GridTab gridTab, List childs, InputStream filestream, Charset charset, String importMode, IProcessUI processUI) { + return fileImport(gridTab, childs, filestream, charset, importMode, null, null, processUI); + } + + @Override + public File fileImport(GridTab gridTab, List childs, InputStream filestream, Charset charset, String importMode, String p_delimiterChar, String p_quoteChar, IProcessUI processUI) { if(!gridTab.isInsertRecord() && isInsertMode()) throwAdempiereException("Insert record disabled for Tab"); @@ -146,6 +150,13 @@ public class GridTabCSVImporter implements IGridTabImporter m_import_mode = importMode; errFile = new File(errFileName); errFileW = new PrintWriter(errFile, charset.name()); + + if (p_delimiterChar != null) + delimiterChar = p_delimiterChar; + if (p_quoteChar != null) + quoteChar = p_quoteChar; + CsvPreference csvpref = new CsvPreference.Builder(quoteChar.charAt(0), delimiterChar.charAt(0), "\r\n" /* ignored */).build(); + mapReader = new CsvMapReader(new InputStreamReader(filestream, charset), csvpref); header = Arrays.asList(mapReader.getHeader(true)); @@ -178,7 +189,7 @@ public class GridTabCSVImporter implements IGridTabImporter m_isError = false; // write the header String rawHeader = mapReader.getUntokenizedRow(); - errFileW.write(rawHeader + delimiter + ERROR_HEADER + "\n"); + errFileW.write(rawHeader + delimiterChar + ERROR_HEADER + "\n"); data = new ArrayList>(); rawData = new ArrayList(); @@ -191,7 +202,7 @@ public class GridTabCSVImporter implements IGridTabImporter logFile = new File(logFileName); logFileW = new PrintWriter(logFile, charset.name()); // write the header - logFileW.write(rawHeader + delimiter + LOG_HEADER + "\n"); + logFileW.write(rawHeader + delimiterChar + LOG_HEADER + "\n"); // no errors found - process header and then details isMasterok = true; isDetailok = true; @@ -226,11 +237,11 @@ public class GridTabCSVImporter implements IGridTabImporter } if (!isMasterok && isDetail){ - rawLine = rawLine + delimiter + quoteChar + Msg.getMsg(Env.getCtx(),"NotProcessed") + quoteChar + "\n"; + rawLine = rawLine + delimiterChar + quoteChar + Msg.getMsg(Env.getCtx(),"NotProcessed") + quoteChar + "\n"; rowsTmpResult.add(rawLine); continue; }else if(isMasterok && isDetail && !isDetailok){ - rawLine = rawLine + delimiter + quoteChar + "Record not processed due to detail record failure" + quoteChar + "\n"; + rawLine = rawLine + delimiterChar + quoteChar + "Record not processed due to detail record failure" + quoteChar + "\n"; rowsTmpResult.add(rawLine); continue; } @@ -249,7 +260,7 @@ public class GridTabCSVImporter implements IGridTabImporter rowResult.append(recordResult); // write - rawLine = rawLine + delimiter + quoteChar + rowResult.toString().replaceAll(delimiter, "") + quoteChar + "\n"; + rawLine = rawLine + delimiterChar + quoteChar + rowResult.toString().replaceAll(delimiterChar, "") + quoteChar + "\n"; rowsTmpResult.add(rawLine); if( isSingleTrx() && isError() ) @@ -530,7 +541,7 @@ public class GridTabCSVImporter implements IGridTabImporter rawData.add(rawLine); } // write - rawLine = rawLine + delimiter + quoteChar + errMsg.toString().replaceAll(quoteChar, "") + quoteChar + "\n"; + rawLine = rawLine + delimiterChar + quoteChar + errMsg.toString().replaceAll(quoteChar, "") + quoteChar + "\n"; errFileW.write(rawLine); } }//preProcess @@ -1747,4 +1758,5 @@ public class GridTabCSVImporter implements IGridTabImporter return -1; } } + } diff --git a/org.adempiere.base/src/org/compiere/model/I_AD_ImportTemplate.java b/org.adempiere.base/src/org/compiere/model/I_AD_ImportTemplate.java index b5926b7244..daa56bd6f7 100644 --- a/org.adempiere.base/src/org/compiere/model/I_AD_ImportTemplate.java +++ b/org.adempiere.base/src/org/compiere/model/I_AD_ImportTemplate.java @@ -166,6 +166,15 @@ public interface I_AD_ImportTemplate */ public String getDescription(); + /** Column name ImportTemplateType */ + public static final String COLUMNNAME_ImportTemplateType = "ImportTemplateType"; + + /** Set Import Template Type */ + public void setImportTemplateType (String ImportTemplateType); + + /** Get Import Template Type */ + public String getImportTemplateType(); + /** Column name IsActive */ public static final String COLUMNNAME_IsActive = "IsActive"; @@ -192,6 +201,24 @@ public interface I_AD_ImportTemplate */ public String getName(); + /** Column name QuoteChar */ + public static final String COLUMNNAME_QuoteChar = "QuoteChar"; + + /** Set Quote Char */ + public void setQuoteChar (String QuoteChar); + + /** Get Quote Char */ + public String getQuoteChar(); + + /** Column name SeparatorChar */ + public static final String COLUMNNAME_SeparatorChar = "SeparatorChar"; + + /** Set Separator Character */ + public void setSeparatorChar (String SeparatorChar); + + /** Get Separator Character */ + public String getSeparatorChar(); + /** Column name Updated */ public static final String COLUMNNAME_Updated = "Updated"; diff --git a/org.adempiere.base/src/org/compiere/model/MImportTemplate.java b/org.adempiere.base/src/org/compiere/model/MImportTemplate.java index 6e6e4f7e8f..0247ce3c2f 100644 --- a/org.adempiere.base/src/org/compiere/model/MImportTemplate.java +++ b/org.adempiere.base/src/org/compiere/model/MImportTemplate.java @@ -23,13 +23,26 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.nio.charset.Charset; import java.sql.ResultSet; +import java.text.DecimalFormat; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; import java.util.List; import java.util.Properties; +import java.util.logging.Level; import org.adempiere.exceptions.AdempiereException; +import org.apache.poi.hssf.usermodel.HSSFWorkbookFactory; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.CellType; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.Workbook; +import org.apache.poi.xssf.usermodel.XSSFWorkbookFactory; import org.compiere.util.CCache; import org.compiere.util.CLogger; import org.compiere.util.DB; +import org.compiere.util.DisplayType; import org.compiere.util.Env; import org.compiere.util.Msg; import org.idempiere.cache.ImmutablePOSupport; @@ -41,11 +54,11 @@ import org.idempiere.cache.ImmutablePOSupport; */ public class MImportTemplate extends X_AD_ImportTemplate implements ImmutablePOSupport { - /** - * generated serial id + /** + * */ - private static final long serialVersionUID = -4681075469110529774L; - + private static final long serialVersionUID = 7751951143524784901L; + /** Logger */ @SuppressWarnings("unused") private static CLogger s_log = CLogger.getCLogger(MImportTemplate.class); @@ -123,6 +136,8 @@ public class MImportTemplate extends X_AD_ImportTemplate implements ImmutablePOS log.saveError("Error", Msg.parseTranslation(getCtx(), "@Invalid@ @CharacterSet@")); return false; } + if (is_new() || is_ValueChanged(COLUMNNAME_CSVHeader) || is_ValueChanged(COLUMNNAME_AD_Tab_ID)) + calculateColumnTypes(); // this throws an Exception if there are wrong columns in the CSV Header return super.beforeSave(newRecord); } @@ -196,35 +211,51 @@ public class MImportTemplate extends X_AD_ImportTemplate implements ImmutablePOS int cnt = DB.getSQLValueEx(get_TrxName(), sql.toString(), getAD_ImportTemplate_ID(), roleID, roleID); return cnt > 0; } - + /** - * Validate that InputStream header is CSVHeader or AliasCSVHeader.
- * If the header is AliasCSVHeader it replaces it with the CSVHeader so it can be - * processed. + * Byte Order Mark character for UTF-16BE + */ + public static final String UTF16BE_BOM = "\uFEFF"; + + /** + * - If needed preProcess the file when an Excel file and generate a CSV file, then
+ * - Validate that InputStream header is CSVHeader or AliasCSVHeader.
+ * - If the header is AliasCSVHeader it replaces it with the CSVHeader so it can be processed.
* @param in input file * @return InputStream with the CSVHeader that can be processed by CsvMapReader */ public InputStream validateFile(InputStream in) { - // because the input stream cannot be reset we need to copy here the file to a new one (replacing the header if it's the alias) + + if ( MImportTemplate.IMPORTTEMPLATETYPE_XLS.equals(getImportTemplateType()) + || MImportTemplate.IMPORTTEMPLATETYPE_XLSX.equals(getImportTemplateType())) { + try { + in = convertExcelToCSV(in); + } catch (Exception e) { + throw new AdempiereException(Msg.getMsg(Env.getCtx(), "ErrorConvertingXlsToCsv") + " -> " + e.getLocalizedMessage(), e); + } + } + Charset charset = Charset.forName(getCharacterSet()); BufferedReader reader = new BufferedReader(new InputStreamReader(in, charset)); File tmpfile = null; InputStream is = null; BufferedWriter bw = null; try { - tmpfile = File.createTempFile("CSVImportAction", "csv"); - bw = new BufferedWriter(new FileWriter(tmpfile,charset)); String firstLine = null; String line = null; while ((line = reader.readLine()) != null) { if (firstLine == null) { firstLine = line; - /* Validate that m_file_istream header is CSVHeader or AliasCSVHeader */ + if (firstLine.startsWith(UTF16BE_BOM)) + firstLine = firstLine.substring(1); + /* Validate that file header is CSVHeader or AliasCSVHeader */ if ( firstLine.equals(getCSVHeader()) || firstLine.equals(getCSVAliasHeader())) { + // we copy here the file to a new one (replacing the header if it's the alias) + tmpfile = File.createTempFile("CSVImportAction", ".csv"); + bw = new BufferedWriter(new FileWriter(tmpfile,charset)); bw.write(getCSVHeader()); } else { - reader.close(); throw new AdempiereException(Msg.getMsg(Env.getCtx(), "WrongCSVHeader")); } } else { @@ -257,6 +288,231 @@ public class MImportTemplate extends X_AD_ImportTemplate implements ImmutablePOS return is; } + /** + * Convert an Excel (XLS or XLSX) file to CSV + * @param excelIs input stream containing XLS/XLSX + * @return input stream containing CSV + * @throws IOException + */ + public InputStream convertExcelToCSV(InputStream excelIs) throws IOException { + + Workbook workbook = null; + if (MImportTemplate.IMPORTTEMPLATETYPE_XLS.equals(getImportTemplateType())) { + HSSFWorkbookFactory xlsWbf = new HSSFWorkbookFactory(); + workbook = xlsWbf.create(excelIs); + } else if (MImportTemplate.IMPORTTEMPLATETYPE_XLSX.equals(getImportTemplateType())) { + XSSFWorkbookFactory xlsxWbf = new XSSFWorkbookFactory(); + workbook = xlsxWbf.create(excelIs); + } else { + // unexpected error + throw new AdempiereException("Wrong template type -> " + getImportTemplateType()); + } + + List colTypes = calculateColumnTypes(); + + Sheet sheet = workbook.getSheetAt(0); // First sheet + + File tmpfile = File.createTempFile("CSVImportActionConvert", ".csv"); + Charset charset = Charset.forName(getCharacterSet()); + try (BufferedWriter bw = new BufferedWriter(new FileWriter(tmpfile, charset))) { + for (Row row : sheet) { + boolean firstCell = true; + for (int cn = 0; cn < colTypes.size(); cn++) { + int displayType = colTypes.get(cn); + Cell cell = row.getCell(cn, Row.MissingCellPolicy.CREATE_NULL_AS_BLANK); + if (firstCell) { + firstCell = false; + } else { + bw.append(getSeparatorChar()); + } + + CellType cellType = cell.getCellType(); + if (cellType.equals(CellType.FORMULA)) + cellType = cell.getCachedFormulaResultType(); + if (! ( cellType.equals(CellType.BLANK) + || cellType.equals(CellType.STRING) + || cellType.equals(CellType.BOOLEAN) + || cellType.equals(CellType.NUMERIC))) + throw new IllegalStateException("Unsupported cell type: " + cell.getCellType()); + if (! cellType.equals(CellType.BLANK)) { // blank cells are not processed - write them just as empty strings + + if (cellType.equals(CellType.BOOLEAN)) { + // boolean cells are converted to Y/N iDempiere notation + if (cell.getBooleanCellValue()) + bw.append("Y"); + else + bw.append("N"); + + } else if (DisplayType.YesNo == displayType) { + // a Yes/No value is expected, try to convert true/false and 1/0 to Y/N + if (cellType.equals(CellType.NUMERIC)) { + double doubleValue = cell.getNumericCellValue(); + if (doubleValue == 1.0) + bw.append("Y"); + else if (doubleValue == 0.0) + bw.append("N"); + else + addNumeric(bw, doubleValue, displayType); + } else { // STRING + String stringValue = cell.getStringCellValue(); + if (stringValue.equalsIgnoreCase("true") || stringValue.equals("y")) + stringValue = "Y"; + else if (stringValue.equalsIgnoreCase("false") || stringValue.equals("n")) + stringValue = "N"; + addString(bw, stringValue); + } + + } else if (DisplayType.isDate(displayType)) { + // a date time is expected + if (cellType.equals(CellType.NUMERIC)) { + SimpleDateFormat sdf; + if (displayType == DisplayType.Date) + sdf = DisplayType.getDateFormat_JDBC(); + else + sdf = DisplayType.getTimestampFormat_Default(); + Date date = cell.getDateCellValue(); + addString(bw, sdf.format(date)); + } else { // STRING + addString(bw, cell.getStringCellValue()); + } + + } else { + if (cellType.equals(CellType.NUMERIC)) { + addNumeric(bw, cell.getNumericCellValue(), displayType); + } else { // STRING + addString(bw, cell.getStringCellValue()); + } + + } + } + + } + bw.newLine(); + } + } + workbook.close(); + InputStream is = new FileInputStream(tmpfile); + return is; + } + + /** + * Add a String to the BufferedWriter + * @param bw BufferedWriter + * @param value String + * @throws IOException + */ + private void addString(BufferedWriter bw, String value) throws IOException { + boolean addQuotes = (value.contains(getQuoteChar()) || value.contains(getSeparatorChar())); + if (addQuotes) + bw.append(getQuoteChar()); + bw.append(value.replace(getQuoteChar(), getQuoteChar().concat(getQuoteChar()))); + if (addQuotes) + bw.append(getQuoteChar()); + } + + /** + * Add a Numeric value to the BufferedWriter + * @param bw BufferedWriter + * @param doubleValue double + * @param displayType + * @throws IOException + */ + private void addNumeric(BufferedWriter bw, double doubleValue, int displayType) throws IOException { + int intValue = (int) doubleValue; + DecimalFormat df = DisplayType.getNumberFormat(displayType); + if ( intValue == doubleValue + || displayType == DisplayType.Integer + || displayType == DisplayType.Table + || displayType == DisplayType.TableDir + || displayType == DisplayType.Search + || displayType == DisplayType.RecordID) { + df.setParseIntegerOnly(true); + df.setMaximumFractionDigits(0); + } + df.setGroupingSize(0); + String str = df.format(doubleValue); + addString(bw, str); + } + + /** + * Calculate the expected column types depending on the columns defined in the CSVHeader + * Every column can have the format + * Column + * Column[ForeignColumn] + * DetailTableName>Column + * DetailTableName>Column[ForeignColumn] + * Any column can end with /K (can be ignored) + * @return List of expected DisplayType for every column + */ + private List calculateColumnTypes() { + List retValue = new ArrayList(); + String[] csvHeaders = getCSVHeader().split(getSeparatorChar()); + MTab mainTab = MTab.get(getAD_Tab_ID()); + MTable mainTable = MTable.get(mainTab.getAD_Table_ID()); + for (String csvHeader : csvHeaders) { + String columnHeader = csvHeader; + // first remove the /K mark + if (columnHeader.endsWith("/K")) + columnHeader = columnHeader.substring(0, columnHeader.length()-2); + // check if there is a foreign column that defines the type + String foreignColumnName = null; + int idxOpen = columnHeader.indexOf("["); + if (idxOpen > 0) { + int idxClose = columnHeader.indexOf("]"); + if (idxClose > 0 && idxClose > idxOpen) { + foreignColumnName = columnHeader.substring(idxOpen+1, idxClose); + columnHeader = columnHeader.substring(0, idxOpen); + } + } + // if this is from a detail table + int idxTableSep = columnHeader.lastIndexOf(">"); + MTable table = null; + String tableName = null; + String columnName = null; + if (idxTableSep > 0) { + tableName = columnHeader.substring(0, idxTableSep); + if (tableName.lastIndexOf(">") > 0) { + tableName = tableName.substring(tableName.lastIndexOf(">")+1); + } + table = MTable.get(getCtx(), tableName); + columnName = columnHeader.substring(idxTableSep+1); + } else { + table = mainTable; + tableName = table.getTableName(); + columnName = columnHeader; + } + if (table == null) + throwCSVHeaderNotFound(csvHeader); + MColumn column = table.getColumn(columnName); + if (column == null) + throwCSVHeaderNotFound(csvHeader); + String foreignTableName = null; + if (foreignColumnName != null) { + foreignTableName = column.getReferenceTableName(); + MTable foreignTable = MTable.get(getCtx(), foreignTableName); + if (foreignTable == null) + throwCSVHeaderNotFound(csvHeader); + column = foreignTable.getColumn(foreignColumnName); + if (column == null) + throwCSVHeaderNotFound(csvHeader); + } + if (log.isLoggable(Level.INFO)) + log.info("CSV Header column found -> " + csvHeader + " -> " + tableName + "." + columnName + + (foreignColumnName != null ? "[" + foreignTableName + "." + foreignColumnName + "]" : "") ); + int ref = column.getAD_Reference_ID(); + retValue.add(ref); + } + return retValue; + } + + /** + * Throws an exception indicating a CSV Header that could not be found + * @param csvHeader + */ + private void throwCSVHeaderNotFound(String csvHeader) { + throw new AdempiereException(Msg.getMsg(getCtx(), "CSVHeaderColumnNotFound", new Object[]{csvHeader})); + } + @Override public MImportTemplate markImmutable() { if (is_Immutable()) diff --git a/org.adempiere.base/src/org/compiere/model/X_AD_ImportTemplate.java b/org.adempiere.base/src/org/compiere/model/X_AD_ImportTemplate.java index 98291f192f..5cc228ef54 100644 --- a/org.adempiere.base/src/org/compiere/model/X_AD_ImportTemplate.java +++ b/org.adempiere.base/src/org/compiere/model/X_AD_ImportTemplate.java @@ -31,7 +31,7 @@ public class X_AD_ImportTemplate extends PO implements I_AD_ImportTemplate, I_Pe /** * */ - private static final long serialVersionUID = 20231222L; + private static final long serialVersionUID = 20240327L; /** Standard Constructor */ public X_AD_ImportTemplate (Properties ctx, int AD_ImportTemplate_ID, String trxName) @@ -44,7 +44,13 @@ public class X_AD_ImportTemplate extends PO implements I_AD_ImportTemplate, I_Pe setAD_Window_ID (0); setCharacterSet (null); setCSVHeader (null); + setImportTemplateType (null); +// CSV setName (null); + setQuoteChar (null); +// " + setSeparatorChar (null); +// , } */ } @@ -59,7 +65,13 @@ public class X_AD_ImportTemplate extends PO implements I_AD_ImportTemplate, I_Pe setAD_Window_ID (0); setCharacterSet (null); setCSVHeader (null); + setImportTemplateType (null); +// CSV setName (null); + setQuoteChar (null); +// " + setSeparatorChar (null); +// , } */ } @@ -74,7 +86,13 @@ public class X_AD_ImportTemplate extends PO implements I_AD_ImportTemplate, I_Pe setAD_Window_ID (0); setCharacterSet (null); setCSVHeader (null); + setImportTemplateType (null); +// CSV setName (null); + setQuoteChar (null); +// " + setSeparatorChar (null); +// , } */ } @@ -89,7 +107,13 @@ public class X_AD_ImportTemplate extends PO implements I_AD_ImportTemplate, I_Pe setAD_Window_ID (0); setCharacterSet (null); setCSVHeader (null); + setImportTemplateType (null); +// CSV setName (null); + setQuoteChar (null); +// " + setSeparatorChar (null); +// , } */ } @@ -274,6 +298,30 @@ public class X_AD_ImportTemplate extends PO implements I_AD_ImportTemplate, I_Pe return (String)get_Value(COLUMNNAME_Description); } + /** ImportTemplateType AD_Reference_ID=200268 */ + public static final int IMPORTTEMPLATETYPE_AD_Reference_ID=200268; + /** CSV = CSV */ + public static final String IMPORTTEMPLATETYPE_CSV = "CSV"; + /** XLS = XLS */ + public static final String IMPORTTEMPLATETYPE_XLS = "XLS"; + /** XLSX = XLSX */ + public static final String IMPORTTEMPLATETYPE_XLSX = "XLSX"; + /** Set Import Template Type. + @param ImportTemplateType Import Template Type + */ + public void setImportTemplateType (String ImportTemplateType) + { + + set_Value (COLUMNNAME_ImportTemplateType, ImportTemplateType); + } + + /** Get Import Template Type. + @return Import Template Type */ + public String getImportTemplateType() + { + return (String)get_Value(COLUMNNAME_ImportTemplateType); + } + /** Set Name. @param Name Alphanumeric identifier of the entity */ @@ -297,4 +345,34 @@ public class X_AD_ImportTemplate extends PO implements I_AD_ImportTemplate, I_Pe { return new KeyNamePair(get_ID(), getName()); } + + /** Set Quote Char. + @param QuoteChar Quote Char + */ + public void setQuoteChar (String QuoteChar) + { + set_Value (COLUMNNAME_QuoteChar, QuoteChar); + } + + /** Get Quote Char. + @return Quote Char */ + public String getQuoteChar() + { + return (String)get_Value(COLUMNNAME_QuoteChar); + } + + /** Set Separator Character. + @param SeparatorChar Separator Character + */ + public void setSeparatorChar (String SeparatorChar) + { + set_Value (COLUMNNAME_SeparatorChar, SeparatorChar); + } + + /** Get Separator Character. + @return Separator Character */ + public String getSeparatorChar() + { + return (String)get_Value(COLUMNNAME_SeparatorChar); + } } \ No newline at end of file diff --git a/org.adempiere.ui.zk/WEB-INF/src/org/adempiere/webui/panel/action/CSVImportAction.java b/org.adempiere.ui.zk/WEB-INF/src/org/adempiere/webui/panel/action/CSVImportAction.java index 85517c49cc..f0405454e3 100644 --- a/org.adempiere.ui.zk/WEB-INF/src/org/adempiere/webui/panel/action/CSVImportAction.java +++ b/org.adempiere.ui.zk/WEB-INF/src/org/adempiere/webui/panel/action/CSVImportAction.java @@ -352,7 +352,7 @@ public class CSVImportAction implements EventListener String iMode = (String)importItem.getValue(); m_file_istream = theTemplate.validateFile(m_file_istream); - File outFile = theCSVImporter.fileImport(panel.getActiveGridTab(), childs, m_file_istream, charset,iMode); + File outFile = theCSVImporter.fileImport(panel.getActiveGridTab(), childs, m_file_istream, charset, iMode, theTemplate.getSeparatorChar(), theTemplate.getQuoteChar(), null); winImportFile.onClose(); winImportFile = null;