midsuit
/
core-jgi
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

IDEMPIERE-6040 Improvements for CSV import template (#2279) 

* IDEMPIERE-6040 Improvements for CSV import template

- Delimiter (Field Separator) configurable
- Quote delimiter configurable
- Fix issue when the UTF-8 file comes with BOM character

* - add support for preprocessing excel files

* - convert Excel to CSV in a proper format according to the column types
This commit is contained in:
Carlos Ruiz 2024-04-01 05:51:47 +02:00
parent af10c3a674
commit d16538ae38
9 changed files with 632 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

110
migration/iD11/oracle/202403262247_IDEMPIERE-6040.sql Normal file
View File

@ -0,0 +1,110 @@
-- IDEMPIERE-6040 Improvements for CSV import template
SELECT register_migration_script('202403262247_IDEMPIERE-6040.sql') FROM dual;
SET SQLBLANKLINES ON
SET DEFINE OFF
-- Mar 26, 2024, 10:47:25 PM CET
INSERT INTO AD_Column (AD_Column_ID,Version,Name,AD_Table_ID,ColumnName,DefaultValue,FieldLength,IsKey,IsParent,IsMandatory,IsTranslated,IsIdentifier,SeqNo,IsEncrypted,AD_Reference_ID,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Element_ID,IsUpdateable,IsSelectionColumn,EntityType,IsSyncDatabase,IsAlwaysUpdateable,IsAutocomplete,IsAllowLogging,AD_Column_UU,IsAllowCopy,SeqNoSelection,IsToolbarButton,IsSecure,IsHtml,IsPartitionKey) VALUES (216584,0,'Separator Character',200153,'SeparatorChar',',',1,'N','N','Y','N','N',0,'N',10,0,0,'Y',TO_TIMESTAMP('2024-03-26 22:47:24','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-26 22:47:24','YYYY-MM-DD HH24:MI:SS'),100,54158,'Y','N','D','N','N','N','Y','3083364e-7dac-4ee2-8fa9-7ffe047b0461','Y',0,'N','N','N','N')
;
-- Mar 26, 2024, 10:47:28 PM CET
ALTER TABLE AD_ImportTemplate ADD SeparatorChar VARCHAR2(1 CHAR) DEFAULT ',' NOT NULL
;
-- Mar 26, 2024, 10:48:07 PM CET
INSERT INTO AD_Element (AD_Element_ID,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,ColumnName,Name,Description,Help,PrintName,EntityType,AD_Element_UU) VALUES (203928,0,0,'Y',TO_TIMESTAMP('2024-03-26 22:47:56','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-26 22:47:56','YYYY-MM-DD HH24:MI:SS'),100,'QuoteChar','Quote Char',NULL,NULL,'Quote Char','D','9bdf8508-f83b-46bc-baca-c27d5b1fac5e')
;
-- Mar 26, 2024, 10:48:24 PM CET
INSERT INTO AD_Column (AD_Column_ID,Version,Name,AD_Table_ID,ColumnName,DefaultValue,FieldLength,IsKey,IsParent,IsMandatory,IsTranslated,IsIdentifier,SeqNo,IsEncrypted,AD_Reference_ID,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Element_ID,IsUpdateable,IsSelectionColumn,EntityType,IsSyncDatabase,IsAlwaysUpdateable,IsAutocomplete,IsAllowLogging,AD_Column_UU,IsAllowCopy,SeqNoSelection,IsToolbarButton,IsSecure,IsHtml,IsPartitionKey) VALUES (216585,0,'Quote Char',200153,'QuoteChar','"',1,'N','N','Y','N','N',0,'N',10,0,0,'Y',TO_TIMESTAMP('2024-03-26 22:48:23','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-26 22:48:23','YYYY-MM-DD HH24:MI:SS'),100,203928,'Y','N','D','N','N','N','Y','9b51ff6c-2c9c-4f1b-ad66-a8fa4e0b1f1b','Y',0,'N','N','N','N')
;
-- Mar 26, 2024, 10:48:27 PM CET
ALTER TABLE AD_ImportTemplate ADD QuoteChar VARCHAR2(1 CHAR) DEFAULT '"' NOT NULL
;
-- Mar 26, 2024, 10:48:44 PM CET
INSERT INTO AD_Field (AD_Field_ID,Name,AD_Tab_ID,AD_Column_ID,IsDisplayed,DisplayLength,SeqNo,IsSameLine,IsHeading,IsFieldOnly,IsEncrypted,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,IsReadOnly,IsCentrallyMaintained,EntityType,AD_Field_UU,IsDisplayedGrid,SeqNoGrid,ColumnSpan) VALUES (208474,'Separator Character',200167,216584,'Y',1,110,'N','N','N','N',0,0,'Y',TO_TIMESTAMP('2024-03-26 22:48:44','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-26 22:48:44','YYYY-MM-DD HH24:MI:SS'),100,'N','Y','D','ada7dc92-28b4-4887-a577-d49226450b6c','Y',110,2)
;
-- Mar 26, 2024, 10:48:45 PM CET
INSERT INTO AD_Field (AD_Field_ID,Name,AD_Tab_ID,AD_Column_ID,IsDisplayed,DisplayLength,SeqNo,IsSameLine,IsHeading,IsFieldOnly,IsEncrypted,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,IsReadOnly,IsCentrallyMaintained,EntityType,AD_Field_UU,IsDisplayedGrid,SeqNoGrid,ColumnSpan) VALUES (208475,'Quote Char',200167,216585,'Y',1,120,'N','N','N','N',0,0,'Y',TO_TIMESTAMP('2024-03-26 22:48:44','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-26 22:48:44','YYYY-MM-DD HH24:MI:SS'),100,'N','Y','D','6f9a0de7-2df9-46b2-a491-a450f61d594e','Y',120,2)
;
-- Mar 26, 2024, 10:49:29 PM CET
UPDATE AD_Field SET SeqNo=70, ColumnSpan=1,Updated=TO_TIMESTAMP('2024-03-26 22:49:29','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=203455
;
-- Mar 26, 2024, 10:49:29 PM CET
UPDATE AD_Field SET IsDisplayed='Y', SeqNo=80, XPosition=3, ColumnSpan=1,Updated=TO_TIMESTAMP('2024-03-26 22:49:29','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=208474
;
-- Mar 26, 2024, 10:49:29 PM CET
UPDATE AD_Field SET IsDisplayed='Y', SeqNo=90, XPosition=5, ColumnSpan=1,Updated=TO_TIMESTAMP('2024-03-26 22:49:29','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=208475
;
-- Mar 26, 2024, 10:49:29 PM CET
UPDATE AD_Field SET SeqNo=100,Updated=TO_TIMESTAMP('2024-03-26 22:49:29','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=203456
;
-- Mar 26, 2024, 10:49:29 PM CET
UPDATE AD_Field SET SeqNo=110,Updated=TO_TIMESTAMP('2024-03-26 22:49:29','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=203457
;
-- Mar 26, 2024, 10:49:29 PM CET
UPDATE AD_Field SET SeqNo=120,Updated=TO_TIMESTAMP('2024-03-26 22:49:29','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=203458
;
-- Mar 26, 2024, 10:53:52 PM CET
UPDATE AD_Field SET DefaultValue='@SQL=SELECT '','' FROM Dual',Updated=TO_TIMESTAMP('2024-03-26 22:53:52','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=208474
;
-- Mar 27, 2024, 12:23:08 PM CET
INSERT INTO AD_Reference (AD_Reference_ID,Name,ValidationType,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,EntityType,IsOrderByValue,AD_Reference_UU,ShowInactive) VALUES (200268,'ImportTemplateType','L',0,0,'Y',TO_TIMESTAMP('2024-03-27 12:23:08','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:23:08','YYYY-MM-DD HH24:MI:SS'),100,'D','N','7926c8b4-2e48-4dd3-9054-8e26245c8128','N')
;
-- Mar 27, 2024, 12:23:20 PM CET
INSERT INTO AD_Ref_List (AD_Ref_List_ID,Name,AD_Reference_ID,Value,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,EntityType,AD_Ref_List_UU) VALUES (200704,'CSV',200268,'CSV',0,0,'Y',TO_TIMESTAMP('2024-03-27 12:23:20','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:23:20','YYYY-MM-DD HH24:MI:SS'),100,'D','7259e211-5477-4f1b-b2a3-1ce67f919749')
;
-- Mar 27, 2024, 12:23:26 PM CET
INSERT INTO AD_Ref_List (AD_Ref_List_ID,Name,AD_Reference_ID,Value,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,EntityType,AD_Ref_List_UU) VALUES (200705,'XLS',200268,'XLS',0,0,'Y',TO_TIMESTAMP('2024-03-27 12:23:26','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:23:26','YYYY-MM-DD HH24:MI:SS'),100,'D','4c979a06-6550-4763-9cd2-066c0d897b9c')
;
-- Mar 27, 2024, 12:23:34 PM CET
INSERT INTO AD_Ref_List (AD_Ref_List_ID,Name,AD_Reference_ID,Value,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,EntityType,AD_Ref_List_UU) VALUES (200706,'XLSX',200268,'XLSX',0,0,'Y',TO_TIMESTAMP('2024-03-27 12:23:33','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:23:33','YYYY-MM-DD HH24:MI:SS'),100,'D','0f46a993-00ac-4572-935c-0590629d092b')
;
-- Mar 27, 2024, 12:23:54 PM CET
INSERT INTO AD_Element (AD_Element_ID,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,ColumnName,Name,Description,Help,PrintName,EntityType,AD_Element_UU) VALUES (203929,0,0,'Y',TO_TIMESTAMP('2024-03-27 12:23:42','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:23:42','YYYY-MM-DD HH24:MI:SS'),100,'ImportTemplateType','Import Template Type',NULL,NULL,'Import Template Type','D','6855e8ac-5c66-428f-84e9-dd81002aa03e')
;
-- Mar 27, 2024, 12:24:26 PM CET
INSERT INTO AD_Column (AD_Column_ID,Version,Name,AD_Table_ID,ColumnName,DefaultValue,FieldLength,IsKey,IsParent,IsMandatory,IsTranslated,IsIdentifier,SeqNo,IsEncrypted,AD_Reference_ID,AD_Reference_Value_ID,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Element_ID,IsUpdateable,IsSelectionColumn,EntityType,IsSyncDatabase,IsAlwaysUpdateable,IsAutocomplete,IsAllowLogging,AD_Column_UU,IsAllowCopy,SeqNoSelection,IsToolbarButton,IsSecure,IsHtml,IsPartitionKey) VALUES (216586,0,'Import Template Type',200153,'ImportTemplateType','CSV',4,'N','N','Y','N','N',0,'N',17,200268,0,0,'Y',TO_TIMESTAMP('2024-03-27 12:24:26','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:24:26','YYYY-MM-DD HH24:MI:SS'),100,203929,'Y','N','D','N','N','N','Y','4a5dce57-eb09-4bbc-b2d7-1e09d49eb7f3','Y',0,'N','N','N','N')
;
-- Mar 27, 2024, 12:24:30 PM CET
ALTER TABLE AD_ImportTemplate ADD ImportTemplateType VARCHAR2(4 CHAR) DEFAULT 'CSV' NOT NULL
;
-- Mar 27, 2024, 12:24:41 PM CET
INSERT INTO AD_Field (AD_Field_ID,Name,AD_Tab_ID,AD_Column_ID,IsDisplayed,DisplayLength,SeqNo,IsSameLine,IsHeading,IsFieldOnly,IsEncrypted,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,IsReadOnly,IsCentrallyMaintained,EntityType,AD_Field_UU,IsDisplayedGrid,SeqNoGrid,ColumnSpan) VALUES (208476,'Import Template Type',200167,216586,'Y',4,130,'N','N','N','N',0,0,'Y',TO_TIMESTAMP('2024-03-27 12:24:40','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:24:40','YYYY-MM-DD HH24:MI:SS'),100,'N','Y','D','88a997fd-241c-429c-a5fe-9f64d71c2a3a','Y',130,2)
;
-- Mar 27, 2024, 12:25:34 PM CET
UPDATE AD_Field SET IsDisplayed='Y', SeqNo=120, XPosition=1, ColumnSpan=1,Updated=TO_TIMESTAMP('2024-03-27 12:25:34','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=208476
;
-- Mar 27, 2024, 12:25:34 PM CET
UPDATE AD_Field SET IsDisplayed='Y', SeqNo=130, XPosition=5,Updated=TO_TIMESTAMP('2024-03-27 12:25:34','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=203458
;
-- Mar 27, 2024, 12:26:14 PM CET
INSERT INTO AD_Message (MsgType,MsgText,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Message_ID,Value,EntityType,AD_Message_UU) VALUES ('E','There was an error converting the Excel file to CSV',0,0,'Y',TO_TIMESTAMP('2024-03-27 12:26:13','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:26:13','YYYY-MM-DD HH24:MI:SS'),100,200878,'ErrorConvertingXlsToCsv','D','e36c4d43-93f8-4b45-bfcc-ec4e7fba72cd')
;
-- Mar 27, 2024, 9:44:59 PM CET
INSERT INTO AD_Message (MsgType,MsgText,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Message_ID,Value,EntityType,AD_Message_UU) VALUES ('E','CSV Header Column not found -> {0}',0,0,'Y',TO_TIMESTAMP('2024-03-27 21:44:59','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 21:44:59','YYYY-MM-DD HH24:MI:SS'),100,200879,'CSVHeaderColumnNotFound','D','f3fdb148-1a0a-4c62-b8fa-55ab40e0e434')
;

107
migration/iD11/postgresql/202403262247_IDEMPIERE-6040.sql Normal file
View File

@ -0,0 +1,107 @@
-- IDEMPIERE-6040 Improvements for CSV import template
SELECT register_migration_script('202403262247_IDEMPIERE-6040.sql') FROM dual;
-- Mar 26, 2024, 10:47:25 PM CET
INSERT INTO AD_Column (AD_Column_ID,Version,Name,AD_Table_ID,ColumnName,DefaultValue,FieldLength,IsKey,IsParent,IsMandatory,IsTranslated,IsIdentifier,SeqNo,IsEncrypted,AD_Reference_ID,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Element_ID,IsUpdateable,IsSelectionColumn,EntityType,IsSyncDatabase,IsAlwaysUpdateable,IsAutocomplete,IsAllowLogging,AD_Column_UU,IsAllowCopy,SeqNoSelection,IsToolbarButton,IsSecure,IsHtml,IsPartitionKey) VALUES (216584,0,'Separator Character',200153,'SeparatorChar',',',1,'N','N','Y','N','N',0,'N',10,0,0,'Y',TO_TIMESTAMP('2024-03-26 22:47:24','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-26 22:47:24','YYYY-MM-DD HH24:MI:SS'),100,54158,'Y','N','D','N','N','N','Y','3083364e-7dac-4ee2-8fa9-7ffe047b0461','Y',0,'N','N','N','N')
;
-- Mar 26, 2024, 10:47:28 PM CET
ALTER TABLE AD_ImportTemplate ADD COLUMN SeparatorChar VARCHAR(1) DEFAULT ',' NOT NULL
;
-- Mar 26, 2024, 10:48:07 PM CET
INSERT INTO AD_Element (AD_Element_ID,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,ColumnName,Name,Description,Help,PrintName,EntityType,AD_Element_UU) VALUES (203928,0,0,'Y',TO_TIMESTAMP('2024-03-26 22:47:56','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-26 22:47:56','YYYY-MM-DD HH24:MI:SS'),100,'QuoteChar','Quote Char',NULL,NULL,'Quote Char','D','9bdf8508-f83b-46bc-baca-c27d5b1fac5e')
;
-- Mar 26, 2024, 10:48:24 PM CET
INSERT INTO AD_Column (AD_Column_ID,Version,Name,AD_Table_ID,ColumnName,DefaultValue,FieldLength,IsKey,IsParent,IsMandatory,IsTranslated,IsIdentifier,SeqNo,IsEncrypted,AD_Reference_ID,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Element_ID,IsUpdateable,IsSelectionColumn,EntityType,IsSyncDatabase,IsAlwaysUpdateable,IsAutocomplete,IsAllowLogging,AD_Column_UU,IsAllowCopy,SeqNoSelection,IsToolbarButton,IsSecure,IsHtml,IsPartitionKey) VALUES (216585,0,'Quote Char',200153,'QuoteChar','"',1,'N','N','Y','N','N',0,'N',10,0,0,'Y',TO_TIMESTAMP('2024-03-26 22:48:23','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-26 22:48:23','YYYY-MM-DD HH24:MI:SS'),100,203928,'Y','N','D','N','N','N','Y','9b51ff6c-2c9c-4f1b-ad66-a8fa4e0b1f1b','Y',0,'N','N','N','N')
;
-- Mar 26, 2024, 10:48:27 PM CET
ALTER TABLE AD_ImportTemplate ADD COLUMN QuoteChar VARCHAR(1) DEFAULT '"' NOT NULL
;
-- Mar 26, 2024, 10:48:44 PM CET
INSERT INTO AD_Field (AD_Field_ID,Name,AD_Tab_ID,AD_Column_ID,IsDisplayed,DisplayLength,SeqNo,IsSameLine,IsHeading,IsFieldOnly,IsEncrypted,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,IsReadOnly,IsCentrallyMaintained,EntityType,AD_Field_UU,IsDisplayedGrid,SeqNoGrid,ColumnSpan) VALUES (208474,'Separator Character',200167,216584,'Y',1,110,'N','N','N','N',0,0,'Y',TO_TIMESTAMP('2024-03-26 22:48:44','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-26 22:48:44','YYYY-MM-DD HH24:MI:SS'),100,'N','Y','D','ada7dc92-28b4-4887-a577-d49226450b6c','Y',110,2)
;
-- Mar 26, 2024, 10:48:45 PM CET
INSERT INTO AD_Field (AD_Field_ID,Name,AD_Tab_ID,AD_Column_ID,IsDisplayed,DisplayLength,SeqNo,IsSameLine,IsHeading,IsFieldOnly,IsEncrypted,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,IsReadOnly,IsCentrallyMaintained,EntityType,AD_Field_UU,IsDisplayedGrid,SeqNoGrid,ColumnSpan) VALUES (208475,'Quote Char',200167,216585,'Y',1,120,'N','N','N','N',0,0,'Y',TO_TIMESTAMP('2024-03-26 22:48:44','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-26 22:48:44','YYYY-MM-DD HH24:MI:SS'),100,'N','Y','D','6f9a0de7-2df9-46b2-a491-a450f61d594e','Y',120,2)
;
-- Mar 26, 2024, 10:49:29 PM CET
UPDATE AD_Field SET SeqNo=70, ColumnSpan=1,Updated=TO_TIMESTAMP('2024-03-26 22:49:29','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=203455
;
-- Mar 26, 2024, 10:49:29 PM CET
UPDATE AD_Field SET IsDisplayed='Y', SeqNo=80, XPosition=3, ColumnSpan=1,Updated=TO_TIMESTAMP('2024-03-26 22:49:29','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=208474
;
-- Mar 26, 2024, 10:49:29 PM CET
UPDATE AD_Field SET IsDisplayed='Y', SeqNo=90, XPosition=5, ColumnSpan=1,Updated=TO_TIMESTAMP('2024-03-26 22:49:29','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=208475
;
-- Mar 26, 2024, 10:49:29 PM CET
UPDATE AD_Field SET SeqNo=100,Updated=TO_TIMESTAMP('2024-03-26 22:49:29','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=203456
;
-- Mar 26, 2024, 10:49:29 PM CET
UPDATE AD_Field SET SeqNo=110,Updated=TO_TIMESTAMP('2024-03-26 22:49:29','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=203457
;
-- Mar 26, 2024, 10:49:29 PM CET
UPDATE AD_Field SET SeqNo=120,Updated=TO_TIMESTAMP('2024-03-26 22:49:29','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=203458
;
-- Mar 26, 2024, 10:53:52 PM CET
UPDATE AD_Field SET DefaultValue='@SQL=SELECT '','' FROM Dual',Updated=TO_TIMESTAMP('2024-03-26 22:53:52','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=208474
;
-- Mar 27, 2024, 12:23:08 PM CET
INSERT INTO AD_Reference (AD_Reference_ID,Name,ValidationType,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,EntityType,IsOrderByValue,AD_Reference_UU,ShowInactive) VALUES (200268,'ImportTemplateType','L',0,0,'Y',TO_TIMESTAMP('2024-03-27 12:23:08','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:23:08','YYYY-MM-DD HH24:MI:SS'),100,'D','N','7926c8b4-2e48-4dd3-9054-8e26245c8128','N')
;
-- Mar 27, 2024, 12:23:20 PM CET
INSERT INTO AD_Ref_List (AD_Ref_List_ID,Name,AD_Reference_ID,Value,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,EntityType,AD_Ref_List_UU) VALUES (200704,'CSV',200268,'CSV',0,0,'Y',TO_TIMESTAMP('2024-03-27 12:23:20','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:23:20','YYYY-MM-DD HH24:MI:SS'),100,'D','7259e211-5477-4f1b-b2a3-1ce67f919749')
;
-- Mar 27, 2024, 12:23:26 PM CET
INSERT INTO AD_Ref_List (AD_Ref_List_ID,Name,AD_Reference_ID,Value,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,EntityType,AD_Ref_List_UU) VALUES (200705,'XLS',200268,'XLS',0,0,'Y',TO_TIMESTAMP('2024-03-27 12:23:26','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:23:26','YYYY-MM-DD HH24:MI:SS'),100,'D','4c979a06-6550-4763-9cd2-066c0d897b9c')
;
-- Mar 27, 2024, 12:23:34 PM CET
INSERT INTO AD_Ref_List (AD_Ref_List_ID,Name,AD_Reference_ID,Value,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,EntityType,AD_Ref_List_UU) VALUES (200706,'XLSX',200268,'XLSX',0,0,'Y',TO_TIMESTAMP('2024-03-27 12:23:33','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:23:33','YYYY-MM-DD HH24:MI:SS'),100,'D','0f46a993-00ac-4572-935c-0590629d092b')
;
-- Mar 27, 2024, 12:23:54 PM CET
INSERT INTO AD_Element (AD_Element_ID,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,ColumnName,Name,Description,Help,PrintName,EntityType,AD_Element_UU) VALUES (203929,0,0,'Y',TO_TIMESTAMP('2024-03-27 12:23:42','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:23:42','YYYY-MM-DD HH24:MI:SS'),100,'ImportTemplateType','Import Template Type',NULL,NULL,'Import Template Type','D','6855e8ac-5c66-428f-84e9-dd81002aa03e')
;
-- Mar 27, 2024, 12:24:26 PM CET
INSERT INTO AD_Column (AD_Column_ID,Version,Name,AD_Table_ID,ColumnName,DefaultValue,FieldLength,IsKey,IsParent,IsMandatory,IsTranslated,IsIdentifier,SeqNo,IsEncrypted,AD_Reference_ID,AD_Reference_Value_ID,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Element_ID,IsUpdateable,IsSelectionColumn,EntityType,IsSyncDatabase,IsAlwaysUpdateable,IsAutocomplete,IsAllowLogging,AD_Column_UU,IsAllowCopy,SeqNoSelection,IsToolbarButton,IsSecure,IsHtml,IsPartitionKey) VALUES (216586,0,'Import Template Type',200153,'ImportTemplateType','CSV',4,'N','N','Y','N','N',0,'N',17,200268,0,0,'Y',TO_TIMESTAMP('2024-03-27 12:24:26','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:24:26','YYYY-MM-DD HH24:MI:SS'),100,203929,'Y','N','D','N','N','N','Y','4a5dce57-eb09-4bbc-b2d7-1e09d49eb7f3','Y',0,'N','N','N','N')
;
-- Mar 27, 2024, 12:24:30 PM CET
ALTER TABLE AD_ImportTemplate ADD COLUMN ImportTemplateType VARCHAR(4) DEFAULT 'CSV' NOT NULL
;
-- Mar 27, 2024, 12:24:41 PM CET
INSERT INTO AD_Field (AD_Field_ID,Name,AD_Tab_ID,AD_Column_ID,IsDisplayed,DisplayLength,SeqNo,IsSameLine,IsHeading,IsFieldOnly,IsEncrypted,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,IsReadOnly,IsCentrallyMaintained,EntityType,AD_Field_UU,IsDisplayedGrid,SeqNoGrid,ColumnSpan) VALUES (208476,'Import Template Type',200167,216586,'Y',4,130,'N','N','N','N',0,0,'Y',TO_TIMESTAMP('2024-03-27 12:24:40','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:24:40','YYYY-MM-DD HH24:MI:SS'),100,'N','Y','D','88a997fd-241c-429c-a5fe-9f64d71c2a3a','Y',130,2)
;
-- Mar 27, 2024, 12:25:34 PM CET
UPDATE AD_Field SET IsDisplayed='Y', SeqNo=120, XPosition=1, ColumnSpan=1,Updated=TO_TIMESTAMP('2024-03-27 12:25:34','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=208476
;
-- Mar 27, 2024, 12:25:34 PM CET
UPDATE AD_Field SET IsDisplayed='Y', SeqNo=130, XPosition=5,Updated=TO_TIMESTAMP('2024-03-27 12:25:34','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=203458
;
-- Mar 27, 2024, 12:26:14 PM CET
INSERT INTO AD_Message (MsgType,MsgText,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Message_ID,Value,EntityType,AD_Message_UU) VALUES ('E','There was an error converting the Excel file to CSV',0,0,'Y',TO_TIMESTAMP('2024-03-27 12:26:13','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:26:13','YYYY-MM-DD HH24:MI:SS'),100,200878,'ErrorConvertingXlsToCsv','D','e36c4d43-93f8-4b45-bfcc-ec4e7fba72cd')
;
-- Mar 27, 2024, 9:44:59 PM CET
INSERT INTO AD_Message (MsgType,MsgText,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Message_ID,Value,EntityType,AD_Message_UU) VALUES ('E','CSV Header Column not found -> {0}',0,0,'Y',TO_TIMESTAMP('2024-03-27 21:44:59','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 21:44:59','YYYY-MM-DD HH24:MI:SS'),100,200879,'CSVHeaderColumnNotFound','D','f3fdb148-1a0a-4c62-b8fa-55ab40e0e434')
;

2
org.adempiere.base.process/src/org/idempiere/process/ImportCSVProcess.java
View File

@ -147,7 +147,7 @@ public class ImportCSVProcess extends SvrProcess implements DataStatusListener {
m_file_istream = new FileInputStream(filePath); m_file_istream = new FileInputStream(filePath);
m_file_istream = m_importTemplate.validateFile(m_file_istream); m_file_istream = m_importTemplate.validateFile(m_file_istream);
File outFile = csvImporter.fileImport(activeTab, childTabs, m_file_istream, Charset.forName(m_importTemplate.getCharacterSet()), p_ImportMode, processUI); File outFile = csvImporter.fileImport(activeTab, childTabs, m_file_istream, Charset.forName(m_importTemplate.getCharacterSet()), p_ImportMode, m_importTemplate.getSeparatorChar(), m_importTemplate.getQuoteChar(), processUI);
// TODO: Potential improvement - traverse the outFile and call addLog with the results // TODO: Potential improvement - traverse the outFile and call addLog with the results
if (processUI != null) if (processUI != null)

14
org.adempiere.base/src/org/adempiere/base/IGridTabImporter.java
View File

@ -49,6 +49,20 @@ public interface IGridTabImporter {
*/ */
public File fileImport(GridTab gridTab, List<GridTab> childs, InputStream filestream, Charset charset, String importMode, IProcessUI processUI); public File fileImport(GridTab gridTab, List<GridTab> childs, InputStream filestream, Charset charset, String importMode, IProcessUI processUI);
/**
* Import data from filestream to gridTab
* @param gridTab
* @param childs
* @param filestream
* @param charset
* @param importMode
* @param delimiterChar
* @param quoteChar
* @param processUI
* @return File for import log
*/
public File fileImport(GridTab gridTab, List<GridTab> childs, InputStream filestream, Charset charset, String importMode, String delimiterChar, String quoteChar, IProcessUI processUI);
/** /**
* @return file extension * @return file extension
*/ */

32
org.adempiere.base/src/org/adempiere/impexp/GridTabCSVImporter.java
View File

@ -117,9 +117,8 @@ public class GridTabCSVImporter implements IGridTabImporter
private PrintWriter errFileW; private PrintWriter errFileW;
private PrintWriter logFileW; private PrintWriter logFileW;
private CsvPreference csvpref = CsvPreference.STANDARD_PREFERENCE; private String delimiterChar = ",";
private String delimiter = String.valueOf((char) csvpref.getDelimiterChar()); private String quoteChar = "\"";
private String quoteChar = String.valueOf((char) csvpref.getQuoteChar());
//Trx //Trx
private Trx trx; private Trx trx;
@ -131,11 +130,16 @@ public class GridTabCSVImporter implements IGridTabImporter
@Override @Override
public File fileImport(GridTab gridTab, List<GridTab> childs, InputStream filestream, Charset charset , String importMode) { public File fileImport(GridTab gridTab, List<GridTab> childs, InputStream filestream, Charset charset , String importMode) {
return fileImport(gridTab, childs, filestream, charset, importMode, null); return fileImport(gridTab, childs, filestream, charset, importMode, null, null, null);
}//fileImport }//fileImport
@Override @Override
public File fileImport(GridTab gridTab, List<GridTab> childs, InputStream filestream, Charset charset, String importMode, IProcessUI processUI) { public File fileImport(GridTab gridTab, List<GridTab> childs, InputStream filestream, Charset charset, String importMode, IProcessUI processUI) {
return fileImport(gridTab, childs, filestream, charset, importMode, null, null, processUI);
}
@Override
public File fileImport(GridTab gridTab, List<GridTab> childs, InputStream filestream, Charset charset, String importMode, String p_delimiterChar, String p_quoteChar, IProcessUI processUI) {
if(!gridTab.isInsertRecord() && isInsertMode()) if(!gridTab.isInsertRecord() && isInsertMode())
throwAdempiereException("Insert record disabled for Tab"); throwAdempiereException("Insert record disabled for Tab");
@ -146,6 +150,13 @@ public class GridTabCSVImporter implements IGridTabImporter
m_import_mode = importMode; m_import_mode = importMode;
errFile = new File(errFileName); errFile = new File(errFileName);
errFileW = new PrintWriter(errFile, charset.name()); errFileW = new PrintWriter(errFile, charset.name());
if (p_delimiterChar != null)
delimiterChar = p_delimiterChar;
if (p_quoteChar != null)
quoteChar = p_quoteChar;
CsvPreference csvpref = new CsvPreference.Builder(quoteChar.charAt(0), delimiterChar.charAt(0), "\r\n" /* ignored */).build();
mapReader = new CsvMapReader(new InputStreamReader(filestream, charset), csvpref); mapReader = new CsvMapReader(new InputStreamReader(filestream, charset), csvpref);
header = Arrays.asList(mapReader.getHeader(true)); header = Arrays.asList(mapReader.getHeader(true));
@ -178,7 +189,7 @@ public class GridTabCSVImporter implements IGridTabImporter
m_isError = false; m_isError = false;
// write the header // write the header
String rawHeader = mapReader.getUntokenizedRow(); String rawHeader = mapReader.getUntokenizedRow();
errFileW.write(rawHeader + delimiter + ERROR_HEADER + "\n"); errFileW.write(rawHeader + delimiterChar + ERROR_HEADER + "\n");
data = new ArrayList<Map<String, Object>>(); data = new ArrayList<Map<String, Object>>();
rawData = new ArrayList<String>(); rawData = new ArrayList<String>();
@ -191,7 +202,7 @@ public class GridTabCSVImporter implements IGridTabImporter
logFile = new File(logFileName); logFile = new File(logFileName);
logFileW = new PrintWriter(logFile, charset.name()); logFileW = new PrintWriter(logFile, charset.name());
// write the header // write the header
logFileW.write(rawHeader + delimiter + LOG_HEADER + "\n"); logFileW.write(rawHeader + delimiterChar + LOG_HEADER + "\n");
// no errors found - process header and then details // no errors found - process header and then details
isMasterok = true; isMasterok = true;
isDetailok = true; isDetailok = true;
@ -226,11 +237,11 @@ public class GridTabCSVImporter implements IGridTabImporter
} }
if (!isMasterok && isDetail){ if (!isMasterok && isDetail){
rawLine = rawLine + delimiter + quoteChar + Msg.getMsg(Env.getCtx(),"NotProcessed") + quoteChar + "\n"; rawLine = rawLine + delimiterChar + quoteChar + Msg.getMsg(Env.getCtx(),"NotProcessed") + quoteChar + "\n";
rowsTmpResult.add(rawLine); rowsTmpResult.add(rawLine);
continue; continue;
}else if(isMasterok && isDetail && !isDetailok){ }else if(isMasterok && isDetail && !isDetailok){
rawLine = rawLine + delimiter + quoteChar + "Record not processed due to detail record failure" + quoteChar + "\n"; rawLine = rawLine + delimiterChar + quoteChar + "Record not processed due to detail record failure" + quoteChar + "\n";
rowsTmpResult.add(rawLine); rowsTmpResult.add(rawLine);
continue; continue;
} }
@ -249,7 +260,7 @@ public class GridTabCSVImporter implements IGridTabImporter
rowResult.append(recordResult); rowResult.append(recordResult);
// write // write
rawLine = rawLine + delimiter + quoteChar + rowResult.toString().replaceAll(delimiter, "") + quoteChar + "\n"; rawLine = rawLine + delimiterChar + quoteChar + rowResult.toString().replaceAll(delimiterChar, "") + quoteChar + "\n";
rowsTmpResult.add(rawLine); rowsTmpResult.add(rawLine);
if( isSingleTrx() && isError() ) if( isSingleTrx() && isError() )
@ -530,7 +541,7 @@ public class GridTabCSVImporter implements IGridTabImporter
rawData.add(rawLine); rawData.add(rawLine);
} }
// write // write
rawLine = rawLine + delimiter + quoteChar + errMsg.toString().replaceAll(quoteChar, "") + quoteChar + "\n"; rawLine = rawLine + delimiterChar + quoteChar + errMsg.toString().replaceAll(quoteChar, "") + quoteChar + "\n";
errFileW.write(rawLine); errFileW.write(rawLine);
} }
}//preProcess }//preProcess
@ -1747,4 +1758,5 @@ public class GridTabCSVImporter implements IGridTabImporter
return -1; return -1;
} }
} }
} }

27
org.adempiere.base/src/org/compiere/model/I_AD_ImportTemplate.java
View File

@ -166,6 +166,15 @@ public interface I_AD_ImportTemplate
*/ */
public String getDescription(); public String getDescription();
/** Column name ImportTemplateType */
public static final String COLUMNNAME_ImportTemplateType = "ImportTemplateType";
/** Set Import Template Type */
public void setImportTemplateType (String ImportTemplateType);
/** Get Import Template Type */
public String getImportTemplateType();
/** Column name IsActive */ /** Column name IsActive */
public static final String COLUMNNAME_IsActive = "IsActive"; public static final String COLUMNNAME_IsActive = "IsActive";
@ -192,6 +201,24 @@ public interface I_AD_ImportTemplate
*/ */
public String getName(); public String getName();
/** Column name QuoteChar */
public static final String COLUMNNAME_QuoteChar = "QuoteChar";
/** Set Quote Char */
public void setQuoteChar (String QuoteChar);
/** Get Quote Char */
public String getQuoteChar();
/** Column name SeparatorChar */
public static final String COLUMNNAME_SeparatorChar = "SeparatorChar";
/** Set Separator Character */
public void setSeparatorChar (String SeparatorChar);
/** Get Separator Character */
public String getSeparatorChar();
/** Column name Updated */ /** Column name Updated */
public static final String COLUMNNAME_Updated = "Updated"; public static final String COLUMNNAME_Updated = "Updated";

276
org.adempiere.base/src/org/compiere/model/MImportTemplate.java
View File

@ -23,13 +23,26 @@ import java.io.InputStream;
import java.io.InputStreamReader; import java.io.InputStreamReader;
import java.nio.charset.Charset; import java.nio.charset.Charset;
import java.sql.ResultSet; import java.sql.ResultSet;
import java.text.DecimalFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List; import java.util.List;
import java.util.Properties; import java.util.Properties;
import java.util.logging.Level;
import org.adempiere.exceptions.AdempiereException; import org.adempiere.exceptions.AdempiereException;
import org.apache.poi.hssf.usermodel.HSSFWorkbookFactory;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.CellType;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbookFactory;
import org.compiere.util.CCache; import org.compiere.util.CCache;
import org.compiere.util.CLogger; import org.compiere.util.CLogger;
import org.compiere.util.DB; import org.compiere.util.DB;
import org.compiere.util.DisplayType;
import org.compiere.util.Env; import org.compiere.util.Env;
import org.compiere.util.Msg; import org.compiere.util.Msg;
import org.idempiere.cache.ImmutablePOSupport; import org.idempiere.cache.ImmutablePOSupport;
@ -42,9 +55,9 @@ import org.idempiere.cache.ImmutablePOSupport;
public class MImportTemplate extends X_AD_ImportTemplate implements ImmutablePOSupport public class MImportTemplate extends X_AD_ImportTemplate implements ImmutablePOSupport
{ {
/** /**
* generated serial id *
*/ */
private static final long serialVersionUID = -4681075469110529774L; private static final long serialVersionUID = 7751951143524784901L;
/** Logger */ /** Logger */
@SuppressWarnings("unused") @SuppressWarnings("unused")
@ -123,6 +136,8 @@ public class MImportTemplate extends X_AD_ImportTemplate implements ImmutablePOS
log.saveError("Error", Msg.parseTranslation(getCtx(), "@Invalid@ @CharacterSet@")); log.saveError("Error", Msg.parseTranslation(getCtx(), "@Invalid@ @CharacterSet@"));
return false; return false;
} }
if (is_new() || is_ValueChanged(COLUMNNAME_CSVHeader) || is_ValueChanged(COLUMNNAME_AD_Tab_ID))
calculateColumnTypes(); // this throws an Exception if there are wrong columns in the CSV Header
return super.beforeSave(newRecord); return super.beforeSave(newRecord);
} }
@ -198,33 +213,49 @@ public class MImportTemplate extends X_AD_ImportTemplate implements ImmutablePOS
} }
/** /**
* Validate that InputStream header is CSVHeader or AliasCSVHeader.<br/> * Byte Order Mark character for UTF-16BE
* If the header is AliasCSVHeader it replaces it with the CSVHeader so it can be */
* processed. public static final String UTF16BE_BOM = "\uFEFF";
/**
* - If needed preProcess the file when an Excel file and generate a CSV file, then<br/>
* - Validate that InputStream header is CSVHeader or AliasCSVHeader.<br/>
* - If the header is AliasCSVHeader it replaces it with the CSVHeader so it can be processed.<br/>
* @param in input file * @param in input file
* @return InputStream with the CSVHeader that can be processed by CsvMapReader * @return InputStream with the CSVHeader that can be processed by CsvMapReader
*/ */
public InputStream validateFile(InputStream in) { public InputStream validateFile(InputStream in) {
// because the input stream cannot be reset we need to copy here the file to a new one (replacing the header if it's the alias)
if ( MImportTemplate.IMPORTTEMPLATETYPE_XLS.equals(getImportTemplateType())
|| MImportTemplate.IMPORTTEMPLATETYPE_XLSX.equals(getImportTemplateType())) {
try {
in = convertExcelToCSV(in);
} catch (Exception e) {
throw new AdempiereException(Msg.getMsg(Env.getCtx(), "ErrorConvertingXlsToCsv") + " -> " + e.getLocalizedMessage(), e);
}
}
Charset charset = Charset.forName(getCharacterSet()); Charset charset = Charset.forName(getCharacterSet());
BufferedReader reader = new BufferedReader(new InputStreamReader(in, charset)); BufferedReader reader = new BufferedReader(new InputStreamReader(in, charset));
File tmpfile = null; File tmpfile = null;
InputStream is = null; InputStream is = null;
BufferedWriter bw = null; BufferedWriter bw = null;
try { try {
tmpfile = File.createTempFile("CSVImportAction", "csv");
bw = new BufferedWriter(new FileWriter(tmpfile,charset));
String firstLine = null; String firstLine = null;
String line = null; String line = null;
while ((line = reader.readLine()) != null) { while ((line = reader.readLine()) != null) {
if (firstLine == null) { if (firstLine == null) {
firstLine = line; firstLine = line;
/* Validate that m_file_istream header is CSVHeader or AliasCSVHeader */ if (firstLine.startsWith(UTF16BE_BOM))
firstLine = firstLine.substring(1);
/* Validate that file header is CSVHeader or AliasCSVHeader */
if ( firstLine.equals(getCSVHeader()) if ( firstLine.equals(getCSVHeader())
|| firstLine.equals(getCSVAliasHeader())) { || firstLine.equals(getCSVAliasHeader())) {
// we copy here the file to a new one (replacing the header if it's the alias)
tmpfile = File.createTempFile("CSVImportAction", ".csv");
bw = new BufferedWriter(new FileWriter(tmpfile,charset));
bw.write(getCSVHeader()); bw.write(getCSVHeader());
} else { } else {
reader.close();
throw new AdempiereException(Msg.getMsg(Env.getCtx(), "WrongCSVHeader")); throw new AdempiereException(Msg.getMsg(Env.getCtx(), "WrongCSVHeader"));
} }
} else { } else {
@ -257,6 +288,231 @@ public class MImportTemplate extends X_AD_ImportTemplate implements ImmutablePOS
return is; return is;
} }
/**
* Convert an Excel (XLS or XLSX) file to CSV
* @param excelIs input stream containing XLS/XLSX
* @return input stream containing CSV
* @throws IOException
*/
public InputStream convertExcelToCSV(InputStream excelIs) throws IOException {
Workbook workbook = null;
if (MImportTemplate.IMPORTTEMPLATETYPE_XLS.equals(getImportTemplateType())) {
HSSFWorkbookFactory xlsWbf = new HSSFWorkbookFactory();
workbook = xlsWbf.create(excelIs);
} else if (MImportTemplate.IMPORTTEMPLATETYPE_XLSX.equals(getImportTemplateType())) {
XSSFWorkbookFactory xlsxWbf = new XSSFWorkbookFactory();
workbook = xlsxWbf.create(excelIs);
} else {
// unexpected error
throw new AdempiereException("Wrong template type -> " + getImportTemplateType());
}
List<Integer> colTypes = calculateColumnTypes();
Sheet sheet = workbook.getSheetAt(0); // First sheet
File tmpfile = File.createTempFile("CSVImportActionConvert", ".csv");
Charset charset = Charset.forName(getCharacterSet());
try (BufferedWriter bw = new BufferedWriter(new FileWriter(tmpfile, charset))) {
for (Row row : sheet) {
boolean firstCell = true;
for (int cn = 0; cn < colTypes.size(); cn++) {
int displayType = colTypes.get(cn);
Cell cell = row.getCell(cn, Row.MissingCellPolicy.CREATE_NULL_AS_BLANK);
if (firstCell) {
firstCell = false;
} else {
bw.append(getSeparatorChar());
}
CellType cellType = cell.getCellType();
if (cellType.equals(CellType.FORMULA))
cellType = cell.getCachedFormulaResultType();
if (! ( cellType.equals(CellType.BLANK)
|| cellType.equals(CellType.STRING)
|| cellType.equals(CellType.BOOLEAN)
|| cellType.equals(CellType.NUMERIC)))
throw new IllegalStateException("Unsupported cell type: " + cell.getCellType());
if (! cellType.equals(CellType.BLANK)) { // blank cells are not processed - write them just as empty strings
if (cellType.equals(CellType.BOOLEAN)) {
// boolean cells are converted to Y/N iDempiere notation
if (cell.getBooleanCellValue())
bw.append("Y");
else
bw.append("N");
} else if (DisplayType.YesNo == displayType) {
// a Yes/No value is expected, try to convert true/false and 1/0 to Y/N
if (cellType.equals(CellType.NUMERIC)) {
double doubleValue = cell.getNumericCellValue();
if (doubleValue == 1.0)
bw.append("Y");
else if (doubleValue == 0.0)
bw.append("N");
else
addNumeric(bw, doubleValue, displayType);
} else { // STRING
String stringValue = cell.getStringCellValue();
if (stringValue.equalsIgnoreCase("true") || stringValue.equals("y"))
stringValue = "Y";
else if (stringValue.equalsIgnoreCase("false") || stringValue.equals("n"))
stringValue = "N";
addString(bw, stringValue);
}
} else if (DisplayType.isDate(displayType)) {
// a date time is expected
if (cellType.equals(CellType.NUMERIC)) {
SimpleDateFormat sdf;
if (displayType == DisplayType.Date)
sdf = DisplayType.getDateFormat_JDBC();
else
sdf = DisplayType.getTimestampFormat_Default();
Date date = cell.getDateCellValue();
addString(bw, sdf.format(date));
} else { // STRING
addString(bw, cell.getStringCellValue());
}
} else {
if (cellType.equals(CellType.NUMERIC)) {
addNumeric(bw, cell.getNumericCellValue(), displayType);
} else { // STRING
addString(bw, cell.getStringCellValue());
}
}
}
}
bw.newLine();
}
}
workbook.close();
InputStream is = new FileInputStream(tmpfile);
return is;
}
/**
* Add a String to the BufferedWriter
* @param bw BufferedWriter
* @param value String
* @throws IOException
*/
private void addString(BufferedWriter bw, String value) throws IOException {
boolean addQuotes = (value.contains(getQuoteChar()) || value.contains(getSeparatorChar()));
if (addQuotes)
bw.append(getQuoteChar());
bw.append(value.replace(getQuoteChar(), getQuoteChar().concat(getQuoteChar())));
if (addQuotes)
bw.append(getQuoteChar());
}
/**
* Add a Numeric value to the BufferedWriter
* @param bw BufferedWriter
* @param doubleValue double
* @param displayType
* @throws IOException
*/
private void addNumeric(BufferedWriter bw, double doubleValue, int displayType) throws IOException {
int intValue = (int) doubleValue;
DecimalFormat df = DisplayType.getNumberFormat(displayType);
if ( intValue == doubleValue
|| displayType == DisplayType.Integer
|| displayType == DisplayType.Table
|| displayType == DisplayType.TableDir
|| displayType == DisplayType.Search
|| displayType == DisplayType.RecordID) {
df.setParseIntegerOnly(true);
df.setMaximumFractionDigits(0);
}
df.setGroupingSize(0);
String str = df.format(doubleValue);
addString(bw, str);
}
/**
* Calculate the expected column types depending on the columns defined in the CSVHeader
* Every column can have the format
* Column
* Column[ForeignColumn]
* DetailTableName>Column
* DetailTableName>Column[ForeignColumn]
* Any column can end with /K (can be ignored)
* @return List of expected DisplayType for every column
*/
private List<Integer> calculateColumnTypes() {
List<Integer> retValue = new ArrayList<Integer>();
String[] csvHeaders = getCSVHeader().split(getSeparatorChar());
MTab mainTab = MTab.get(getAD_Tab_ID());
MTable mainTable = MTable.get(mainTab.getAD_Table_ID());
for (String csvHeader : csvHeaders) {
String columnHeader = csvHeader;
// first remove the /K mark
if (columnHeader.endsWith("/K"))
columnHeader = columnHeader.substring(0, columnHeader.length()-2);
// check if there is a foreign column that defines the type
String foreignColumnName = null;
int idxOpen = columnHeader.indexOf("[");
if (idxOpen > 0) {
int idxClose = columnHeader.indexOf("]");
if (idxClose > 0 && idxClose > idxOpen) {
foreignColumnName = columnHeader.substring(idxOpen+1, idxClose);
columnHeader = columnHeader.substring(0, idxOpen);
}
}
// if this is from a detail table
int idxTableSep = columnHeader.lastIndexOf(">");
MTable table = null;
String tableName = null;
String columnName = null;
if (idxTableSep > 0) {
tableName = columnHeader.substring(0, idxTableSep);
if (tableName.lastIndexOf(">") > 0) {
tableName = tableName.substring(tableName.lastIndexOf(">")+1);
}
table = MTable.get(getCtx(), tableName);
columnName = columnHeader.substring(idxTableSep+1);
} else {
table = mainTable;
tableName = table.getTableName();
columnName = columnHeader;
}
if (table == null)
throwCSVHeaderNotFound(csvHeader);
MColumn column = table.getColumn(columnName);
if (column == null)
throwCSVHeaderNotFound(csvHeader);
String foreignTableName = null;
if (foreignColumnName != null) {
foreignTableName = column.getReferenceTableName();
MTable foreignTable = MTable.get(getCtx(), foreignTableName);
if (foreignTable == null)
throwCSVHeaderNotFound(csvHeader);
column = foreignTable.getColumn(foreignColumnName);
if (column == null)
throwCSVHeaderNotFound(csvHeader);
}
if (log.isLoggable(Level.INFO))
log.info("CSV Header column found -> " + csvHeader + " -> " + tableName + "." + columnName
+ (foreignColumnName != null ? "[" + foreignTableName + "." + foreignColumnName + "]" : "") );
int ref = column.getAD_Reference_ID();
retValue.add(ref);
}
return retValue;
}
/**
* Throws an exception indicating a CSV Header that could not be found
* @param csvHeader
*/
private void throwCSVHeaderNotFound(String csvHeader) {
throw new AdempiereException(Msg.getMsg(getCtx(), "CSVHeaderColumnNotFound", new Object[]{csvHeader}));
}
@Override @Override
public MImportTemplate markImmutable() { public MImportTemplate markImmutable() {
if (is_Immutable()) if (is_Immutable())

80
org.adempiere.base/src/org/compiere/model/X_AD_ImportTemplate.java
View File

@ -31,7 +31,7 @@ public class X_AD_ImportTemplate extends PO implements I_AD_ImportTemplate, I_Pe
/** /**
* *
*/ */
private static final long serialVersionUID = 20231222L; private static final long serialVersionUID = 20240327L;
/** Standard Constructor */ /** Standard Constructor */
public X_AD_ImportTemplate (Properties ctx, int AD_ImportTemplate_ID, String trxName) public X_AD_ImportTemplate (Properties ctx, int AD_ImportTemplate_ID, String trxName)
@ -44,7 +44,13 @@ public class X_AD_ImportTemplate extends PO implements I_AD_ImportTemplate, I_Pe
setAD_Window_ID (0); setAD_Window_ID (0);
setCharacterSet (null); setCharacterSet (null);
setCSVHeader (null); setCSVHeader (null);
setImportTemplateType (null);
// CSV
setName (null); setName (null);
setQuoteChar (null);
// "
setSeparatorChar (null);
// ,
} */ } */
} }
@ -59,7 +65,13 @@ public class X_AD_ImportTemplate extends PO implements I_AD_ImportTemplate, I_Pe
setAD_Window_ID (0); setAD_Window_ID (0);
setCharacterSet (null); setCharacterSet (null);
setCSVHeader (null); setCSVHeader (null);
setImportTemplateType (null);
// CSV
setName (null); setName (null);
setQuoteChar (null);
// "
setSeparatorChar (null);
// ,
} */ } */
} }
@ -74,7 +86,13 @@ public class X_AD_ImportTemplate extends PO implements I_AD_ImportTemplate, I_Pe
setAD_Window_ID (0); setAD_Window_ID (0);
setCharacterSet (null); setCharacterSet (null);
setCSVHeader (null); setCSVHeader (null);
setImportTemplateType (null);
// CSV
setName (null); setName (null);
setQuoteChar (null);
// "
setSeparatorChar (null);
// ,
} */ } */
} }
@ -89,7 +107,13 @@ public class X_AD_ImportTemplate extends PO implements I_AD_ImportTemplate, I_Pe
setAD_Window_ID (0); setAD_Window_ID (0);
setCharacterSet (null); setCharacterSet (null);
setCSVHeader (null); setCSVHeader (null);
setImportTemplateType (null);
// CSV
setName (null); setName (null);
setQuoteChar (null);
// "
setSeparatorChar (null);
// ,
} */ } */
} }
@ -274,6 +298,30 @@ public class X_AD_ImportTemplate extends PO implements I_AD_ImportTemplate, I_Pe
return (String)get_Value(COLUMNNAME_Description); return (String)get_Value(COLUMNNAME_Description);
} }
/** ImportTemplateType AD_Reference_ID=200268 */
public static final int IMPORTTEMPLATETYPE_AD_Reference_ID=200268;
/** CSV = CSV */
public static final String IMPORTTEMPLATETYPE_CSV = "CSV";
/** XLS = XLS */
public static final String IMPORTTEMPLATETYPE_XLS = "XLS";
/** XLSX = XLSX */
public static final String IMPORTTEMPLATETYPE_XLSX = "XLSX";
/** Set Import Template Type.
@param ImportTemplateType Import Template Type
*/
public void setImportTemplateType (String ImportTemplateType)
{
set_Value (COLUMNNAME_ImportTemplateType, ImportTemplateType);
}
/** Get Import Template Type.
@return Import Template Type */
public String getImportTemplateType()
{
return (String)get_Value(COLUMNNAME_ImportTemplateType);
}
/** Set Name. /** Set Name.
@param Name Alphanumeric identifier of the entity @param Name Alphanumeric identifier of the entity
*/ */
@ -297,4 +345,34 @@ public class X_AD_ImportTemplate extends PO implements I_AD_ImportTemplate, I_Pe
{ {
return new KeyNamePair(get_ID(), getName()); return new KeyNamePair(get_ID(), getName());
} }
/** Set Quote Char.
@param QuoteChar Quote Char
*/
public void setQuoteChar (String QuoteChar)
{
set_Value (COLUMNNAME_QuoteChar, QuoteChar);
}
/** Get Quote Char.
@return Quote Char */
public String getQuoteChar()
{
return (String)get_Value(COLUMNNAME_QuoteChar);
}
/** Set Separator Character.
@param SeparatorChar Separator Character
*/
public void setSeparatorChar (String SeparatorChar)
{
set_Value (COLUMNNAME_SeparatorChar, SeparatorChar);
}
/** Get Separator Character.
@return Separator Character */
public String getSeparatorChar()
{
return (String)get_Value(COLUMNNAME_SeparatorChar);
}
} }

2
org.adempiere.ui.zk/WEB-INF/src/org/adempiere/webui/panel/action/CSVImportAction.java
View File

@ -352,7 +352,7 @@ public class CSVImportAction implements EventListener<Event>
String iMode = (String)importItem.getValue(); String iMode = (String)importItem.getValue();
m_file_istream = theTemplate.validateFile(m_file_istream); m_file_istream = theTemplate.validateFile(m_file_istream);
File outFile = theCSVImporter.fileImport(panel.getActiveGridTab(), childs, m_file_istream, charset,iMode); File outFile = theCSVImporter.fileImport(panel.getActiveGridTab(), childs, m_file_istream, charset, iMode, theTemplate.getSeparatorChar(), theTemplate.getQuoteChar(), null);
winImportFile.onClose(); winImportFile.onClose();
winImportFile = null; winImportFile = null;