IDEMPIERE-6040 Improvements for CSV import template (#2279)
* IDEMPIERE-6040 Improvements for CSV import template - Delimiter (Field Separator) configurable - Quote delimiter configurable - Fix issue when the UTF-8 file comes with BOM character * - add support for preprocessing excel files * - convert Excel to CSV in a proper format according to the column types
This commit is contained in:
Carlos Ruiz
parent
af10c3a674
commit
d16538ae38
|
|
@ -0,0 +1,110 @@
|
|||
-- IDEMPIERE-6040 Improvements for CSV import template
|
||||
SELECT register_migration_script('202403262247_IDEMPIERE-6040.sql') FROM dual;
|
||||
|
||||
SET SQLBLANKLINES ON
|
||||
SET DEFINE OFF
|
||||
|
||||
-- Mar 26, 2024, 10:47:25 PM CET
|
||||
INSERT INTO AD_Column (AD_Column_ID,Version,Name,AD_Table_ID,ColumnName,DefaultValue,FieldLength,IsKey,IsParent,IsMandatory,IsTranslated,IsIdentifier,SeqNo,IsEncrypted,AD_Reference_ID,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Element_ID,IsUpdateable,IsSelectionColumn,EntityType,IsSyncDatabase,IsAlwaysUpdateable,IsAutocomplete,IsAllowLogging,AD_Column_UU,IsAllowCopy,SeqNoSelection,IsToolbarButton,IsSecure,IsHtml,IsPartitionKey) VALUES (216584,0,'Separator Character',200153,'SeparatorChar',',',1,'N','N','Y','N','N',0,'N',10,0,0,'Y',TO_TIMESTAMP('2024-03-26 22:47:24','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-26 22:47:24','YYYY-MM-DD HH24:MI:SS'),100,54158,'Y','N','D','N','N','N','Y','3083364e-7dac-4ee2-8fa9-7ffe047b0461','Y',0,'N','N','N','N')
|
||||
;
|
||||
|
||||
-- Mar 26, 2024, 10:47:28 PM CET
|
||||
ALTER TABLE AD_ImportTemplate ADD SeparatorChar VARCHAR2(1 CHAR) DEFAULT ',' NOT NULL
|
||||
;
|
||||
|
||||
-- Mar 26, 2024, 10:48:07 PM CET
|
||||
INSERT INTO AD_Element (AD_Element_ID,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,ColumnName,Name,Description,Help,PrintName,EntityType,AD_Element_UU) VALUES (203928,0,0,'Y',TO_TIMESTAMP('2024-03-26 22:47:56','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-26 22:47:56','YYYY-MM-DD HH24:MI:SS'),100,'QuoteChar','Quote Char',NULL,NULL,'Quote Char','D','9bdf8508-f83b-46bc-baca-c27d5b1fac5e')
|
||||
;
|
||||
|
||||
-- Mar 26, 2024, 10:48:24 PM CET
|
||||
INSERT INTO AD_Column (AD_Column_ID,Version,Name,AD_Table_ID,ColumnName,DefaultValue,FieldLength,IsKey,IsParent,IsMandatory,IsTranslated,IsIdentifier,SeqNo,IsEncrypted,AD_Reference_ID,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Element_ID,IsUpdateable,IsSelectionColumn,EntityType,IsSyncDatabase,IsAlwaysUpdateable,IsAutocomplete,IsAllowLogging,AD_Column_UU,IsAllowCopy,SeqNoSelection,IsToolbarButton,IsSecure,IsHtml,IsPartitionKey) VALUES (216585,0,'Quote Char',200153,'QuoteChar','"',1,'N','N','Y','N','N',0,'N',10,0,0,'Y',TO_TIMESTAMP('2024-03-26 22:48:23','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-26 22:48:23','YYYY-MM-DD HH24:MI:SS'),100,203928,'Y','N','D','N','N','N','Y','9b51ff6c-2c9c-4f1b-ad66-a8fa4e0b1f1b','Y',0,'N','N','N','N')
|
||||
;
|
||||
|
||||
-- Mar 26, 2024, 10:48:27 PM CET
|
||||
ALTER TABLE AD_ImportTemplate ADD QuoteChar VARCHAR2(1 CHAR) DEFAULT '"' NOT NULL
|
||||
;
|
||||
|
||||
-- Mar 26, 2024, 10:48:44 PM CET
|
||||
INSERT INTO AD_Field (AD_Field_ID,Name,AD_Tab_ID,AD_Column_ID,IsDisplayed,DisplayLength,SeqNo,IsSameLine,IsHeading,IsFieldOnly,IsEncrypted,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,IsReadOnly,IsCentrallyMaintained,EntityType,AD_Field_UU,IsDisplayedGrid,SeqNoGrid,ColumnSpan) VALUES (208474,'Separator Character',200167,216584,'Y',1,110,'N','N','N','N',0,0,'Y',TO_TIMESTAMP('2024-03-26 22:48:44','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-26 22:48:44','YYYY-MM-DD HH24:MI:SS'),100,'N','Y','D','ada7dc92-28b4-4887-a577-d49226450b6c','Y',110,2)
|
||||
;
|
||||
|
||||
-- Mar 26, 2024, 10:48:45 PM CET
|
||||
INSERT INTO AD_Field (AD_Field_ID,Name,AD_Tab_ID,AD_Column_ID,IsDisplayed,DisplayLength,SeqNo,IsSameLine,IsHeading,IsFieldOnly,IsEncrypted,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,IsReadOnly,IsCentrallyMaintained,EntityType,AD_Field_UU,IsDisplayedGrid,SeqNoGrid,ColumnSpan) VALUES (208475,'Quote Char',200167,216585,'Y',1,120,'N','N','N','N',0,0,'Y',TO_TIMESTAMP('2024-03-26 22:48:44','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-26 22:48:44','YYYY-MM-DD HH24:MI:SS'),100,'N','Y','D','6f9a0de7-2df9-46b2-a491-a450f61d594e','Y',120,2)
|
||||
;
|
||||
|
||||
-- Mar 26, 2024, 10:49:29 PM CET
|
||||
UPDATE AD_Field SET SeqNo=70, ColumnSpan=1,Updated=TO_TIMESTAMP('2024-03-26 22:49:29','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=203455
|
||||
;
|
||||
|
||||
-- Mar 26, 2024, 10:49:29 PM CET
|
||||
UPDATE AD_Field SET IsDisplayed='Y', SeqNo=80, XPosition=3, ColumnSpan=1,Updated=TO_TIMESTAMP('2024-03-26 22:49:29','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=208474
|
||||
;
|
||||
|
||||
-- Mar 26, 2024, 10:49:29 PM CET
|
||||
UPDATE AD_Field SET IsDisplayed='Y', SeqNo=90, XPosition=5, ColumnSpan=1,Updated=TO_TIMESTAMP('2024-03-26 22:49:29','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=208475
|
||||
;
|
||||
|
||||
-- Mar 26, 2024, 10:49:29 PM CET
|
||||
UPDATE AD_Field SET SeqNo=100,Updated=TO_TIMESTAMP('2024-03-26 22:49:29','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=203456
|
||||
;
|
||||
|
||||
-- Mar 26, 2024, 10:49:29 PM CET
|
||||
UPDATE AD_Field SET SeqNo=110,Updated=TO_TIMESTAMP('2024-03-26 22:49:29','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=203457
|
||||
;
|
||||
|
||||
-- Mar 26, 2024, 10:49:29 PM CET
|
||||
UPDATE AD_Field SET SeqNo=120,Updated=TO_TIMESTAMP('2024-03-26 22:49:29','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=203458
|
||||
;
|
||||
|
||||
-- Mar 26, 2024, 10:53:52 PM CET
|
||||
UPDATE AD_Field SET DefaultValue='@SQL=SELECT '','' FROM Dual',Updated=TO_TIMESTAMP('2024-03-26 22:53:52','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=208474
|
||||
;
|
||||
|
||||
-- Mar 27, 2024, 12:23:08 PM CET
|
||||
INSERT INTO AD_Reference (AD_Reference_ID,Name,ValidationType,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,EntityType,IsOrderByValue,AD_Reference_UU,ShowInactive) VALUES (200268,'ImportTemplateType','L',0,0,'Y',TO_TIMESTAMP('2024-03-27 12:23:08','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:23:08','YYYY-MM-DD HH24:MI:SS'),100,'D','N','7926c8b4-2e48-4dd3-9054-8e26245c8128','N')
|
||||
;
|
||||
|
||||
-- Mar 27, 2024, 12:23:20 PM CET
|
||||
INSERT INTO AD_Ref_List (AD_Ref_List_ID,Name,AD_Reference_ID,Value,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,EntityType,AD_Ref_List_UU) VALUES (200704,'CSV',200268,'CSV',0,0,'Y',TO_TIMESTAMP('2024-03-27 12:23:20','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:23:20','YYYY-MM-DD HH24:MI:SS'),100,'D','7259e211-5477-4f1b-b2a3-1ce67f919749')
|
||||
;
|
||||
|
||||
-- Mar 27, 2024, 12:23:26 PM CET
|
||||
INSERT INTO AD_Ref_List (AD_Ref_List_ID,Name,AD_Reference_ID,Value,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,EntityType,AD_Ref_List_UU) VALUES (200705,'XLS',200268,'XLS',0,0,'Y',TO_TIMESTAMP('2024-03-27 12:23:26','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:23:26','YYYY-MM-DD HH24:MI:SS'),100,'D','4c979a06-6550-4763-9cd2-066c0d897b9c')
|
||||
;
|
||||
|
||||
-- Mar 27, 2024, 12:23:34 PM CET
|
||||
INSERT INTO AD_Ref_List (AD_Ref_List_ID,Name,AD_Reference_ID,Value,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,EntityType,AD_Ref_List_UU) VALUES (200706,'XLSX',200268,'XLSX',0,0,'Y',TO_TIMESTAMP('2024-03-27 12:23:33','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:23:33','YYYY-MM-DD HH24:MI:SS'),100,'D','0f46a993-00ac-4572-935c-0590629d092b')
|
||||
;
|
||||
|
||||
-- Mar 27, 2024, 12:23:54 PM CET
|
||||
INSERT INTO AD_Element (AD_Element_ID,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,ColumnName,Name,Description,Help,PrintName,EntityType,AD_Element_UU) VALUES (203929,0,0,'Y',TO_TIMESTAMP('2024-03-27 12:23:42','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:23:42','YYYY-MM-DD HH24:MI:SS'),100,'ImportTemplateType','Import Template Type',NULL,NULL,'Import Template Type','D','6855e8ac-5c66-428f-84e9-dd81002aa03e')
|
||||
;
|
||||
|
||||
-- Mar 27, 2024, 12:24:26 PM CET
|
||||
INSERT INTO AD_Column (AD_Column_ID,Version,Name,AD_Table_ID,ColumnName,DefaultValue,FieldLength,IsKey,IsParent,IsMandatory,IsTranslated,IsIdentifier,SeqNo,IsEncrypted,AD_Reference_ID,AD_Reference_Value_ID,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Element_ID,IsUpdateable,IsSelectionColumn,EntityType,IsSyncDatabase,IsAlwaysUpdateable,IsAutocomplete,IsAllowLogging,AD_Column_UU,IsAllowCopy,SeqNoSelection,IsToolbarButton,IsSecure,IsHtml,IsPartitionKey) VALUES (216586,0,'Import Template Type',200153,'ImportTemplateType','CSV',4,'N','N','Y','N','N',0,'N',17,200268,0,0,'Y',TO_TIMESTAMP('2024-03-27 12:24:26','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:24:26','YYYY-MM-DD HH24:MI:SS'),100,203929,'Y','N','D','N','N','N','Y','4a5dce57-eb09-4bbc-b2d7-1e09d49eb7f3','Y',0,'N','N','N','N')
|
||||
;
|
||||
|
||||
-- Mar 27, 2024, 12:24:30 PM CET
|
||||
ALTER TABLE AD_ImportTemplate ADD ImportTemplateType VARCHAR2(4 CHAR) DEFAULT 'CSV' NOT NULL
|
||||
;
|
||||
|
||||
-- Mar 27, 2024, 12:24:41 PM CET
|
||||
INSERT INTO AD_Field (AD_Field_ID,Name,AD_Tab_ID,AD_Column_ID,IsDisplayed,DisplayLength,SeqNo,IsSameLine,IsHeading,IsFieldOnly,IsEncrypted,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,IsReadOnly,IsCentrallyMaintained,EntityType,AD_Field_UU,IsDisplayedGrid,SeqNoGrid,ColumnSpan) VALUES (208476,'Import Template Type',200167,216586,'Y',4,130,'N','N','N','N',0,0,'Y',TO_TIMESTAMP('2024-03-27 12:24:40','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:24:40','YYYY-MM-DD HH24:MI:SS'),100,'N','Y','D','88a997fd-241c-429c-a5fe-9f64d71c2a3a','Y',130,2)
|
||||
;
|
||||
|
||||
-- Mar 27, 2024, 12:25:34 PM CET
|
||||
UPDATE AD_Field SET IsDisplayed='Y', SeqNo=120, XPosition=1, ColumnSpan=1,Updated=TO_TIMESTAMP('2024-03-27 12:25:34','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=208476
|
||||
;
|
||||
|
||||
-- Mar 27, 2024, 12:25:34 PM CET
|
||||
UPDATE AD_Field SET IsDisplayed='Y', SeqNo=130, XPosition=5,Updated=TO_TIMESTAMP('2024-03-27 12:25:34','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=203458
|
||||
;
|
||||
|
||||
-- Mar 27, 2024, 12:26:14 PM CET
|
||||
INSERT INTO AD_Message (MsgType,MsgText,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Message_ID,Value,EntityType,AD_Message_UU) VALUES ('E','There was an error converting the Excel file to CSV',0,0,'Y',TO_TIMESTAMP('2024-03-27 12:26:13','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:26:13','YYYY-MM-DD HH24:MI:SS'),100,200878,'ErrorConvertingXlsToCsv','D','e36c4d43-93f8-4b45-bfcc-ec4e7fba72cd')
|
||||
;
|
||||
|
||||
-- Mar 27, 2024, 9:44:59 PM CET
|
||||
INSERT INTO AD_Message (MsgType,MsgText,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Message_ID,Value,EntityType,AD_Message_UU) VALUES ('E','CSV Header Column not found -> {0}',0,0,'Y',TO_TIMESTAMP('2024-03-27 21:44:59','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 21:44:59','YYYY-MM-DD HH24:MI:SS'),100,200879,'CSVHeaderColumnNotFound','D','f3fdb148-1a0a-4c62-b8fa-55ab40e0e434')
|
||||
;
|
||||
|
||||
|
|
@ -0,0 +1,107 @@
|
|||
-- IDEMPIERE-6040 Improvements for CSV import template
|
||||
SELECT register_migration_script('202403262247_IDEMPIERE-6040.sql') FROM dual;
|
||||
|
||||
-- Mar 26, 2024, 10:47:25 PM CET
|
||||
INSERT INTO AD_Column (AD_Column_ID,Version,Name,AD_Table_ID,ColumnName,DefaultValue,FieldLength,IsKey,IsParent,IsMandatory,IsTranslated,IsIdentifier,SeqNo,IsEncrypted,AD_Reference_ID,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Element_ID,IsUpdateable,IsSelectionColumn,EntityType,IsSyncDatabase,IsAlwaysUpdateable,IsAutocomplete,IsAllowLogging,AD_Column_UU,IsAllowCopy,SeqNoSelection,IsToolbarButton,IsSecure,IsHtml,IsPartitionKey) VALUES (216584,0,'Separator Character',200153,'SeparatorChar',',',1,'N','N','Y','N','N',0,'N',10,0,0,'Y',TO_TIMESTAMP('2024-03-26 22:47:24','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-26 22:47:24','YYYY-MM-DD HH24:MI:SS'),100,54158,'Y','N','D','N','N','N','Y','3083364e-7dac-4ee2-8fa9-7ffe047b0461','Y',0,'N','N','N','N')
|
||||
;
|
||||
|
||||
-- Mar 26, 2024, 10:47:28 PM CET
|
||||
ALTER TABLE AD_ImportTemplate ADD COLUMN SeparatorChar VARCHAR(1) DEFAULT ',' NOT NULL
|
||||
;
|
||||
|
||||
-- Mar 26, 2024, 10:48:07 PM CET
|
||||
INSERT INTO AD_Element (AD_Element_ID,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,ColumnName,Name,Description,Help,PrintName,EntityType,AD_Element_UU) VALUES (203928,0,0,'Y',TO_TIMESTAMP('2024-03-26 22:47:56','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-26 22:47:56','YYYY-MM-DD HH24:MI:SS'),100,'QuoteChar','Quote Char',NULL,NULL,'Quote Char','D','9bdf8508-f83b-46bc-baca-c27d5b1fac5e')
|
||||
;
|
||||
|
||||
-- Mar 26, 2024, 10:48:24 PM CET
|
||||
INSERT INTO AD_Column (AD_Column_ID,Version,Name,AD_Table_ID,ColumnName,DefaultValue,FieldLength,IsKey,IsParent,IsMandatory,IsTranslated,IsIdentifier,SeqNo,IsEncrypted,AD_Reference_ID,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Element_ID,IsUpdateable,IsSelectionColumn,EntityType,IsSyncDatabase,IsAlwaysUpdateable,IsAutocomplete,IsAllowLogging,AD_Column_UU,IsAllowCopy,SeqNoSelection,IsToolbarButton,IsSecure,IsHtml,IsPartitionKey) VALUES (216585,0,'Quote Char',200153,'QuoteChar','"',1,'N','N','Y','N','N',0,'N',10,0,0,'Y',TO_TIMESTAMP('2024-03-26 22:48:23','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-26 22:48:23','YYYY-MM-DD HH24:MI:SS'),100,203928,'Y','N','D','N','N','N','Y','9b51ff6c-2c9c-4f1b-ad66-a8fa4e0b1f1b','Y',0,'N','N','N','N')
|
||||
;
|
||||
|
||||
-- Mar 26, 2024, 10:48:27 PM CET
|
||||
ALTER TABLE AD_ImportTemplate ADD COLUMN QuoteChar VARCHAR(1) DEFAULT '"' NOT NULL
|
||||
;
|
||||
|
||||
-- Mar 26, 2024, 10:48:44 PM CET
|
||||
INSERT INTO AD_Field (AD_Field_ID,Name,AD_Tab_ID,AD_Column_ID,IsDisplayed,DisplayLength,SeqNo,IsSameLine,IsHeading,IsFieldOnly,IsEncrypted,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,IsReadOnly,IsCentrallyMaintained,EntityType,AD_Field_UU,IsDisplayedGrid,SeqNoGrid,ColumnSpan) VALUES (208474,'Separator Character',200167,216584,'Y',1,110,'N','N','N','N',0,0,'Y',TO_TIMESTAMP('2024-03-26 22:48:44','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-26 22:48:44','YYYY-MM-DD HH24:MI:SS'),100,'N','Y','D','ada7dc92-28b4-4887-a577-d49226450b6c','Y',110,2)
|
||||
;
|
||||
|
||||
-- Mar 26, 2024, 10:48:45 PM CET
|
||||
INSERT INTO AD_Field (AD_Field_ID,Name,AD_Tab_ID,AD_Column_ID,IsDisplayed,DisplayLength,SeqNo,IsSameLine,IsHeading,IsFieldOnly,IsEncrypted,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,IsReadOnly,IsCentrallyMaintained,EntityType,AD_Field_UU,IsDisplayedGrid,SeqNoGrid,ColumnSpan) VALUES (208475,'Quote Char',200167,216585,'Y',1,120,'N','N','N','N',0,0,'Y',TO_TIMESTAMP('2024-03-26 22:48:44','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-26 22:48:44','YYYY-MM-DD HH24:MI:SS'),100,'N','Y','D','6f9a0de7-2df9-46b2-a491-a450f61d594e','Y',120,2)
|
||||
;
|
||||
|
||||
-- Mar 26, 2024, 10:49:29 PM CET
|
||||
UPDATE AD_Field SET SeqNo=70, ColumnSpan=1,Updated=TO_TIMESTAMP('2024-03-26 22:49:29','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=203455
|
||||
;
|
||||
|
||||
-- Mar 26, 2024, 10:49:29 PM CET
|
||||
UPDATE AD_Field SET IsDisplayed='Y', SeqNo=80, XPosition=3, ColumnSpan=1,Updated=TO_TIMESTAMP('2024-03-26 22:49:29','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=208474
|
||||
;
|
||||
|
||||
-- Mar 26, 2024, 10:49:29 PM CET
|
||||
UPDATE AD_Field SET IsDisplayed='Y', SeqNo=90, XPosition=5, ColumnSpan=1,Updated=TO_TIMESTAMP('2024-03-26 22:49:29','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=208475
|
||||
;
|
||||
|
||||
-- Mar 26, 2024, 10:49:29 PM CET
|
||||
UPDATE AD_Field SET SeqNo=100,Updated=TO_TIMESTAMP('2024-03-26 22:49:29','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=203456
|
||||
;
|
||||
|
||||
-- Mar 26, 2024, 10:49:29 PM CET
|
||||
UPDATE AD_Field SET SeqNo=110,Updated=TO_TIMESTAMP('2024-03-26 22:49:29','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=203457
|
||||
;
|
||||
|
||||
-- Mar 26, 2024, 10:49:29 PM CET
|
||||
UPDATE AD_Field SET SeqNo=120,Updated=TO_TIMESTAMP('2024-03-26 22:49:29','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=203458
|
||||
;
|
||||
|
||||
-- Mar 26, 2024, 10:53:52 PM CET
|
||||
UPDATE AD_Field SET DefaultValue='@SQL=SELECT '','' FROM Dual',Updated=TO_TIMESTAMP('2024-03-26 22:53:52','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=208474
|
||||
;
|
||||
|
||||
-- Mar 27, 2024, 12:23:08 PM CET
|
||||
INSERT INTO AD_Reference (AD_Reference_ID,Name,ValidationType,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,EntityType,IsOrderByValue,AD_Reference_UU,ShowInactive) VALUES (200268,'ImportTemplateType','L',0,0,'Y',TO_TIMESTAMP('2024-03-27 12:23:08','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:23:08','YYYY-MM-DD HH24:MI:SS'),100,'D','N','7926c8b4-2e48-4dd3-9054-8e26245c8128','N')
|
||||
;
|
||||
|
||||
-- Mar 27, 2024, 12:23:20 PM CET
|
||||
INSERT INTO AD_Ref_List (AD_Ref_List_ID,Name,AD_Reference_ID,Value,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,EntityType,AD_Ref_List_UU) VALUES (200704,'CSV',200268,'CSV',0,0,'Y',TO_TIMESTAMP('2024-03-27 12:23:20','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:23:20','YYYY-MM-DD HH24:MI:SS'),100,'D','7259e211-5477-4f1b-b2a3-1ce67f919749')
|
||||
;
|
||||
|
||||
-- Mar 27, 2024, 12:23:26 PM CET
|
||||
INSERT INTO AD_Ref_List (AD_Ref_List_ID,Name,AD_Reference_ID,Value,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,EntityType,AD_Ref_List_UU) VALUES (200705,'XLS',200268,'XLS',0,0,'Y',TO_TIMESTAMP('2024-03-27 12:23:26','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:23:26','YYYY-MM-DD HH24:MI:SS'),100,'D','4c979a06-6550-4763-9cd2-066c0d897b9c')
|
||||
;
|
||||
|
||||
-- Mar 27, 2024, 12:23:34 PM CET
|
||||
INSERT INTO AD_Ref_List (AD_Ref_List_ID,Name,AD_Reference_ID,Value,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,EntityType,AD_Ref_List_UU) VALUES (200706,'XLSX',200268,'XLSX',0,0,'Y',TO_TIMESTAMP('2024-03-27 12:23:33','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:23:33','YYYY-MM-DD HH24:MI:SS'),100,'D','0f46a993-00ac-4572-935c-0590629d092b')
|
||||
;
|
||||
|
||||
-- Mar 27, 2024, 12:23:54 PM CET
|
||||
INSERT INTO AD_Element (AD_Element_ID,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,ColumnName,Name,Description,Help,PrintName,EntityType,AD_Element_UU) VALUES (203929,0,0,'Y',TO_TIMESTAMP('2024-03-27 12:23:42','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:23:42','YYYY-MM-DD HH24:MI:SS'),100,'ImportTemplateType','Import Template Type',NULL,NULL,'Import Template Type','D','6855e8ac-5c66-428f-84e9-dd81002aa03e')
|
||||
;
|
||||
|
||||
-- Mar 27, 2024, 12:24:26 PM CET
|
||||
INSERT INTO AD_Column (AD_Column_ID,Version,Name,AD_Table_ID,ColumnName,DefaultValue,FieldLength,IsKey,IsParent,IsMandatory,IsTranslated,IsIdentifier,SeqNo,IsEncrypted,AD_Reference_ID,AD_Reference_Value_ID,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Element_ID,IsUpdateable,IsSelectionColumn,EntityType,IsSyncDatabase,IsAlwaysUpdateable,IsAutocomplete,IsAllowLogging,AD_Column_UU,IsAllowCopy,SeqNoSelection,IsToolbarButton,IsSecure,IsHtml,IsPartitionKey) VALUES (216586,0,'Import Template Type',200153,'ImportTemplateType','CSV',4,'N','N','Y','N','N',0,'N',17,200268,0,0,'Y',TO_TIMESTAMP('2024-03-27 12:24:26','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:24:26','YYYY-MM-DD HH24:MI:SS'),100,203929,'Y','N','D','N','N','N','Y','4a5dce57-eb09-4bbc-b2d7-1e09d49eb7f3','Y',0,'N','N','N','N')
|
||||
;
|
||||
|
||||
-- Mar 27, 2024, 12:24:30 PM CET
|
||||
ALTER TABLE AD_ImportTemplate ADD COLUMN ImportTemplateType VARCHAR(4) DEFAULT 'CSV' NOT NULL
|
||||
;
|
||||
|
||||
-- Mar 27, 2024, 12:24:41 PM CET
|
||||
INSERT INTO AD_Field (AD_Field_ID,Name,AD_Tab_ID,AD_Column_ID,IsDisplayed,DisplayLength,SeqNo,IsSameLine,IsHeading,IsFieldOnly,IsEncrypted,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,IsReadOnly,IsCentrallyMaintained,EntityType,AD_Field_UU,IsDisplayedGrid,SeqNoGrid,ColumnSpan) VALUES (208476,'Import Template Type',200167,216586,'Y',4,130,'N','N','N','N',0,0,'Y',TO_TIMESTAMP('2024-03-27 12:24:40','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:24:40','YYYY-MM-DD HH24:MI:SS'),100,'N','Y','D','88a997fd-241c-429c-a5fe-9f64d71c2a3a','Y',130,2)
|
||||
;
|
||||
|
||||
-- Mar 27, 2024, 12:25:34 PM CET
|
||||
UPDATE AD_Field SET IsDisplayed='Y', SeqNo=120, XPosition=1, ColumnSpan=1,Updated=TO_TIMESTAMP('2024-03-27 12:25:34','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=208476
|
||||
;
|
||||
|
||||
-- Mar 27, 2024, 12:25:34 PM CET
|
||||
UPDATE AD_Field SET IsDisplayed='Y', SeqNo=130, XPosition=5,Updated=TO_TIMESTAMP('2024-03-27 12:25:34','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Field_ID=203458
|
||||
;
|
||||
|
||||
-- Mar 27, 2024, 12:26:14 PM CET
|
||||
INSERT INTO AD_Message (MsgType,MsgText,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Message_ID,Value,EntityType,AD_Message_UU) VALUES ('E','There was an error converting the Excel file to CSV',0,0,'Y',TO_TIMESTAMP('2024-03-27 12:26:13','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 12:26:13','YYYY-MM-DD HH24:MI:SS'),100,200878,'ErrorConvertingXlsToCsv','D','e36c4d43-93f8-4b45-bfcc-ec4e7fba72cd')
|
||||
;
|
||||
|
||||
-- Mar 27, 2024, 9:44:59 PM CET
|
||||
INSERT INTO AD_Message (MsgType,MsgText,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Message_ID,Value,EntityType,AD_Message_UU) VALUES ('E','CSV Header Column not found -> {0}',0,0,'Y',TO_TIMESTAMP('2024-03-27 21:44:59','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-03-27 21:44:59','YYYY-MM-DD HH24:MI:SS'),100,200879,'CSVHeaderColumnNotFound','D','f3fdb148-1a0a-4c62-b8fa-55ab40e0e434')
|
||||
;
|
||||
|
||||
|
|
@ -147,7 +147,7 @@ public class ImportCSVProcess extends SvrProcess implements DataStatusListener {
|
|||
m_file_istream = new FileInputStream(filePath);
|
||||
|
||||
m_file_istream = m_importTemplate.validateFile(m_file_istream);
|
||||
File outFile = csvImporter.fileImport(activeTab, childTabs, m_file_istream, Charset.forName(m_importTemplate.getCharacterSet()), p_ImportMode, processUI);
|
||||
File outFile = csvImporter.fileImport(activeTab, childTabs, m_file_istream, Charset.forName(m_importTemplate.getCharacterSet()), p_ImportMode, m_importTemplate.getSeparatorChar(), m_importTemplate.getQuoteChar(), processUI);
|
||||
// TODO: Potential improvement - traverse the outFile and call addLog with the results
|
||||
|
||||
if (processUI != null)
|
||||
|
|
|
|||
|
|
@ -49,6 +49,20 @@ public interface IGridTabImporter {
|
|||
*/
|
||||
public File fileImport(GridTab gridTab, List<GridTab> childs, InputStream filestream, Charset charset, String importMode, IProcessUI processUI);
|
||||
|
||||
/**
|
||||
* Import data from filestream to gridTab
|
||||
* @param gridTab
|
||||
* @param childs
|
||||
* @param filestream
|
||||
* @param charset
|
||||
* @param importMode
|
||||
* @param delimiterChar
|
||||
* @param quoteChar
|
||||
* @param processUI
|
||||
* @return File for import log
|
||||
*/
|
||||
public File fileImport(GridTab gridTab, List<GridTab> childs, InputStream filestream, Charset charset, String importMode, String delimiterChar, String quoteChar, IProcessUI processUI);
|
||||
|
||||
/**
|
||||
* @return file extension
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -117,9 +117,8 @@ public class GridTabCSVImporter implements IGridTabImporter
|
|||
private PrintWriter errFileW;
|
||||
private PrintWriter logFileW;
|
||||
|
||||
private CsvPreference csvpref = CsvPreference.STANDARD_PREFERENCE;
|
||||
private String delimiter = String.valueOf((char) csvpref.getDelimiterChar());
|
||||
private String quoteChar = String.valueOf((char) csvpref.getQuoteChar());
|
||||
private String delimiterChar = ",";
|
||||
private String quoteChar = "\"";
|
||||
|
||||
//Trx
|
||||
private Trx trx;
|
||||
|
|
@ -131,11 +130,16 @@ public class GridTabCSVImporter implements IGridTabImporter
|
|||
|
||||
@Override
|
||||
public File fileImport(GridTab gridTab, List<GridTab> childs, InputStream filestream, Charset charset , String importMode) {
|
||||
return fileImport(gridTab, childs, filestream, charset, importMode, null);
|
||||
return fileImport(gridTab, childs, filestream, charset, importMode, null, null, null);
|
||||
}//fileImport
|
||||
|
||||
@Override
|
||||
public File fileImport(GridTab gridTab, List<GridTab> childs, InputStream filestream, Charset charset, String importMode, IProcessUI processUI) {
|
||||
return fileImport(gridTab, childs, filestream, charset, importMode, null, null, processUI);
|
||||
}
|
||||
|
||||
@Override
|
||||
public File fileImport(GridTab gridTab, List<GridTab> childs, InputStream filestream, Charset charset, String importMode, String p_delimiterChar, String p_quoteChar, IProcessUI processUI) {
|
||||
|
||||
if(!gridTab.isInsertRecord() && isInsertMode())
|
||||
throwAdempiereException("Insert record disabled for Tab");
|
||||
|
|
@ -146,6 +150,13 @@ public class GridTabCSVImporter implements IGridTabImporter
|
|||
m_import_mode = importMode;
|
||||
errFile = new File(errFileName);
|
||||
errFileW = new PrintWriter(errFile, charset.name());
|
||||
|
||||
if (p_delimiterChar != null)
|
||||
delimiterChar = p_delimiterChar;
|
||||
if (p_quoteChar != null)
|
||||
quoteChar = p_quoteChar;
|
||||
CsvPreference csvpref = new CsvPreference.Builder(quoteChar.charAt(0), delimiterChar.charAt(0), "\r\n" /* ignored */).build();
|
||||
|
||||
mapReader = new CsvMapReader(new InputStreamReader(filestream, charset), csvpref);
|
||||
header = Arrays.asList(mapReader.getHeader(true));
|
||||
|
||||
|
|
@ -178,7 +189,7 @@ public class GridTabCSVImporter implements IGridTabImporter
|
|||
m_isError = false;
|
||||
// write the header
|
||||
String rawHeader = mapReader.getUntokenizedRow();
|
||||
errFileW.write(rawHeader + delimiter + ERROR_HEADER + "\n");
|
||||
errFileW.write(rawHeader + delimiterChar + ERROR_HEADER + "\n");
|
||||
data = new ArrayList<Map<String, Object>>();
|
||||
rawData = new ArrayList<String>();
|
||||
|
||||
|
|
@ -191,7 +202,7 @@ public class GridTabCSVImporter implements IGridTabImporter
|
|||
logFile = new File(logFileName);
|
||||
logFileW = new PrintWriter(logFile, charset.name());
|
||||
// write the header
|
||||
logFileW.write(rawHeader + delimiter + LOG_HEADER + "\n");
|
||||
logFileW.write(rawHeader + delimiterChar + LOG_HEADER + "\n");
|
||||
// no errors found - process header and then details
|
||||
isMasterok = true;
|
||||
isDetailok = true;
|
||||
|
|
@ -226,11 +237,11 @@ public class GridTabCSVImporter implements IGridTabImporter
|
|||
}
|
||||
|
||||
if (!isMasterok && isDetail){
|
||||
rawLine = rawLine + delimiter + quoteChar + Msg.getMsg(Env.getCtx(),"NotProcessed") + quoteChar + "\n";
|
||||
rawLine = rawLine + delimiterChar + quoteChar + Msg.getMsg(Env.getCtx(),"NotProcessed") + quoteChar + "\n";
|
||||
rowsTmpResult.add(rawLine);
|
||||
continue;
|
||||
}else if(isMasterok && isDetail && !isDetailok){
|
||||
rawLine = rawLine + delimiter + quoteChar + "Record not processed due to detail record failure" + quoteChar + "\n";
|
||||
rawLine = rawLine + delimiterChar + quoteChar + "Record not processed due to detail record failure" + quoteChar + "\n";
|
||||
rowsTmpResult.add(rawLine);
|
||||
continue;
|
||||
}
|
||||
|
|
@ -249,7 +260,7 @@ public class GridTabCSVImporter implements IGridTabImporter
|
|||
rowResult.append(recordResult);
|
||||
|
||||
// write
|
||||
rawLine = rawLine + delimiter + quoteChar + rowResult.toString().replaceAll(delimiter, "") + quoteChar + "\n";
|
||||
rawLine = rawLine + delimiterChar + quoteChar + rowResult.toString().replaceAll(delimiterChar, "") + quoteChar + "\n";
|
||||
rowsTmpResult.add(rawLine);
|
||||
|
||||
if( isSingleTrx() && isError() )
|
||||
|
|
@ -530,7 +541,7 @@ public class GridTabCSVImporter implements IGridTabImporter
|
|||
rawData.add(rawLine);
|
||||
}
|
||||
// write
|
||||
rawLine = rawLine + delimiter + quoteChar + errMsg.toString().replaceAll(quoteChar, "") + quoteChar + "\n";
|
||||
rawLine = rawLine + delimiterChar + quoteChar + errMsg.toString().replaceAll(quoteChar, "") + quoteChar + "\n";
|
||||
errFileW.write(rawLine);
|
||||
}
|
||||
}//preProcess
|
||||
|
|
@ -1747,4 +1758,5 @@ public class GridTabCSVImporter implements IGridTabImporter
|
|||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -166,6 +166,15 @@ public interface I_AD_ImportTemplate
|
|||
*/
|
||||
public String getDescription();
|
||||
|
||||
/** Column name ImportTemplateType */
|
||||
public static final String COLUMNNAME_ImportTemplateType = "ImportTemplateType";
|
||||
|
||||
/** Set Import Template Type */
|
||||
public void setImportTemplateType (String ImportTemplateType);
|
||||
|
||||
/** Get Import Template Type */
|
||||
public String getImportTemplateType();
|
||||
|
||||
/** Column name IsActive */
|
||||
public static final String COLUMNNAME_IsActive = "IsActive";
|
||||
|
||||
|
|
@ -192,6 +201,24 @@ public interface I_AD_ImportTemplate
|
|||
*/
|
||||
public String getName();
|
||||
|
||||
/** Column name QuoteChar */
|
||||
public static final String COLUMNNAME_QuoteChar = "QuoteChar";
|
||||
|
||||
/** Set Quote Char */
|
||||
public void setQuoteChar (String QuoteChar);
|
||||
|
||||
/** Get Quote Char */
|
||||
public String getQuoteChar();
|
||||
|
||||
/** Column name SeparatorChar */
|
||||
public static final String COLUMNNAME_SeparatorChar = "SeparatorChar";
|
||||
|
||||
/** Set Separator Character */
|
||||
public void setSeparatorChar (String SeparatorChar);
|
||||
|
||||
/** Get Separator Character */
|
||||
public String getSeparatorChar();
|
||||
|
||||
/** Column name Updated */
|
||||
public static final String COLUMNNAME_Updated = "Updated";
|
||||
|
||||
|
|
|
|||
|
|
@ -23,13 +23,26 @@ import java.io.InputStream;
|
|||
import java.io.InputStreamReader;
|
||||
import java.nio.charset.Charset;
|
||||
import java.sql.ResultSet;
|
||||
import java.text.DecimalFormat;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
import java.util.logging.Level;
|
||||
|
||||
import org.adempiere.exceptions.AdempiereException;
|
||||
import org.apache.poi.hssf.usermodel.HSSFWorkbookFactory;
|
||||
import org.apache.poi.ss.usermodel.Cell;
|
||||
import org.apache.poi.ss.usermodel.CellType;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.ss.usermodel.Workbook;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbookFactory;
|
||||
import org.compiere.util.CCache;
|
||||
import org.compiere.util.CLogger;
|
||||
import org.compiere.util.DB;
|
||||
import org.compiere.util.DisplayType;
|
||||
import org.compiere.util.Env;
|
||||
import org.compiere.util.Msg;
|
||||
import org.idempiere.cache.ImmutablePOSupport;
|
||||
|
|
@ -41,10 +54,10 @@ import org.idempiere.cache.ImmutablePOSupport;
|
|||
*/
|
||||
public class MImportTemplate extends X_AD_ImportTemplate implements ImmutablePOSupport
|
||||
{
|
||||
/**
|
||||
* generated serial id
|
||||
/**
|
||||
*
|
||||
*/
|
||||
private static final long serialVersionUID = -4681075469110529774L;
|
||||
private static final long serialVersionUID = 7751951143524784901L;
|
||||
|
||||
/** Logger */
|
||||
@SuppressWarnings("unused")
|
||||
|
|
@ -123,6 +136,8 @@ public class MImportTemplate extends X_AD_ImportTemplate implements ImmutablePOS
|
|||
log.saveError("Error", Msg.parseTranslation(getCtx(), "@Invalid@ @CharacterSet@"));
|
||||
return false;
|
||||
}
|
||||
if (is_new() || is_ValueChanged(COLUMNNAME_CSVHeader) || is_ValueChanged(COLUMNNAME_AD_Tab_ID))
|
||||
calculateColumnTypes(); // this throws an Exception if there are wrong columns in the CSV Header
|
||||
return super.beforeSave(newRecord);
|
||||
}
|
||||
|
||||
|
|
@ -198,33 +213,49 @@ public class MImportTemplate extends X_AD_ImportTemplate implements ImmutablePOS
|
|||
}
|
||||
|
||||
/**
|
||||
* Validate that InputStream header is CSVHeader or AliasCSVHeader.<br/>
|
||||
* If the header is AliasCSVHeader it replaces it with the CSVHeader so it can be
|
||||
* processed.
|
||||
* Byte Order Mark character for UTF-16BE
|
||||
*/
|
||||
public static final String UTF16BE_BOM = "\uFEFF";
|
||||
|
||||
/**
|
||||
* - If needed preProcess the file when an Excel file and generate a CSV file, then<br/>
|
||||
* - Validate that InputStream header is CSVHeader or AliasCSVHeader.<br/>
|
||||
* - If the header is AliasCSVHeader it replaces it with the CSVHeader so it can be processed.<br/>
|
||||
* @param in input file
|
||||
* @return InputStream with the CSVHeader that can be processed by CsvMapReader
|
||||
*/
|
||||
public InputStream validateFile(InputStream in) {
|
||||
// because the input stream cannot be reset we need to copy here the file to a new one (replacing the header if it's the alias)
|
||||
|
||||
if ( MImportTemplate.IMPORTTEMPLATETYPE_XLS.equals(getImportTemplateType())
|
||||
|| MImportTemplate.IMPORTTEMPLATETYPE_XLSX.equals(getImportTemplateType())) {
|
||||
try {
|
||||
in = convertExcelToCSV(in);
|
||||
} catch (Exception e) {
|
||||
throw new AdempiereException(Msg.getMsg(Env.getCtx(), "ErrorConvertingXlsToCsv") + " -> " + e.getLocalizedMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
Charset charset = Charset.forName(getCharacterSet());
|
||||
BufferedReader reader = new BufferedReader(new InputStreamReader(in, charset));
|
||||
File tmpfile = null;
|
||||
InputStream is = null;
|
||||
BufferedWriter bw = null;
|
||||
try {
|
||||
tmpfile = File.createTempFile("CSVImportAction", "csv");
|
||||
bw = new BufferedWriter(new FileWriter(tmpfile,charset));
|
||||
String firstLine = null;
|
||||
String line = null;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
if (firstLine == null) {
|
||||
firstLine = line;
|
||||
/* Validate that m_file_istream header is CSVHeader or AliasCSVHeader */
|
||||
if (firstLine.startsWith(UTF16BE_BOM))
|
||||
firstLine = firstLine.substring(1);
|
||||
/* Validate that file header is CSVHeader or AliasCSVHeader */
|
||||
if ( firstLine.equals(getCSVHeader())
|
||||
|| firstLine.equals(getCSVAliasHeader())) {
|
||||
// we copy here the file to a new one (replacing the header if it's the alias)
|
||||
tmpfile = File.createTempFile("CSVImportAction", ".csv");
|
||||
bw = new BufferedWriter(new FileWriter(tmpfile,charset));
|
||||
bw.write(getCSVHeader());
|
||||
} else {
|
||||
reader.close();
|
||||
throw new AdempiereException(Msg.getMsg(Env.getCtx(), "WrongCSVHeader"));
|
||||
}
|
||||
} else {
|
||||
|
|
@ -257,6 +288,231 @@ public class MImportTemplate extends X_AD_ImportTemplate implements ImmutablePOS
|
|||
return is;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert an Excel (XLS or XLSX) file to CSV
|
||||
* @param excelIs input stream containing XLS/XLSX
|
||||
* @return input stream containing CSV
|
||||
* @throws IOException
|
||||
*/
|
||||
public InputStream convertExcelToCSV(InputStream excelIs) throws IOException {
|
||||
|
||||
Workbook workbook = null;
|
||||
if (MImportTemplate.IMPORTTEMPLATETYPE_XLS.equals(getImportTemplateType())) {
|
||||
HSSFWorkbookFactory xlsWbf = new HSSFWorkbookFactory();
|
||||
workbook = xlsWbf.create(excelIs);
|
||||
} else if (MImportTemplate.IMPORTTEMPLATETYPE_XLSX.equals(getImportTemplateType())) {
|
||||
XSSFWorkbookFactory xlsxWbf = new XSSFWorkbookFactory();
|
||||
workbook = xlsxWbf.create(excelIs);
|
||||
} else {
|
||||
// unexpected error
|
||||
throw new AdempiereException("Wrong template type -> " + getImportTemplateType());
|
||||
}
|
||||
|
||||
List<Integer> colTypes = calculateColumnTypes();
|
||||
|
||||
Sheet sheet = workbook.getSheetAt(0); // First sheet
|
||||
|
||||
File tmpfile = File.createTempFile("CSVImportActionConvert", ".csv");
|
||||
Charset charset = Charset.forName(getCharacterSet());
|
||||
try (BufferedWriter bw = new BufferedWriter(new FileWriter(tmpfile, charset))) {
|
||||
for (Row row : sheet) {
|
||||
boolean firstCell = true;
|
||||
for (int cn = 0; cn < colTypes.size(); cn++) {
|
||||
int displayType = colTypes.get(cn);
|
||||
Cell cell = row.getCell(cn, Row.MissingCellPolicy.CREATE_NULL_AS_BLANK);
|
||||
if (firstCell) {
|
||||
firstCell = false;
|
||||
} else {
|
||||
bw.append(getSeparatorChar());
|
||||
}
|
||||
|
||||
CellType cellType = cell.getCellType();
|
||||
if (cellType.equals(CellType.FORMULA))
|
||||
cellType = cell.getCachedFormulaResultType();
|
||||
if (! ( cellType.equals(CellType.BLANK)
|
||||
|| cellType.equals(CellType.STRING)
|
||||
|| cellType.equals(CellType.BOOLEAN)
|
||||
|| cellType.equals(CellType.NUMERIC)))
|
||||
throw new IllegalStateException("Unsupported cell type: " + cell.getCellType());
|
||||
if (! cellType.equals(CellType.BLANK)) { // blank cells are not processed - write them just as empty strings
|
||||
|
||||
if (cellType.equals(CellType.BOOLEAN)) {
|
||||
// boolean cells are converted to Y/N iDempiere notation
|
||||
if (cell.getBooleanCellValue())
|
||||
bw.append("Y");
|
||||
else
|
||||
bw.append("N");
|
||||
|
||||
} else if (DisplayType.YesNo == displayType) {
|
||||
// a Yes/No value is expected, try to convert true/false and 1/0 to Y/N
|
||||
if (cellType.equals(CellType.NUMERIC)) {
|
||||
double doubleValue = cell.getNumericCellValue();
|
||||
if (doubleValue == 1.0)
|
||||
bw.append("Y");
|
||||
else if (doubleValue == 0.0)
|
||||
bw.append("N");
|
||||
else
|
||||
addNumeric(bw, doubleValue, displayType);
|
||||
} else { // STRING
|
||||
String stringValue = cell.getStringCellValue();
|
||||
if (stringValue.equalsIgnoreCase("true") || stringValue.equals("y"))
|
||||
stringValue = "Y";
|
||||
else if (stringValue.equalsIgnoreCase("false") || stringValue.equals("n"))
|
||||
stringValue = "N";
|
||||
addString(bw, stringValue);
|
||||
}
|
||||
|
||||
} else if (DisplayType.isDate(displayType)) {
|
||||
// a date time is expected
|
||||
if (cellType.equals(CellType.NUMERIC)) {
|
||||
SimpleDateFormat sdf;
|
||||
if (displayType == DisplayType.Date)
|
||||
sdf = DisplayType.getDateFormat_JDBC();
|
||||
else
|
||||
sdf = DisplayType.getTimestampFormat_Default();
|
||||
Date date = cell.getDateCellValue();
|
||||
addString(bw, sdf.format(date));
|
||||
} else { // STRING
|
||||
addString(bw, cell.getStringCellValue());
|
||||
}
|
||||
|
||||
} else {
|
||||
if (cellType.equals(CellType.NUMERIC)) {
|
||||
addNumeric(bw, cell.getNumericCellValue(), displayType);
|
||||
} else { // STRING
|
||||
addString(bw, cell.getStringCellValue());
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
bw.newLine();
|
||||
}
|
||||
}
|
||||
workbook.close();
|
||||
InputStream is = new FileInputStream(tmpfile);
|
||||
return is;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a String to the BufferedWriter
|
||||
* @param bw BufferedWriter
|
||||
* @param value String
|
||||
* @throws IOException
|
||||
*/
|
||||
private void addString(BufferedWriter bw, String value) throws IOException {
|
||||
boolean addQuotes = (value.contains(getQuoteChar()) || value.contains(getSeparatorChar()));
|
||||
if (addQuotes)
|
||||
bw.append(getQuoteChar());
|
||||
bw.append(value.replace(getQuoteChar(), getQuoteChar().concat(getQuoteChar())));
|
||||
if (addQuotes)
|
||||
bw.append(getQuoteChar());
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a Numeric value to the BufferedWriter
|
||||
* @param bw BufferedWriter
|
||||
* @param doubleValue double
|
||||
* @param displayType
|
||||
* @throws IOException
|
||||
*/
|
||||
private void addNumeric(BufferedWriter bw, double doubleValue, int displayType) throws IOException {
|
||||
int intValue = (int) doubleValue;
|
||||
DecimalFormat df = DisplayType.getNumberFormat(displayType);
|
||||
if ( intValue == doubleValue
|
||||
|| displayType == DisplayType.Integer
|
||||
|| displayType == DisplayType.Table
|
||||
|| displayType == DisplayType.TableDir
|
||||
|| displayType == DisplayType.Search
|
||||
|| displayType == DisplayType.RecordID) {
|
||||
df.setParseIntegerOnly(true);
|
||||
df.setMaximumFractionDigits(0);
|
||||
}
|
||||
df.setGroupingSize(0);
|
||||
String str = df.format(doubleValue);
|
||||
addString(bw, str);
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate the expected column types depending on the columns defined in the CSVHeader
|
||||
* Every column can have the format
|
||||
* Column
|
||||
* Column[ForeignColumn]
|
||||
* DetailTableName>Column
|
||||
* DetailTableName>Column[ForeignColumn]
|
||||
* Any column can end with /K (can be ignored)
|
||||
* @return List of expected DisplayType for every column
|
||||
*/
|
||||
private List<Integer> calculateColumnTypes() {
|
||||
List<Integer> retValue = new ArrayList<Integer>();
|
||||
String[] csvHeaders = getCSVHeader().split(getSeparatorChar());
|
||||
MTab mainTab = MTab.get(getAD_Tab_ID());
|
||||
MTable mainTable = MTable.get(mainTab.getAD_Table_ID());
|
||||
for (String csvHeader : csvHeaders) {
|
||||
String columnHeader = csvHeader;
|
||||
// first remove the /K mark
|
||||
if (columnHeader.endsWith("/K"))
|
||||
columnHeader = columnHeader.substring(0, columnHeader.length()-2);
|
||||
// check if there is a foreign column that defines the type
|
||||
String foreignColumnName = null;
|
||||
int idxOpen = columnHeader.indexOf("[");
|
||||
if (idxOpen > 0) {
|
||||
int idxClose = columnHeader.indexOf("]");
|
||||
if (idxClose > 0 && idxClose > idxOpen) {
|
||||
foreignColumnName = columnHeader.substring(idxOpen+1, idxClose);
|
||||
columnHeader = columnHeader.substring(0, idxOpen);
|
||||
}
|
||||
}
|
||||
// if this is from a detail table
|
||||
int idxTableSep = columnHeader.lastIndexOf(">");
|
||||
MTable table = null;
|
||||
String tableName = null;
|
||||
String columnName = null;
|
||||
if (idxTableSep > 0) {
|
||||
tableName = columnHeader.substring(0, idxTableSep);
|
||||
if (tableName.lastIndexOf(">") > 0) {
|
||||
tableName = tableName.substring(tableName.lastIndexOf(">")+1);
|
||||
}
|
||||
table = MTable.get(getCtx(), tableName);
|
||||
columnName = columnHeader.substring(idxTableSep+1);
|
||||
} else {
|
||||
table = mainTable;
|
||||
tableName = table.getTableName();
|
||||
columnName = columnHeader;
|
||||
}
|
||||
if (table == null)
|
||||
throwCSVHeaderNotFound(csvHeader);
|
||||
MColumn column = table.getColumn(columnName);
|
||||
if (column == null)
|
||||
throwCSVHeaderNotFound(csvHeader);
|
||||
String foreignTableName = null;
|
||||
if (foreignColumnName != null) {
|
||||
foreignTableName = column.getReferenceTableName();
|
||||
MTable foreignTable = MTable.get(getCtx(), foreignTableName);
|
||||
if (foreignTable == null)
|
||||
throwCSVHeaderNotFound(csvHeader);
|
||||
column = foreignTable.getColumn(foreignColumnName);
|
||||
if (column == null)
|
||||
throwCSVHeaderNotFound(csvHeader);
|
||||
}
|
||||
if (log.isLoggable(Level.INFO))
|
||||
log.info("CSV Header column found -> " + csvHeader + " -> " + tableName + "." + columnName
|
||||
+ (foreignColumnName != null ? "[" + foreignTableName + "." + foreignColumnName + "]" : "") );
|
||||
int ref = column.getAD_Reference_ID();
|
||||
retValue.add(ref);
|
||||
}
|
||||
return retValue;
|
||||
}
|
||||
|
||||
/**
|
||||
* Throws an exception indicating a CSV Header that could not be found
|
||||
* @param csvHeader
|
||||
*/
|
||||
private void throwCSVHeaderNotFound(String csvHeader) {
|
||||
throw new AdempiereException(Msg.getMsg(getCtx(), "CSVHeaderColumnNotFound", new Object[]{csvHeader}));
|
||||
}
|
||||
|
||||
@Override
|
||||
public MImportTemplate markImmutable() {
|
||||
if (is_Immutable())
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ public class X_AD_ImportTemplate extends PO implements I_AD_ImportTemplate, I_Pe
|
|||
/**
|
||||
*
|
||||
*/
|
||||
private static final long serialVersionUID = 20231222L;
|
||||
private static final long serialVersionUID = 20240327L;
|
||||
|
||||
/** Standard Constructor */
|
||||
public X_AD_ImportTemplate (Properties ctx, int AD_ImportTemplate_ID, String trxName)
|
||||
|
|
@ -44,7 +44,13 @@ public class X_AD_ImportTemplate extends PO implements I_AD_ImportTemplate, I_Pe
|
|||
setAD_Window_ID (0);
|
||||
setCharacterSet (null);
|
||||
setCSVHeader (null);
|
||||
setImportTemplateType (null);
|
||||
// CSV
|
||||
setName (null);
|
||||
setQuoteChar (null);
|
||||
// "
|
||||
setSeparatorChar (null);
|
||||
// ,
|
||||
} */
|
||||
}
|
||||
|
||||
|
|
@ -59,7 +65,13 @@ public class X_AD_ImportTemplate extends PO implements I_AD_ImportTemplate, I_Pe
|
|||
setAD_Window_ID (0);
|
||||
setCharacterSet (null);
|
||||
setCSVHeader (null);
|
||||
setImportTemplateType (null);
|
||||
// CSV
|
||||
setName (null);
|
||||
setQuoteChar (null);
|
||||
// "
|
||||
setSeparatorChar (null);
|
||||
// ,
|
||||
} */
|
||||
}
|
||||
|
||||
|
|
@ -74,7 +86,13 @@ public class X_AD_ImportTemplate extends PO implements I_AD_ImportTemplate, I_Pe
|
|||
setAD_Window_ID (0);
|
||||
setCharacterSet (null);
|
||||
setCSVHeader (null);
|
||||
setImportTemplateType (null);
|
||||
// CSV
|
||||
setName (null);
|
||||
setQuoteChar (null);
|
||||
// "
|
||||
setSeparatorChar (null);
|
||||
// ,
|
||||
} */
|
||||
}
|
||||
|
||||
|
|
@ -89,7 +107,13 @@ public class X_AD_ImportTemplate extends PO implements I_AD_ImportTemplate, I_Pe
|
|||
setAD_Window_ID (0);
|
||||
setCharacterSet (null);
|
||||
setCSVHeader (null);
|
||||
setImportTemplateType (null);
|
||||
// CSV
|
||||
setName (null);
|
||||
setQuoteChar (null);
|
||||
// "
|
||||
setSeparatorChar (null);
|
||||
// ,
|
||||
} */
|
||||
}
|
||||
|
||||
|
|
@ -274,6 +298,30 @@ public class X_AD_ImportTemplate extends PO implements I_AD_ImportTemplate, I_Pe
|
|||
return (String)get_Value(COLUMNNAME_Description);
|
||||
}
|
||||
|
||||
/** ImportTemplateType AD_Reference_ID=200268 */
|
||||
public static final int IMPORTTEMPLATETYPE_AD_Reference_ID=200268;
|
||||
/** CSV = CSV */
|
||||
public static final String IMPORTTEMPLATETYPE_CSV = "CSV";
|
||||
/** XLS = XLS */
|
||||
public static final String IMPORTTEMPLATETYPE_XLS = "XLS";
|
||||
/** XLSX = XLSX */
|
||||
public static final String IMPORTTEMPLATETYPE_XLSX = "XLSX";
|
||||
/** Set Import Template Type.
|
||||
@param ImportTemplateType Import Template Type
|
||||
*/
|
||||
public void setImportTemplateType (String ImportTemplateType)
|
||||
{
|
||||
|
||||
set_Value (COLUMNNAME_ImportTemplateType, ImportTemplateType);
|
||||
}
|
||||
|
||||
/** Get Import Template Type.
|
||||
@return Import Template Type */
|
||||
public String getImportTemplateType()
|
||||
{
|
||||
return (String)get_Value(COLUMNNAME_ImportTemplateType);
|
||||
}
|
||||
|
||||
/** Set Name.
|
||||
@param Name Alphanumeric identifier of the entity
|
||||
*/
|
||||
|
|
@ -297,4 +345,34 @@ public class X_AD_ImportTemplate extends PO implements I_AD_ImportTemplate, I_Pe
|
|||
{
|
||||
return new KeyNamePair(get_ID(), getName());
|
||||
}
|
||||
|
||||
/** Set Quote Char.
|
||||
@param QuoteChar Quote Char
|
||||
*/
|
||||
public void setQuoteChar (String QuoteChar)
|
||||
{
|
||||
set_Value (COLUMNNAME_QuoteChar, QuoteChar);
|
||||
}
|
||||
|
||||
/** Get Quote Char.
|
||||
@return Quote Char */
|
||||
public String getQuoteChar()
|
||||
{
|
||||
return (String)get_Value(COLUMNNAME_QuoteChar);
|
||||
}
|
||||
|
||||
/** Set Separator Character.
|
||||
@param SeparatorChar Separator Character
|
||||
*/
|
||||
public void setSeparatorChar (String SeparatorChar)
|
||||
{
|
||||
set_Value (COLUMNNAME_SeparatorChar, SeparatorChar);
|
||||
}
|
||||
|
||||
/** Get Separator Character.
|
||||
@return Separator Character */
|
||||
public String getSeparatorChar()
|
||||
{
|
||||
return (String)get_Value(COLUMNNAME_SeparatorChar);
|
||||
}
|
||||
}
|
||||
|
|
@ -352,7 +352,7 @@ public class CSVImportAction implements EventListener<Event>
|
|||
|
||||
String iMode = (String)importItem.getValue();
|
||||
m_file_istream = theTemplate.validateFile(m_file_istream);
|
||||
File outFile = theCSVImporter.fileImport(panel.getActiveGridTab(), childs, m_file_istream, charset,iMode);
|
||||
File outFile = theCSVImporter.fileImport(panel.getActiveGridTab(), childs, m_file_istream, charset, iMode, theTemplate.getSeparatorChar(), theTemplate.getQuoteChar(), null);
|
||||
winImportFile.onClose();
|
||||
winImportFile = null;
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue