midsuit
/
core-jgi
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

IDEMPIERE-4889 Performance - Attachment management (#798) 

* IDEMPIERE-4889 Performance - Attachment management

* IDEMPIERE-4889 Performance - Attachment management

Implement suggestion from hengsin

* IDEMPIERE-4889 Performance - Attachment management
This commit is contained in:
Carlos Ruiz 2021-07-28 05:01:38 +02:00 committed by GitHub
parent 3e7dd1f7f7
commit 83b383788e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 202 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
org.adempiere.base.process/src/org/idempiere/process/MigrateStorageProvider.java
View File

@ -35,6 +35,7 @@ import org.compiere.model.IAttachmentStore;
import org.compiere.model.IImageStore;
import org.compiere.model.MArchive;
import org.compiere.model.MAttachment;
import org.compiere.model.MAttachmentEntry;
import org.compiere.model.MClient;
import org.compiere.model.MClientInfo;
import org.compiere.model.MImage;
@ -245,7 +246,9 @@ public class MigrateStorageProvider extends SvrProcess {
}
MAttachment attachment = new MAttachment(getCtx(), attachId, get_TrxName());
int oldProviderId = attachment.getAD_StorageProvider_ID();
attachment.getEntries();
for (MAttachmentEntry entry : attachment.getEntries()) {
entry.getData(); // force load in case old provider is delayed
}
attachment.setStorageProvider(newProvider);
attachment.set_ValueNoCheck("Updated", new Timestamp(System.currentTimeMillis())); // to force save
// create file on the new storage provider

77
org.adempiere.base/src/org/compiere/model/AttachmentFileLazyDataSource.java Normal file
View File

@ -0,0 +1,77 @@
/***********************************************************************
* This file is part of iDempiere ERP Open Source *
* http://www.idempiere.org *
* *
* Copyright (C) Contributors *
* *
* This program is free software; you can redistribute it and/or *
* modify it under the terms of the GNU General Public License *
* as published by the Free Software Foundation; either version 2 *
* of the License, or (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
* along with this program; if not, write to the Free Software *
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, *
* MA 02110-1301, USA. *
* *
* Sponsor: *
* - FH *
* Contributors: *
* - Carlos Ruiz *
**********************************************************************/
package org.compiere.model;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import org.compiere.util.CLogger;
/**
* IDEMPIERE-4889
* @author Carlos Ruiz - globalqss
*/
public class AttachmentFileLazyDataSource implements IAttachmentLazyDataSource {
private final CLogger log = CLogger.getCLogger(getClass());
private File m_file;
/**
* Constructor for lazy load - keep the file information
* @param file
*/
public AttachmentFileLazyDataSource(File file) {
m_file = file;
}
/**
* Return a byte array containing the data from the File
* @return
*/
@Override
public byte[] getData() {
// read files into byte[]
final byte[] dataEntry = new byte[(int) m_file.length()];
try {
final FileInputStream fileInputStream = new FileInputStream(m_file);
fileInputStream.read(dataEntry);
fileInputStream.close();
} catch (FileNotFoundException e) {
log.severe("File Not Found.");
e.printStackTrace();
} catch (IOException e1) {
log.severe("Error Reading The File.");
e1.printStackTrace();
}
return dataEntry;
}
}

19
org.adempiere.base/src/org/compiere/model/AttachmentFileSystem.java
View File

@ -17,7 +17,6 @@ import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.channels.FileChannel;
@ -204,21 +203,9 @@ public class AttachmentFileSystem implements IAttachmentStore {
if (log.isLoggable(Level.FINE)) log.fine("filePath: " + filePath);
final File file = new File(filePath);
if (file.exists()) {
// read files into byte[]
final byte[] dataEntry = new byte[(int) file.length()];
try {
final FileInputStream fileInputStream = new FileInputStream(file);
fileInputStream.read(dataEntry);
fileInputStream.close();
} catch (FileNotFoundException e) {
log.severe("File Not Found.");
e.printStackTrace();
} catch (IOException e1) {
log.severe("Error Reading The File.");
e1.printStackTrace();
}
final MAttachmentEntry entry = new MAttachmentEntry(file.getName(),
dataEntry, attach.m_items.size() + 1);
// file data read delayed
IAttachmentLazyDataSource ds = new AttachmentFileLazyDataSource(file);
final MAttachmentEntry entry = new MAttachmentEntry(file.getName(), attach.m_items.size() + 1, ds);
attach.m_items.add(entry);
} else {
log.severe("file not found: " + file.getAbsolutePath());

43
org.adempiere.base/src/org/compiere/model/IAttachmentLazyDataSource.java Normal file
View File

@ -0,0 +1,43 @@
/***********************************************************************
* This file is part of iDempiere ERP Open Source *
* http://www.idempiere.org *
* *
* Copyright (C) Contributors *
* *
* This program is free software; you can redistribute it and/or *
* modify it under the terms of the GNU General Public License *
* as published by the Free Software Foundation; either version 2 *
* of the License, or (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
* along with this program; if not, write to the Free Software *
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, *
* MA 02110-1301, USA. *
* *
* Sponsor: *
* - FH *
* Contributors: *
* - Carlos Ruiz *
**********************************************************************/
package org.compiere.model;
/**
* IDEMPIERE-4889
* @author Carlos Ruiz - globalqss
*/
public interface IAttachmentLazyDataSource {
/**
* Return a byte array containing the data from the Attachment Entry
* Usually the implementing class must have a constructor with the variable(s) required for loading later the data
* @return
*/
public byte[] getData();
}

1
org.adempiere.base/src/org/compiere/model/MAttachment.java
View File

@ -318,6 +318,7 @@ public class MAttachment extends X_AD_Attachment
boolean retValue = false;
if (item == null)
return false;
item.getData(); // in case of lazy load enforce reading
if (m_items == null)
loadLOBData();
for (int i = 0; i < m_items.size(); i++) {

108
org.adempiere.base/src/org/compiere/model/MAttachmentEntry.java
View File

@ -48,18 +48,7 @@ public class MAttachmentEntry
super ();
setName (name);
setData (data);
if (index > 0)
m_index = index;
else
{
long now = System.currentTimeMillis();
if (s_seed+3600000l < now) // older then 1 hour
{
s_seed = now;
s_random = new Random(s_seed);
}
m_index = s_random.nextInt();
}
setIndex(index);
} // MAttachmentItem
/**
@ -72,12 +61,27 @@ public class MAttachmentEntry
this (name, data, 0);
} // MAttachmentItem
/**
* Constructor for delayed load
* @param parent
* @param name
* @param index
* @param lazy data source
*/
public MAttachmentEntry (String name, int index, IAttachmentLazyDataSource ds) {
super ();
setName (name);
setIndex(index);
setLazyDataSource(ds);
}
/**
* Copy constructor
* @param copy
*/
public MAttachmentEntry(MAttachmentEntry copy)
{
this.m_isDataSet = copy.m_isDataSet;
this.m_data = copy.m_data != null ? Arrays.copyOf(copy.m_data, copy.m_data.length) : null;
this.m_index = copy.m_index;
this.m_name = copy.m_name;
@ -85,7 +89,10 @@ public class MAttachmentEntry
/** The Name */
private String m_name = "?";
/** The Data */
/** If m_data has been set */
private boolean m_isDataSet = false;
/** The Data, do not use m_data directly, it can be not loaded yet, always use the method getData to access this variable */
private byte[] m_data = null;
/** Random Seed */
@ -97,13 +104,18 @@ public class MAttachmentEntry
/** Logger */
protected CLogger log = CLogger.getCLogger(getClass());
/** Lazy Data Source */
private IAttachmentLazyDataSource m_ds = null;
/**
* @return Returns the data.
*/
public byte[] getData ()
{
if (! m_isDataSet && m_ds != null) {
setData(m_ds.getData());
}
return m_data;
}
/**
@ -112,6 +124,7 @@ public class MAttachmentEntry
public void setData (byte[] data)
{
m_data = data;
m_isDataSet = true;
}
/**
* @return Returns the name.
@ -134,7 +147,7 @@ public class MAttachmentEntry
/**
* Get Attachment Index
* @return timestamp
* @return int index
*/
public int getIndex()
{
@ -157,13 +170,13 @@ public class MAttachmentEntry
public String toStringX ()
{
StringBuilder sb = new StringBuilder (m_name);
if (m_data != null)
if (getData() != null)
{
sb.append(" (");
//
float size = m_data.length;
float size = getData().length;
if (size <= 1024)
sb.append(m_data.length).append(" B");
sb.append(getData().length).append(" B");
else
{
size /= 1024;
@ -190,34 +203,34 @@ public class MAttachmentEntry
{
StringBuilder hdr = new StringBuilder("----- ").append(getName()).append(" -----");
System.out.println (hdr.toString());
if (m_data == null)
if (getData() == null)
{
System.out.println ("----- no data -----");
return;
}
// raw data
for (int i = 0; i < m_data.length; i++)
for (int i = 0; i < getData().length; i++)
{
char data = (char)m_data[i];
char data = (char)getData()[i];
System.out.print(data);
}
System.out.println ();
System.out.println (hdr.toString());
// Count nulls at end
int ii = m_data.length -1;
int ii = getData().length -1;
int nullCount = 0;
while (m_data[ii--] == 0)
while (getData()[ii--] == 0)
nullCount++;
StringBuilder msgout = new StringBuilder("----- Length=").append(m_data.length).append(", EndNulls=").append(nullCount)
.append(", RealLength=").append((m_data.length-nullCount));
StringBuilder msgout = new StringBuilder("----- Length=").append(getData().length).append(", EndNulls=").append(nullCount)
.append(", RealLength=").append((getData().length-nullCount));
System.out.println(msgout.toString());
/**
// Dump w/o nulls
if (nullCount > 0)
{
for (int i = 0; i < m_data.length-nullCount; i++)
System.out.print((char)m_data[i]);
for (int i = 0; i < getData().length-nullCount; i++)
System.out.print((char)getData()[i]);
System.out.println ();
System.out.println (hdr);
}
@ -252,12 +265,12 @@ public class MAttachmentEntry
*/
public File getFile (File file)
{
if (m_data == null || m_data.length == 0)
if (getData() == null || getData().length == 0)
return null;
try
{
FileOutputStream fos = new FileOutputStream(file);
fos.write(m_data);
fos.write(getData());
fos.close();
}
catch (IOException ioe)
@ -303,13 +316,40 @@ public class MAttachmentEntry
*/
public InputStream getInputStream()
{
if (m_data == null)
if (getData() == null)
return null;
return new ByteArrayInputStream(m_data);
return new ByteArrayInputStream(getData());
} // getInputStream
public void setIndex(int index) {
m_index = index;
if (index > 0)
m_index = index;
else
{
long now = System.currentTimeMillis();
if (s_seed+3600000l < now) // older then 1 hour
{
s_seed = now;
s_random = new Random(s_seed);
}
m_index = s_random.nextInt();
}
}
/**
* Set the lazy data source
* @param obj
*/
public void setLazyDataSource(IAttachmentLazyDataSource ds) {
m_ds = ds;
}
/**
* Get the lazy data source
* @return
*/
public IAttachmentLazyDataSource getLazyDataSource() {
return m_ds;
}
} // MAttachmentItem