midsuit
/
core-jgi
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

IDEMPIERE-454 Easy import - improve memory usage and speed

This commit is contained in:
Juan David Arboleda 2013-04-12 13:41:00 -05:00
parent 483fa5e6fd
commit 08d0c19de4
2 changed files with 75 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

89
org.adempiere.base/src/org/adempiere/impexp/GridTabCSVImporter.java
View File

@ -94,6 +94,7 @@ public class GridTabCSVImporter implements IGridTabImporter
String delimiter = String.valueOf((char) csvpref.getDelimiterChar()); String delimiter = String.valueOf((char) csvpref.getDelimiterChar());
String quoteChar = String.valueOf((char) csvpref.getQuoteChar()); String quoteChar = String.valueOf((char) csvpref.getQuoteChar());
m_import_mode = importMode; m_import_mode = importMode;
PO masterRecord = null;
if(!gridTab.isInsertRecord() && isInsertMode()) if(!gridTab.isInsertRecord() && isInsertMode())
throw new AdempiereException("Insert record disabled for Tab"); throw new AdempiereException("Insert record disabled for Tab");
@ -292,8 +293,7 @@ public class GridTabCSVImporter implements IGridTabImporter
// write the header // write the header
logFileW.write(rawHeader + delimiter + LOG_HEADER + "\n"); logFileW.write(rawHeader + delimiter + LOG_HEADER + "\n");
// no errors found - process header and then details // no errors found - process header and then details
boolean isMasterok = true; boolean isMasterok = true;
PO masterRecord = null;
String trxName= null; String trxName= null;
for (int idx = 0; idx < data.size(); idx++) { for (int idx = 0; idx < data.size(); idx++) {
String rawLine = rawData.get(idx); String rawLine = rawData.get(idx);
@ -302,12 +302,20 @@ public class GridTabCSVImporter implements IGridTabImporter
Trx trx = null; Trx trx = null;
GridTab currentGridTab=null; GridTab currentGridTab=null;
boolean error=false; boolean error=false;
boolean isDetail=false ; boolean isDetail=false;
int currentColumn=0; int currentColumn=0;
if (rawLine.charAt(0)==',') if (rawLine.charAt(0)==','){
isDetail=true; isDetail=true;
//check out if master row comes empty
Map<String, Object> rowMap = data.get(idx);
for(int i=0; i < indxDetail-1; i++){
if(rowMap.get(header.get(i))!=null){
isDetail=false;
break;
}
}
}
//if master failed, then ignore any detail from it. //if master failed, then ignore any detail from it.
if (!isMasterok && isDetail){ if (!isMasterok && isDetail){
rawLine = rawLine + delimiter + quoteChar + Msg.getMsg(Env.getCtx(),"NotProccesed") + quoteChar + "\n"; rawLine = rawLine + delimiter + quoteChar + Msg.getMsg(Env.getCtx(),"NotProccesed") + quoteChar + "\n";
@ -316,7 +324,6 @@ public class GridTabCSVImporter implements IGridTabImporter
} }
try { try {
Map<String, Object> map = data.get(idx);
isMasterok = true; isMasterok = true;
if(!isDetail){ if(!isDetail){
masterRecord = null; masterRecord = null;
@ -335,21 +342,24 @@ public class GridTabCSVImporter implements IGridTabImporter
currentGridTab.getTableModel().setImportingMode(true,trxName); currentGridTab.getTableModel().setImportingMode(true,trxName);
trx = Trx.get(trxName,true); trx = Trx.get(trxName,true);
int j = tabIndex.getValue(); int j = tabIndex.getValue();
logMsg = areValidKeysAndColumns(currentGridTab,map,header,currentColumn,j,masterRecord); logMsg = areValidKeysAndColumns(currentGridTab,data.get(idx),header,currentColumn,j,masterRecord);
if (logMsg == null){ if (logMsg == null){
if (isInsertMode()){ if (isInsertMode()){
if(!currentGridTab.getTableModel().isOpen()) if(!currentGridTab.getTableModel().isOpen())
currentGridTab.getTableModel().open(0); currentGridTab.getTableModel().open(0);
//how to read from status since the warning is coming empty ? //how to read from status since the warning is coming empty ?
if (!currentGridTab.dataNew(false)) if (!currentGridTab.dataNew(false)){
logMsg = "["+currentGridTab.getName()+"]"+"- Was not able to create a new record!"; logMsg = "["+currentGridTab.getName()+"]"+"- Was not able to create a new record!";
}else{
currentGridTab.navigateCurrent();
}
} }
if(logMsg==null) if(logMsg==null)
logMsg = proccessRow(currentGridTab,header,map,currentColumn,j,masterRecord); logMsg = proccessRow(currentGridTab,header,data.get(idx),currentColumn,j,masterRecord);
currentColumn = j + 1; currentColumn = j + 1;
if(!(logMsg == null)){ if(!(logMsg == null)){
@ -357,7 +367,6 @@ public class GridTabCSVImporter implements IGridTabImporter
//Ignore row since there is no data //Ignore row since there is no data
if("NO_DATA_TO_IMPORT".equals(logMsg)){ if("NO_DATA_TO_IMPORT".equals(logMsg)){
logMsg =""; logMsg ="";
currentGridTab.dataIgnore();
continue; continue;
}else }else
error =true; error =true;
@ -379,11 +388,9 @@ public class GridTabCSVImporter implements IGridTabImporter
logMsg = Msg.getMsg(Env.getCtx(), "Updated")+" "+ po.toString(); logMsg = Msg.getMsg(Env.getCtx(), "Updated")+" "+ po.toString();
trx.commit(); trx.commit();
if(isDetail || currentGridTab.isDetail()){ //if it's parent record and there are details, parent needs to be refreshed
currentGridTab.refreshParentTabs(); if(currentGridTab.equals(gridTab) && sortedtTabMapIndexes.size()>1)
currentGridTab.getTableModel().dataRequery(masterRecord.get_WhereClause(true), false, 0); currentGridTab.dataRefresh(false);
}
currentGridTab.dataRefresh(true);
} else { } else {
ValueNamePair ppE = CLogger.retrieveWarning(); ValueNamePair ppE = CLogger.retrieveWarning();
if (ppE==null) if (ppE==null)
@ -428,11 +435,15 @@ public class GridTabCSVImporter implements IGridTabImporter
} }
} catch (Exception e) { } catch (Exception e) {
rowResult.append("<"+currentGridTab.getTableName()+">: "); rowResult.append("<"+currentGridTab.getTableName()+">: ");
rowResult.append(Msg.getMsg(Env.getCtx(), "Error") + " " + e.getLocalizedMessage()); rowResult.append(Msg.getMsg(Env.getCtx(), "Error") + " " + e);
rowResult.append(" / "); rowResult.append(" / ");
currentGridTab.dataIgnore(); currentGridTab.dataIgnore();
trx.rollback(); trx.rollback();
error = true; error = true;
//Master Failed, thus details cannot be imported
if(currentGridTab.equals(gridTab) && masterRecord==null){
isMasterok = false;
}
} finally { } finally {
if (trx != null) { if (trx != null) {
trx.close(); trx.close();
@ -463,9 +474,11 @@ public class GridTabCSVImporter implements IGridTabImporter
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
} }
} }
gridTab.getTableModel().dataRequery("1=1",false,0); if(masterRecord!=null){
gridTab.dataRefreshAll(); Env.setContext(Env.getCtx(),gridTab.getWindowNo(),gridTab.getTabNo(),GridTab.CTX_SQL,masterRecord.get_WhereClause(true));
gridTab.dataRefresh();
}
if (logFile != null) if (logFile != null)
return logFile; return logFile;
else else
@ -649,7 +662,12 @@ public class GridTabCSVImporter implements IGridTabImporter
for(int j= i;j< header.size();j++){ for(int j= i;j< header.size();j++){
if(!header.get(j).contains(MTable.getTableName(Env.getCtx(),MLocation.Table_ID))) if(!header.get(j).contains(MTable.getTableName(Env.getCtx(),MLocation.Table_ID)))
break; break;
//validate if location contains its parent table
if(!header.get(j).contains(gridTab.getTableName()))
return new StringBuilder().append("Invalid location column's name, it must contain its parent table: ")
.append(gridTab.getTableName().toString())
.append("[").append(header.get(j)).append("]");
String columnName = header.get(j); String columnName = header.get(j);
Object value = tmpRow.get(j); Object value = tmpRow.get(j);
if(value!=null){ if(value!=null){
@ -711,12 +729,13 @@ public class GridTabCSVImporter implements IGridTabImporter
if(header.get(i).contains(MTable.getTableName(Env.getCtx(),MLocation.Table_ID))){ if(header.get(i).contains(MTable.getTableName(Env.getCtx(),MLocation.Table_ID))){
if(address == null){ if(address == null){
String trxName= (masterRecord==null?null:masterRecord.get_TrxName());
if(isInsertMode()){ if(isInsertMode()){
address = new MLocation (Env.getCtx(),0,masterRecord.get_TrxName()); address = new MLocation (Env.getCtx(),0,trxName);
}else{ }else{
Object location = gridTab.getValue("C_Location_ID")==null?0:gridTab.getValue("C_Location_ID").toString(); Object location = gridTab.getValue("C_Location_ID")==null?0:gridTab.getValue("C_Location_ID").toString();
int C_Location_ID = Integer.parseInt(location.toString()); int C_Location_ID = Integer.parseInt(location.toString());
address = new MLocation (Env.getCtx(),C_Location_ID,masterRecord.get_TrxName()); address = new MLocation (Env.getCtx(),C_Location_ID,trxName);
} }
} }
@ -784,9 +803,9 @@ public class GridTabCSVImporter implements IGridTabImporter
if(!field.isDisplayed(true)) if(!field.isDisplayed(true))
continue; continue;
if (!isInsertMode() && !field.isEditable(true) && value!=null) { if(!isInsertMode() && (!field.isEditable(true) && !MColumn.get(Env.getCtx(), field.getAD_Column_ID()).isUpdateable()) && value!=null){
logMsg = Msg.getMsg(Env.getCtx(), "FieldNotEditable", new Object[] {header.get(i)}) + "{" + value + "}"; logMsg = Msg.getMsg(Env.getCtx(), "FieldNotEditable", new Object[] {header.get(i)}) + "{" + value + "}";
break; break;
} }
if("(null)".equals(value.toString().trim())){ if("(null)".equals(value.toString().trim())){
logMsg = gridTab.setValue(field,null); logMsg = gridTab.setValue(field,null);
@ -847,6 +866,8 @@ public class GridTabCSVImporter implements IGridTabImporter
logMsg = gridTab.setValue("C_Location_ID",address.get_ID()); logMsg = gridTab.setValue("C_Location_ID",address.get_ID());
if(logMsg.equals("")) if(logMsg.equals(""))
logMsg= null; logMsg= null;
else
return logMsg;
isThereRow =true; isThereRow =true;
} }
@ -981,7 +1002,7 @@ public class GridTabCSVImporter implements IGridTabImporter
if (gridTab.isDetail()){ if (gridTab.isDetail()){
for(String linkColumn : gridTab.getParentColumnNames()){ for(String linkColumn : gridTab.getParentColumnNames()){
if(!pquery.getWhereClause().contains(linkColumn)){ if(!pquery.getWhereClause().contains(linkColumn)){
Object value = masterRecord.get_Value(linkColumn); Object value = masterRecord!=null?masterRecord.get_Value(linkColumn):null;
//resolve key //resolve key
if(value==null){ if(value==null){
String columnName = null; String columnName = null;
@ -1017,8 +1038,7 @@ public class GridTabCSVImporter implements IGridTabImporter
} }
} }
} }
gridTab.getTableModel().dataRequery(pquery.getWhereClause(),false,0); gridTab.getTableModel().dataRequery(pquery.getWhereClause());
gridTab.setCurrentRow(0,true);
if (isInsertMode()){ if (isInsertMode()){
if(gridTab.getTableModel().getRowCount()>=1) if(gridTab.getTableModel().getRowCount()>=1)
logMsg = Msg.getMsg(Env.getCtx(), "AlreadyExists")+" "+pquery; logMsg = Msg.getMsg(Env.getCtx(), "AlreadyExists")+" "+pquery;
@ -1026,17 +1046,20 @@ public class GridTabCSVImporter implements IGridTabImporter
return null; return null;
} }
if (isUpdateMode()){ if (isUpdateMode()){
if(gridTab.getTableModel().getRowCount()==1) if(gridTab.getTableModel().getRowCount()==1){
gridTab.navigateRelative(gridTab.getCurrentRow());
return null; return null;
}
else if(gridTab.getTableModel().getRowCount()<=0) else if(gridTab.getTableModel().getRowCount()<=0)
logMsg = Msg.getMsg(Env.getCtx(), "not.found")+" "+pquery; logMsg = Msg.getMsg(Env.getCtx(), "not.found")+" "+pquery;
else if(gridTab.getTableModel().getRowCount()>1) else if(gridTab.getTableModel().getRowCount()>1)
logMsg = Msg.getMsg(Env.getCtx(),"TooManyRows")+" "+pquery; logMsg = Msg.getMsg(Env.getCtx(),"TooManyRows")+" "+pquery;
} }
if (isMergeMode()){ if (isMergeMode()){
if(gridTab.getTableModel().getRowCount()==1) if(gridTab.getTableModel().getRowCount()==1){
gridTab.navigateRelative(gridTab.getCurrentRow());
m_import_mode = IMPORT_MODE_UPDATE; m_import_mode = IMPORT_MODE_UPDATE;
else if(gridTab.getTableModel().getRowCount()<=0) }else if(gridTab.getTableModel().getRowCount()<=0)
m_import_mode = IMPORT_MODE_INSERT; m_import_mode = IMPORT_MODE_INSERT;
else if(gridTab.getTableModel().getRowCount()>1) else if(gridTab.getTableModel().getRowCount()>1)
logMsg = Msg.getMsg(Env.getCtx(),"TooManyRows")+" "+pquery; logMsg = Msg.getMsg(Env.getCtx(),"TooManyRows")+" "+pquery;

19
org.adempiere.base/src/org/compiere/model/GridTable.java
View File

@ -2936,6 +2936,25 @@ public class GridTable extends AbstractTableModel
return true; return true;
} // dataRequery } // dataRequery
/**
* Requery with new whereClause
* @param whereClause sql where clause
* @return true if success
*/
public boolean dataRequery (String whereClause)
{
if (log.isLoggable(Level.INFO)) log.info(whereClause + "; OnlyCurrent=" + 0);
close(false);
m_onlyCurrentDays = 0;
setSelectWhereClause(whereClause,false, m_onlyCurrentDays);
open(m_maxRows);
// Info
m_rowData = null;
m_changed = false;
m_rowChanged = -1;
m_inserting = false;
return true;
} // dataRequery
/************************************************************************** /**************************************************************************
* Is Cell Editable. * Is Cell Editable.