原创文章，转载请注明出处！
------------------------------------------------------------
今天接到一个项目需求变更，是关于从数据库查询到30W条数据放到一个List中，然后使用StringBuffer把List取出放入到StringBuffer中。最后使用 BufferedWriter.write(buffer.toString());添加到文件中，此时会消耗几个小时才能把数据存放到文件中。
项目源码：
public void generateCommonSMSFile(String localFilePath,String date) throws Exception{
try {
List smsList = getSingleDataSMS("03",date);
List listid = new ArrayList();
List subList = new ArrayList();
StringBuffer buffer = new StringBuffer("S|2|S0355||||");
buffer.append(smsList.size() + "|");
buffer.append("\r\n");
if(smsList != null && smsList.size() != 0) {
for(int i = 0;i < smsList.size();i++) {
OmsSingleDataDTO msdd = (OmsSingleDataDTO)smsList.get(i);
String transDate = msdd.getTransDate();
String leastDate = msdd.getLeastDate();
buffer.append(String.valueOf(i+1));
buffer.append("|");
buffer.append(msdd.getMobile());
buffer.append("|");
buffer.append(msdd.getCustNo());
buffer.append("|");
buffer.append(msdd.getAdvMemo1());
buffer.append("|");
buffer.append(CommonUtil.deleteZero(transDate.substring(5, 7))+"月"+CommonUtil.deleteZero(transDate.substring(8))+"日");
buffer.append("|");
buffer.append(msdd.getTransAmount());
buffer.append("|");
buffer.append(CommonUtil.deleteZero(leastDate.substring(4, 6))+"月"+CommonUtil.deleteZero(leastDate.substring(6))+"日");
buffer.append("|");
buffer.append("FQ"+msdd.getConsumeCode());
buffer.append("|");
buffer.append(msdd.getInstNum());
buffer.append("|");
buffer.append(msdd.getFirstAmount());
buffer.append("|");
buffer.append(msdd.getEachAmount());
buffer.append("|");
buffer.append(msdd.getTotalPoundage());
buffer.append("|");
buffer.append(msdd.getAdvMemo2());
buffer.append("|");
buffer.append("\r\n");
subList.add(msdd.getSeqId());
if((i+1)%5000==0){
listid.add(subList);
subList=new ArrayList();
}
}
listid.add(subList);
}
File file1 = FileUtils.createFile(localFilePath);
BufferedWriter bw = FileUtils.getWrite(file1);
bw.write(buffer.toString());
bw.close();
if(listid != null && listid.size() >= 1) {
for(int i=0;i<listid.size();i++) {
OMSLogger.info("普通消费类 - 第"+ (i+1) +"次运行开始");
long msBefore = System.currentTimeMillis();
updateMosSingleData((List)listid.get(i));
long msAfter = System.currentTimeMillis();
OMSLogger.info("普通消费类 - 第"+ (i+1) +"次运行结束，共耗时" + (msAfter - msBefore) + "毫秒");
}
}
} catch (Exception e) {
OMSLogger.error("生成单笔分期批量短信文件异常");
throw new Exception("生成单笔分期批量短信文件异常",e);
}
}
查询数据库，返回list供上面的数据写入文件的方法使用
private List getSingleDataSMS(String instFlag,String vdate) throws PafaDAOException {
List SmsList = new ArrayList();
List list = new ArrayList();
OmsSingleDataDTO msdd = null;
// list = executeSqlDao.queryForList("SELECT-SMS-SingleData-LIST",null);
HashMap map =new HashMap();
map.put("instFlag", instFlag);
map.put("vdate", vdate);
int total = 0;
total = ((Integer)executeSqlDao.queryForObject("SELECT-SMS-SingleData-COUNT", map)).intValue();
// 防止大于1万条时出错,循环处理
for(int i=0; i<total; i+=5000){
HashMap paramMap =new HashMap();
List rs = new ArrayList();
paramMap.put("startNum", String.valueOf(i+1));
paramMap.put("endNum", String.valueOf(i+5000));
paramMap.put("instFlag", instFlag);
paramMap.put("vDate", vdate);
rs = executeSqlDao.queryForList("SELECT-SMS-SingleData-LIST",paramMap);
if (rs!=null){
for(int j=0; j<rs.size(); j++){
list.add(rs.get(j));
}
}
}
if(list != null && list.size() >= 1) {
Set set = commonService.getLogoValue(null);
for(int m = 0;m < list.size();m++) {
msdd = (OmsSingleDataDTO)list.get(m);
if(!OrderBaseService.getLogIsValid(msdd.getCardNo(),"14",set)) {
// 更新处理状态
this.updateMosSingleData2(msdd.getSeqId());
continue;
}
String custNo = msdd.getCustNo();
custNo = custNo.length() >= 12 ? custNo.substring(custNo.length() - 12):custNo;
msdd.setCustNo(custNo);
SmsList.add(msdd);
}
}
return SmsList;
}
以上是使用的一次全部查询出数据，然后一次写入到文件中，但是这种方法会用几个小时才能完全写入。下面是自己优化后的方案：
public void generateSMSFile(String localFilePath,String date) throws Exception{
try {
File file1 = FileUtils.createFile(localFilePath);
BufferedWriter bw = FileUtils.getWrite(file1);
StringBuffer buffer = new StringBuffer("S|1|S0180||||");
List smslistCount = getSingleDataSMS("03", date);
buffer.append(smslistCount.size() + "|");
buffer.append("\r\n");
List subList = new ArrayList();
List listid = new ArrayList();
int count = 1;
int total = 0;
total = generateSMSFilestpeSize(date);
//防止大于1万条时出错,循环处理
for(int k=0; k<total; k+=5000){
List smsList = getSingleDataSMSstep("03", date, k, k+5000);
if(smsList != null && smsList.size() != 0) {
for(int i = 0;i < smsList.size();i++) {
OmsSingleDataDTO msdd = (OmsSingleDataDTO)smsList.get(i);
String transDate = msdd.getTransDate();
String leastDate = msdd.getLeastDate();
count = count +1;
buffer.append(String.valueOf(count));
buffer.append("|");
buffer.append(msdd.getMobile());
buffer.append("|");
buffer.append(msdd.getCustNo());
buffer.append("|");
buffer.append(transDate.substring(5,7));
buffer.append("|");
buffer.append(transDate.substring(8,10));
buffer.append("|");
buffer.append(msdd.getTransAmount());
buffer.append("|");
buffer.append(msdd.getFirstAmount());
buffer.append("|");
buffer.append(msdd.getEachAmount());
buffer.append("|");
buffer.append(leastDate.substring(4,6));
buffer.append("|");
buffer.append(leastDate.substring(6,8));
buffer.append("|");
buffer.append(msdd.getConsumeCode());
buffer.append("|");
buffer.append("\r\n");
subList.add(msdd.getSeqId());
if((i+1)%5000==0){
listid.add(subList);
subList=new ArrayList();
}
}
listid.add(subList);
}
bw.write(buffer.toString());
buffer = new StringBuffer();
if(listid != null && listid.size() >= 1) {
for(int i=0;i<listid.size();i++) {
OMSLogger.info("保费类 - 第"+ (count) +"次运行开始");
long msBefore = System.currentTimeMillis();
updateMosSingleData((List)listid.get(i));
long msAfter = System.currentTimeMillis();
OMSLogger.info("保费类 - 第"+ (count) +"次运行结束，共耗时" + (msAfter - msBefore) + "毫秒");
}
}
}
bw.close();
} catch (Exception e) {
OMSLogger.error("生成单笔分期批量短信文件异常");
throw new Exception("生成单笔分期批量短信文件异常",e);
}
}
每次获取5000条数据返回list，写入文件之后再次执行该方法在获取5000条数据
private List getSingleDataSMSstep(String instFlag,String vdate,int startNum,int endNum) throws PafaDAOException {
List SmsList = new ArrayList();
List list = new ArrayList();
OmsSingleDataDTO msdd = null;
HashMap paramMap =new HashMap();
List rs = new ArrayList();
paramMap.put("startNum", startNum);
paramMap.put("endNum", endNum);
paramMap.put("instFlag", instFlag);
paramMap.put("vDate", vdate);
rs = executeSqlDao.queryForList("SELECT-SMS-SingleData-LIST",paramMap);
if (rs!=null){
for(int j=0; j<rs.size(); j++){
list.add(rs.get(j));
}
}
if(list != null && list.size() >= 1) {
Set set = commonService.getLogoValue(null);
for(int m = 0;m < list.size();m++) {
msdd = (OmsSingleDataDTO)list.get(m);
if(!OrderBaseService.getLogIsValid(msdd.getCardNo(),"14",set)) {
// 更新处理状态
this.updateMosSingleData2(msdd.getSeqId());
continue;
}
String custNo = msdd.getCustNo();
custNo = custNo.length() >= 12 ? custNo.substring(custNo.length() - 12):custNo;
msdd.setCustNo(custNo);
SmsList.add(msdd);
}
}
return SmsList;
}
补充一个自己写的小例子：
package myTempTest; import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException; public class ioTestBuffer {
public static void main(String[] args) throws IOException {
File file1 = new File("d:\\io\\out.txt");//最终写入文件地址
BufferedWriter bw = new BufferedWriter(new FileWriter(file1));
StringBuffer buffer = new StringBuffer("");
for (int j = ; j < ; j++) {
//之所以I循环1W次就写出一次，是因为buffer超过1W次append之后极容易在次append的时候报错。
for (int i = ; i < ; i++) {
buffer.append(i+""+i+""+i+""+i+""+i+""+i+""+i+""+i+""+i+""+i+"");
buffer.append("\r\n");//换行
}
bw.write(buffer.toString());//1W次I循环结束 写入到文件中
buffer = new StringBuffer("");//格式化buffer。
buffer.append(j+"-------------------------------------");//第 j 此循环加上个记号
buffer.append("\r\n");//一次J循环结束 换行
}
bw.close();
}
}