关键词搜索

源码搜索 ×
×

Java 并发写文件加锁处理(阿里云OSS HTTP文件迁移)

发布2020-06-11浏览992次

详情内容

目录

文件追加的三种方式

OSS HTTP list.txt输出方式

阿里云OSS文件资源行数据迁移：

https://help.aliyun.com/document_detail/95134.html?spm=a2c4g.11186623.6.555.32755641l0V8YD

文件追加的三种方式

文件多线程写的情况下需要加锁处理： 提供三种方式写入文件内容。

  1.  /**
  2.      * 追加写入内容到目标文件
  3.      *
  4.      * @param dwCode
  5.      * @param rowsContent
  6.      * @param destFile
  7.      */
  8.     private synchronized void writeIntoFile(String dwCode,String rowsContent,File destFile){
  9.         log.info("writeIntoFile {} dts data......", dwCode);
  10.         OutputStream os = null;
  11.         try {
  12.             if (!destFile.exists()) {
  13.                 destFile.createNewFile();
  14.             }
  15.             // 追加写入内容
  16.             os = new FileOutputStream(destFile,true);
  17.             os.write(rowsContent.getBytes());
  18.             log.info("writeIntoFile {} dts data......success!", dwCode);
  19.         } catch (FileNotFoundException e) {
  20.             log.info("writeIntoFile {} dts data......FileNotFoundException!", dwCode);
  21.             e.printStackTrace();
  22.         } catch (IOException e) {
  23.             log.info("writeIntoFile {} dts data......IOException!", dwCode);
  24.             e.printStackTrace();
  25.         } finally {
  26.             if (null != os) {
  27.                 try {
  28.                     os.close();
  29.                 } catch (IOException e) {
  30.                     e.printStackTrace();
  31.                 }
  32.             }
  33.         }
  34.     }
  35.  
  36.     /**
  37.      * 追加写入内容到目标文件
  38.      *
  39.      * @param dwCode
  40.      * @param rowsContent
  41.      * @param destFile
  42.      */
  43.     private void writeIntoFileByBufferedWriter(String dwCode,String rowsContent,File destFile){
  44.         log.info("writeIntoFileByBufferedWriter {} dts data......", dwCode);
  45.         BufferedWriter out = null;
  46.         try {
  47.             if (!destFile.exists()) {
  48.                 destFile.createNewFile();
  49.             }
  50.             // 追加写入内容
  51.             out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(destFile, true)));
  52.             out.write(rowsContent);
  53.             log.info("writeIntoFileByBufferedWriter {} dts data......success!", dwCode);
  54.         } catch (FileNotFoundException e) {
  55.             log.info("writeIntoFileByBufferedWriter {} dts data......FileNotFoundException!", dwCode);
  56.             e.printStackTrace();
  57.         } catch (IOException e) {
  58.             log.info("writeIntoFileByBufferedWriter {} dts data......IOException!", dwCode);
  59.             e.printStackTrace();
  60.         } finally {
  61.             if(null != out){
  62.                 try {
  63.                     out.close();
  64.                 } catch (IOException e) {
  65.                     e.printStackTrace();
  66.                 }
  67.             }
  68.         }
  69.     }
  70.  
  71.     /**
  72.      * 追加写入内容到目标文件
  73.      *
  74.      * @param dwCode
  75.      * @param rowsContent
  76.      * @param destFile
  77.      */
  78.     private synchronized void writeIntoFileByFileWriter(String dwCode,String rowsContent,File destFile){
  79.         log.info("writeIntoFileByAppend {} dts data......", dwCode);
  80.         OutputStream os = null;
  81.         try {
  82.             if (!destFile.exists()) {
  83.                 destFile.createNewFile();
  84.             }
  85.             // 追加写入内容
  86.             FileWriter fileWriter =new FileWriter(destFile,true);
  87.             fileWriter.write(rowsContent);
  88.             log.info("writeIntoFileByAppend {} dts data......success!", dwCode);
  89.         } catch (FileNotFoundException e) {
  90.             log.info("writeIntoFileByAppend {} dts data......FileNotFoundException!", dwCode);
  91.             e.printStackTrace();
  92.         } catch (IOException e) {
  93.             log.info("writeIntoFileByAppend {} dts data......IOException!", dwCode);
  94.             e.printStackTrace();
  95.         } finally {
  96.             if (null != os) {
  97.                 try {
  98.                     os.close();
  99.                 } catch (IOException e) {
  100.                     e.printStackTrace();
  101.                 }
  102.             }
  103.         }
  104.     }

OSS HTTP list.txt输出方式

pom.xml

  1.   <dependency>
  2.             <groupId>com.aliyun.oss</groupId>
  3.             <artifactId>aliyun-sdk-oss</artifactId>
  4.             <version>3.8.0</version>
  5.         </dependency>

Java 代码

  1. package com.aliyun.service.oss.tds;
  2.  
  3. import com.aliyun.service.oss.service.BaseService;
  4. import lombok.extern.slf4j.Slf4j;
  5. import org.apache.commons.lang3.StringUtils;
  6. import org.springframework.beans.factory.annotation.Autowired;
  7. import org.springframework.beans.factory.annotation.Value;
  8. import org.springframework.stereotype.Service;
  9. import java.io.*;
  10. import java.util.List;
  11. import java.util.Map;
  12.  
  13. /**
  14.  * @Copyright: 2019-2021
  15.  * @FileName: JilinFileService.java
  16.  * @Author: PJL
  17.  * @Date: 2020/6/10 17:02
  18.  * @Description: **文件服务
  19.  */
  20. @Slf4j
  21. @Service
  22. public class FileService extends BaseService {
  23.  
  24.     @Value("${system.http-base-url}")
  25.     String httpBaseUrl;
  26.  
  27.     @Value("${system.local-save-base-path}")
  28.     String localSaveBasePath;
  29.  
  30.     @Autowired
  31.     FujianService fujianService;
  32.  
  33.     /**
  34.      * 解析文件路径
  35.      *
  36.      * @param map
  37.      * @param sb
  38.      * @param fieldName
  39.      */
  40.     private String parseFilePath(Map<String, Object> map,StringBuffer sb,String fieldName){
  41.         Object value = map.get(fieldName);
  42.         if(null == value){
  43.             return "";// 无值情况不予处理
  44.         }
  45.         String file = value.toString();
  46.         String httpUrl = new StringBuffer(httpBaseUrl).append("/").append(file).toString();
  47.         //String destUrl = new StringBuffer(dwCode).append("/").append(file).toString();
  48.         String destUrl = new StringBuffer(file).toString();
  49.         String rowContent = new StringBuffer(httpUrl).append("\t").append(destUrl).append("\n").toString();
  50.         sb.append(rowContent);
  51.         return destUrl;
  52.     }
  53.  
  54.     /**
  55.      * 解析并封装文件路径集合字符串内容
  56.      *
  57.      * @param dwCode
  58.      * @param dataList
  59.      */
  60.     private String parseDataListContent(String dwCode, List<Map<String, Object>> dataList) {
  61.         StringBuffer sb = new StringBuffer();
  62.         if (null != dataList && dataList.size() > 0) {
  63.             for (Map<String, Object> map : dataList) {
  64.                 // 原文件解析
  65.                 String destUrl = this.parseFilePath(map,sb,"FJ_LJ");
  66.                 // 缩略图解析
  67.                 if(StringUtils.isNotEmpty(destUrl) && !destUrl.contains(".mp4")){
  68.                     this.parseFilePath(map,sb,"FJ_LJ_COMPRES");
  69.                 }
  70.             }
  71.         }
  72.         return sb.toString();
  73.     }
  74.  
  75.     /**
  76.      * 创建路径
  77.      *
  78.      * @param filePath
  79.      */
  80.     private void createDir(String filePath) {
  81.         File dwDirFile = new File(filePath);
  82.         if (!dwDirFile.exists()) {
  83.             dwDirFile.mkdir();
  84.         }
  85.     }
  86.  
  87.     /**
  88.      * 追加写入内容到目标文件
  89.      *
  90.      * @param dwCode
  91.      * @param rowsContent
  92.      * @param destFile
  93.      */
  94.     private synchronized void writeIntoFile(String dwCode,String rowsContent,File destFile){
  95.         log.info("writeIntoFile {} dts data......", dwCode);
  96.         OutputStream os = null;
  97.         try {
  98.             if (!destFile.exists()) {
  99.                 destFile.createNewFile();
  100.             }
  101.             // 追加写入内容
  102.             os = new FileOutputStream(destFile,true);
  103.             os.write(rowsContent.getBytes());
  104.             log.info("writeIntoFile {} dts data......success!", dwCode);
  105.         } catch (FileNotFoundException e) {
  106.             log.info("writeIntoFile {} dts data......FileNotFoundException!", dwCode);
  107.             e.printStackTrace();
  108.         } catch (IOException e) {
  109.             log.info("writeIntoFile {} dts data......IOException!", dwCode);
  110.             e.printStackTrace();
  111.         } finally {
  112.             if (null != os) {
  113.                 try {
  114.                     os.close();
  115.                 } catch (IOException e) {
  116.                     e.printStackTrace();
  117.                 }
  118.             }
  119.         }
  120.     }
  121.  
  122.     /**
  123.      * 追加写入内容到目标文件
  124.      *
  125.      * @param dwCode
  126.      * @param rowsContent
  127.      * @param destFile
  128.      */
  129.     private void writeIntoFileByBufferedWriter(String dwCode,String rowsContent,File destFile){
  130.         log.info("writeIntoFileByBufferedWriter {} dts data......", dwCode);
  131.         BufferedWriter out = null;
  132.         try {
  133.             if (!destFile.exists()) {
  134.                 destFile.createNewFile();
  135.             }
  136.             // 追加写入内容
  137.             out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(destFile, true)));
  138.             out.write(rowsContent);
  139.             log.info("writeIntoFileByBufferedWriter {} dts data......success!", dwCode);
  140.         } catch (FileNotFoundException e) {
  141.             log.info("writeIntoFileByBufferedWriter {} dts data......FileNotFoundException!", dwCode);
  142.             e.printStackTrace();
  143.         } catch (IOException e) {
  144.             log.info("writeIntoFileByBufferedWriter {} dts data......IOException!", dwCode);
  145.             e.printStackTrace();
  146.         } finally {
  147.             if(null != out){
  148.                 try {
  149.                     out.close();
  150.                 } catch (IOException e) {
  151.                     e.printStackTrace();
  152.                 }
  153.             }
  154.         }
  155.     }
  156.  
  157.     /**
  158.      * 追加写入内容到目标文件
  159.      *
  160.      * @param dwCode
  161.      * @param rowsContent
  162.      * @param destFile
  163.      */
  164.     private synchronized void writeIntoFileByFileWriter(String dwCode,String rowsContent,File destFile){
  165.         log.info("writeIntoFileByAppend {} dts data......", dwCode);
  166.         OutputStream os = null;
  167.         try {
  168.             if (!destFile.exists()) {
  169.                 destFile.createNewFile();
  170.             }
  171.             // 追加写入内容
  172.             FileWriter fileWriter =new FileWriter(destFile,true);
  173.             fileWriter.write(rowsContent);
  174.             log.info("writeIntoFileByAppend {} dts data......success!", dwCode);
  175.         } catch (FileNotFoundException e) {
  176.             log.info("writeIntoFileByAppend {} dts data......FileNotFoundException!", dwCode);
  177.             e.printStackTrace();
  178.         } catch (IOException e) {
  179.             log.info("writeIntoFileByAppend {} dts data......IOException!", dwCode);
  180.             e.printStackTrace();
  181.         } finally {
  182.             if (null != os) {
  183.                 try {
  184.                     os.close();
  185.                 } catch (IOException e) {
  186.                     e.printStackTrace();
  187.                 }
  188.             }
  189.         }
  190.     }
  191.  
  192.     /**
  193.      * 写入同一个文件
  194.      *
  195.      * @param dwCode
  196.      * @param rowsContent
  197.      */
  198.     private  void saveToSingleFile(String dwCode, String rowsContent) {
  199.         log.info("save saveToSingleFile {} dts data......", dwCode);
  200.         String filePath = new StringBuffer(localSaveBasePath).append("//").append("list.txt").toString();
  201.         File destFile = new File(filePath);
  202.         this.writeIntoFile(dwCode,rowsContent,destFile);
  203.     }
  204.  
  205.     /**
  206.      * 保存单位附件文件
  207.      *
  208.      * @param type
  209.      * @param dwCode
  210.      * @param rowsContent
  211.      */
  212.     private synchronized void saveToMultiFile(String type, String dwCode, String rowsContent) {
  213.         if (StringUtils.isNotEmpty(rowsContent)) {
  214.             /创建路径//S/
  215.             String dwDir = new StringBuffer(localSaveBasePath).append("//").append(type).toString();
  216.             this.createDir(dwDir);
  217.             String dwDir2 = new StringBuffer(dwDir).append("//").append(dwCode).toString();
  218.             this.createDir(dwDir2);
  219.             /创建路径//E/
  220.             String destFilePath = new StringBuffer(dwDir2).append("//").append("list.txt").toString();
  221.             File destFile = new File(destFilePath);
  222.             this.writeIntoFile(dwCode,rowsContent,destFile);
  223.         }
  224.     }
  225.  
  226.     /**
  227.      * 保存病疫监测事件附件
  228.      *
  229.      */
  230.     private void saveByjcEventFujianAll(boolean writeSingleFile) {
  231.         List<String> list = fujianService.getExportCorpList();
  232.         log.info("ByjcEvent getExportCorpList  size = {}", list.size());
  233.         for (String dwCode : list) {
  234.             List<Map<String, Object>> dataList = fujianService.getEventFujian(FujianService.XH_BYJC_FJ_TB, dwCode);
  235.             // 解析并保存病疫监测附件数据
  236.             String rowsContent = this.parseDataListContent(dwCode, dataList);
  237.             // 保存迁移文件内容
  238.             if(writeSingleFile){
  239.                 this.saveToSingleFile(dwCode, rowsContent);
  240.             }else{
  241.                 this.saveToMultiFile(FujianService.XH_BYJC_FJ_TB, dwCode, rowsContent);
  242.             }
  243.         }
  244.         log.info("ByjcEvent getExportCorpList  size = {}   finished!", list.size());
  245.     }
  246.  
  247.     /**
  248.      * 保存公共事件附件
  249.      *
  250.      */
  251.     private void saveShijianEventFujianAll(boolean writeSingleFile) {
  252.         List<String> list = fujianService.getExportCorpList();
  253.         log.info("ShijianEvent getExportCorpList  size = {}", list.size());
  254.         for (String dwCode : list) {
  255.             List<Map<String, Object>> dataList = fujianService.getEventFujian(FujianService.XH_SHIJIAN_FJ_TB, dwCode);
  256.             // 解析并保存病疫监测附件数据
  257.             String rowsContent = this.parseDataListContent(dwCode, dataList);
  258.             // 保存迁移文件内容
  259.             if(writeSingleFile){
  260.                 this.saveToSingleFile(dwCode, rowsContent);
  261.             }else{
  262.                 this.saveToMultiFile(FujianService.XH_SHIJIAN_FJ_TB, dwCode, rowsContent);
  263.             }
  264.         }
  265.         log.info("ShijianEvent getExportCorpList  size = {}   finished!", list.size());
  266.     }
  267.  
  268.     /**
  269.      * 开启线程处理附件
  270.      *
  271.      */
  272.     public void startTDSStoreThreads(boolean writeSingleFile) {
  273.         log.info("start threads.....");
  274.         new Thread(() -> {
  275.             this.saveByjcEventFujianAll(writeSingleFile);
  276.         }).start();
  277.  
  278.         new Thread(() -> {
  279.             this.saveShijianEventFujianAll(writeSingleFile);
  280.         }).start();
  281.         log.info("start threads.....success!");
  282.     }
  283. }

 

相关技术文章

最新源码

下载排行榜
点击QQ咨询
开通会员
返回顶部
×
微信扫码支付 元
微信扫码支付
确定支付下载
请使用微信描二维码支付
×

提示信息

×

选择支付方式

  • 微信支付
  • 支付宝付款
确定支付下载