package com.lc.ibps.cloud.file.util;
import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.io.FileUtil;
import com.lc.ibps.api.base.file.FileInfo;
import com.lc.ibps.base.core.exception.BaseException;
import com.lc.ibps.base.core.util.AppUtil;
import com.lc.ibps.base.core.util.EnvUtil;
import com.lc.ibps.base.core.util.string.StringUtil;
import com.lc.ibps.base.core.util.time.DateFormatUtil;
import com.lc.ibps.components.upload.constants.SaveType;
import com.aventrix.jnanoid.jnanoid.NanoIdUtils;
import io.minio.errors.*;
import org.apache.commons.net.ftp.FTP;
import org.apache.commons.net.ftp.FTPClient;
import org.apache.commons.net.ftp.FTPFile;
import org.apache.commons.net.ftp.FTPReply;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
import java.net.SocketException;
import java.nio.charset.StandardCharsets;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.util.*;
public class FtpUtil {
private static final Logger logger = LoggerFactory.getLogger(FtpUtil.class);
protected static final String CHUNK_TMP_PATH = "/chunk/";
/**
* ftp连接
* @return
*/
public static FTPClient connectFtp() {
FTPClient ftpClient = null;
try {
String saveType = EnvUtil.getProperty("file.saveType", "");
if (!SaveType.ftp.name().equals(saveType)) {
return null;
}
String host = EnvUtil.getProperty("ftp.host", "172.31.60.205");
int port = EnvUtil.getProperty("ftp.port", Integer.class, 33);
String userName = EnvUtil.getProperty("ftp.userName", "");
String pwd = EnvUtil.getProperty("ftp.pwd", "");
if(ftpClient == null || !ftpClient.isAvailable()){
try {
ftpClient = new FTPClient();
/**
*
* pcz 20240911
* 下载 大文时，会有文件不完整的情况
* 设置配置应该是在ftp连接前操作
1.设置缓冲区大小为1MB 1024*1024
2.设置超时时间
//被动模式 文件类型应该在connect 后操作
3.设置被动模式
4.文件类型
**/
//字节
ftpClient.setBufferSize(1024*1024);
ftpClient.setReceiveBufferSize(1024 * 1024);
ftpClient.setDataTimeout(60 * 1000); // 设置数据传输超时时间为 60 秒
ftpClient.setConnectTimeout(65 * 1000); // 设置连接超时时间为 65 秒
//连接FTP 服务器
ftpClient.connect(host, port);
if(userName!=null && pwd!=null) {
ftpClient.login(userName, pwd);
ftpClient.enterLocalPassiveMode();
ftpClient.setFileType(FTP.BINARY_FILE_TYPE);
//
//ftpClient.setBufferSize(1024*1024);
//ftpClient.setReceiveBufferSize(1024 * 1024);
}else {
ftpClient.login("", "");
}
} catch (SocketException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
if (!FTPReply.isPositiveCompletion(ftpClient.getReplyCode())) {
logger.info("连接FTP失败，用户名或密码错误。");
ftpClient.disconnect();
}
//ftpClient.setRemoteVerificationEnabled(false);
//ftpClient.setFileType(FTPClient.BINARY_FILE_TYPE); // 二进制
} catch (Exception e) {
logger.info("登陆FTP失败，请检查FTP相关配置信息是否正确！" + e);
return null;
}
return ftpClient;
}
public static String upload(FTPClient ftpClient, String bucket, String fileId, String filename,InputStream inputStream)
throws InvalidKeyException, ErrorResponseException, IllegalArgumentException,
InsufficientDataException, InternalException, InvalidBucketNameException, InvalidResponseException,
NoSuchAlgorithmException, ServerException, XmlParserException, RegionConflictException, IOException {
String path = bucket(bucket) + getPath() ;
try {
/* if (BeanUtil.isEmpty(ftpClient) || !ftpClient.isConnected()){
}*/
ftpClient = FtpUtil.connectFtp();
// 中文目录处理存在问题， 转化为ftp能够识别中文的字符集
if (FTPReply.isPositiveCompletion(ftpClient.sendCommand("OPTS UTF8", "ON"))) {
ftpClient.setControlEncoding("UTF-8");
} else {
//FTP协议里面，规定文件名编码为iso-8859-1
ftpClient.setControlEncoding("ISO-8859-1");
}
//createDirecroty(ftpClient, path);
boolean dir = mkDir(ftpClient, path);
//ftpClient.setBufferSize(1024);
//// 设置文件类型（二进制）
//ftpClient.setFileType(ftpClient.BINARY_FILE_TYPE);
if (!dir){
logger.info("创建目录失败："+ path);
return null;
}
/* boolean directory = ftpClient.changeWorkingDirectory(path);
if (!directory){
logger.info("切换目录失败："+directory);
return null;
}*/
boolean result = ftpClient.storeFile(new String(filename.getBytes("GBK"), "iso-8859-1"), inputStream);
logger.info("图片上传结果：{},文件名称：{}",result,filename);
if (!result){
logger.info("图片上传失败!");
return null;
}
} catch (FileNotFoundException e) {
logger.error("文件上传失败" + e);
return null;
} catch (IOException e) {
logger.error("文件上传失败" + e);
return null;
}finally {
try {
ftpClient.logout();
if (ftpClient.isConnected()){
ftpClient.disconnect();
}
if (null != inputStream) {
inputStream.close();
}
} catch (IOException e) {
logger.error("上传文件失败" + e);
}
}
return path + "/" + filename;
}
public static String uploadChunk(FTPClient ftpClient, String bucket, String chunkPath, String filename,InputStream inputStream)
throws InvalidKeyException, ErrorResponseException, IllegalArgumentException,
InsufficientDataException, InternalException, InvalidBucketNameException, InvalidResponseException,
NoSuchAlgorithmException, ServerException, XmlParserException, RegionConflictException, IOException {
String path = bucket(bucket) + getPath() + chunkPath;
try {
/* if (BeanUtil.isEmpty(ftpClient) || !ftpClient.isConnected()){
}*/
ftpClient = FtpUtil.connectFtp();
// 中文目录处理存在问题， 转化为ftp能够识别中文的字符集
if (FTPReply.isPositiveCompletion(ftpClient.sendCommand("OPTS UTF8", "ON"))) {
ftpClient.setControlEncoding("UTF-8");
} else {
//FTP协议里面，规定文件名编码为iso-8859-1
ftpClient.setControlEncoding("ISO-8859-1");
}
//createDirecroty(ftpClient, path);
boolean dir = mkDir(ftpClient, path);
//ftpClient.setBufferSize(1024);
//// 设置文件类型（二进制）
//ftpClient.setFileType(ftpClient.BINARY_FILE_TYPE);
if (!dir){
logger.info("创建目录失败："+ path);
return null;
}
/* boolean directory = ftpClient.changeWorkingDirectory(path);
if (!directory){
logger.info("切换目录失败："+directory);
return null;
}*/
boolean result = ftpClient.storeFile(new String(filename.getBytes("GBK"), "iso-8859-1"), inputStream);
logger.info("图片上传结果：{},文件名称：{}",result,filename);
if (!result){
logger.info("图片上传失败!");
return null;
}
} catch (FileNotFoundException e) {
logger.error("文件上传失败" + e);
return null;
} catch (IOException e) {
logger.error("文件上传失败" + e);
return null;
}finally {
try {
ftpClient.logout();
if (ftpClient.isConnected()){
ftpClient.disconnect();
}
if (null != inputStream) {
inputStream.close();
}
} catch (IOException e) {
logger.error("上传文件失败" + e);
}
}
return path + "/" + filename;
}
public static String uploadByBytes(FTPClient ftpClient, String bucket, String fileId, String filename,
byte[] data) throws InvalidKeyException, ErrorResponseException, IllegalArgumentException,
InsufficientDataException, InternalException, InvalidBucketNameException, InvalidResponseException,
NoSuchAlgorithmException, ServerException, XmlParserException, RegionConflictException, IOException {
String result = null;
ByteArrayInputStream stream = null;
try {
/*if (BeanUtil.isEmpty(ftpClient) || !ftpClient.isConnected()){
ftpClient = FtpUtil.connectFtp();
}*/
stream = new ByteArrayInputStream(data);
result = upload(ftpClient, bucket, fileId, filename, stream);
} catch (Exception e) {
throw new BaseException(e);
} finally {
if (Objects.nonNull(stream)) {
try {
stream.close();
} catch (IOException e) {
}
}
}
return result;
}
/**
* 从FTP下载文件到本地
*
*/
public static InputStream download(FTPClient ftpClient, String etag) throws InvalidKeyException, ErrorResponseException,
IllegalArgumentException, InsufficientDataException, InternalException, InvalidBucketNameException, InvalidResponseException,
NoSuchAlgorithmException, ServerException, XmlParserException, RegionConflictException, IOException {
InputStream is = null;
try {
is = downloadFile(ftpClient, etag);
logger.info(etag+" :FTP文件下载成功！");
} catch (Exception e) {
logger.error(etag+" :FTP文件下载失败！",e);
} finally {
try {
if (is != null) is.close();
ftpClient.logout();
if (ftpClient.isConnected()){
ftpClient.disconnect();
}
} catch (IOException e) {
logger.error("下载流关闭失败" , e);
}
}
return is;
}
public static byte[] downloadToBytes(FTPClient ftpClient, String etag) throws InvalidKeyException, ErrorResponseException, IllegalArgumentException,
InsufficientDataException, InternalException, InvalidBucketNameException, InvalidResponseException,
NoSuchAlgorithmException, ServerException, XmlParserException, RegionConflictException, IOException {
byte[] datas = null;
InputStream stream = null;
BufferedInputStream bufferedInputStream = null;
ByteArrayOutputStream baos = null;
try {
//ftpClient = FtpUtil.connectFtp();
//linux系统要使用/分割，window系统使用\分割
etag = etag.replace("\\", File.separator);
stream = downloadFile(ftpClient, etag);
bufferedInputStream = new BufferedInputStream(stream);
baos = new ByteArrayOutputStream();
byte[] buffer = new byte[1024*1024];
int i;
int totalsize = 0;
while ((i = bufferedInputStream.read(buffer)) != -1) {
baos.write(buffer, 0, i);
totalsize = totalsize + i;
baos.flush();
}
datas = baos.toByteArray();
logger.info("下载文件成功大小: "+ totalsize);
logger.info("下载文件成功大小: "+ datas.length);
logger.info("下载文件成功: "+ etag);
} catch (Exception e) {
logger.error("下载文件失败: "+etag,e);
throw new BaseException(e);
} finally {
//if (Objects.nonNull(stream)) {
try {
if (null != stream)stream.close();
if (null != bufferedInputStream)bufferedInputStream.close();
if (null != baos)baos.close();
ftpClient.logout();
if (ftpClient.isConnected()){
ftpClient.disconnect();
}
} catch (IOException e) { }
//}
}
return datas;
}
private static InputStream downloadFile(FTPClient ftpClient, String etag) throws IOException {
/*if (BeanUtil.isEmpty(ftpClient) || !ftpClient.isConnected()){
}*/
ftpClient = FtpUtil.connectFtp();
// 中文目录处理存在问题， 转化为ftp能够识别中文的字符集
if (FTPReply.isPositiveCompletion(ftpClient.sendCommand("OPTS UTF8", "ON"))) {
ftpClient.setControlEncoding("UTF-8");
} else {
//FTP协议里面，规定文件名编码为iso-8859-1
ftpClient.setControlEncoding("ISO-8859-1");
}
logger.info("ftpcommand RETR " + etag);
return ftpClient.retrieveFileStream(new String(etag.getBytes(), "iso-8859-1"));// 获取ftp上的文件
}
private static InputStream downloadFileWithFtpClient(FTPClient ftpClient, String etag) throws IOException {
//分块文件数量多，不能每次都去开关FTPClient
if (BeanUtil.isEmpty(ftpClient) || !ftpClient.isAvailable() || !ftpClient.isConnected()){
ftpClient = FtpUtil.connectFtp();
}
// 中文目录处理存在问题， 转化为ftp能够识别中文的字符集
if (FTPReply.isPositiveCompletion(ftpClient.sendCommand("OPTS UTF8", "ON"))) {
ftpClient.setControlEncoding("UTF-8");
} else {
//FTP协议里面，规定文件名编码为iso-8859-1
ftpClient.setControlEncoding("ISO-8859-1");
}
InputStream result = ftpClient.retrieveFileStream(new String(etag.getBytes(), "iso-8859-1"));
return result;
}
public static String stat(FTPClient ftpClient, String bucket, String filename, String etag)
throws InvalidKeyException, ErrorResponseException, IllegalArgumentException, InsufficientDataException,
InternalException, InvalidBucketNameException, InvalidResponseException, NoSuchAlgorithmException,
ServerException, XmlParserException, RegionConflictException, IOException {
String path = bucket(bucket) + getPath();
createDirecroty(ftpClient, path);
ftpClient.stat(etag);
return etag;
}
public static void remove(FTPClient ftpClient, String bucket, String fileId, String etag)
throws InvalidKeyException, ErrorResponseException, IllegalArgumentException, InsufficientDataException,
InternalException, InvalidBucketNameException, InvalidResponseException, NoSuchAlgorithmException,
ServerException, XmlParserException, RegionConflictException, IOException {
if (StringUtil.isBlank(fileId)) {
return;
}
bucket = stat(ftpClient, bucket, fileId, etag);
ftpClient.deleteFile(bucket);
}
/**
* 测试文件上传、下载
*
* @param ftpClient
* @throws InvalidBucketNameException
*/
private static void test(FTPClient ftpClient) {
try {
String bucket = "iform";
String filename = "log.txt";
String file = "d:\\log.txt";
String downloadFile = "d:\\log." + NanoIdUtils.randomNanoId() + ".txt";
String fileId = "1";
// 上传
BufferedInputStream stream = FileUtil.getInputStream(file);
System.out.println("开始上传文件");
String etag = upload(ftpClient, bucket, fileId, filename, stream);
System.out.println("完成上传文件");
// 下载
System.out.println("开始下载文件");
InputStream dataStream = download(ftpClient, etag);
FileUtil.writeFromStream(dataStream, downloadFile);
System.out.println("完成下载文件");
// 删除
System.out.println("开始删除文件");
remove(ftpClient, bucket, fileId, etag);
System.out.println("完成删除文件");
} catch (InvalidKeyException | ErrorResponseException | InsufficientDataException | InternalException
| InvalidResponseException | NoSuchAlgorithmException | ServerException | XmlParserException
| IllegalArgumentException | IOException | RegionConflictException | InvalidBucketNameException e) {
e.printStackTrace();
}
}
public static void main(String[] args) {
FTPClient ftpClient = connectFtp();
test(ftpClient);
}
//改变目录路径
private static boolean changeWorkingDirectory(FTPClient ftpClient, String directory) {
boolean flag = true;
try {
flag = ftpClient.changeWorkingDirectory(directory);
if (flag) {
logger.info("进入文件夹" + directory + " 成功！");
} else {
logger.info("进入文件夹" + directory + " 失败！开始创建文件夹");
}
} catch (IOException ioe) {
ioe.printStackTrace();
}
return flag;
}
//创建多层目录文件，如果有ftp服务器已存在该文件，则不创建，如果无，则创建
private static boolean createDirecroty(FTPClient ftpClient, String remote) throws IOException {
boolean success = true;
String directory = remote + "/";
// 如果远程目录不存在，则递归创建远程服务器目录
if (!directory.equalsIgnoreCase("/") && !changeWorkingDirectory(ftpClient, new String(directory))) {
int start = 0;
int end = 0;
if (directory.startsWith("/")) {
start = 1;
} else {
start = 0;
}
end = directory.indexOf("/", start);
String path = "";
String paths = "";
while (true) {
String subDirectory = new String(remote.substring(start, end).getBytes("GBK"), "iso-8859-1");
path = path + "/" + subDirectory;
if (!existFile(ftpClient, path)) {
if (makeDirectory(ftpClient, subDirectory)) {
changeWorkingDirectory(ftpClient, subDirectory);
} else {
logger.info("创建目录[" + subDirectory + "]失败");
changeWorkingDirectory(ftpClient, subDirectory);
}
} else {
changeWorkingDirectory(ftpClient, subDirectory);
}
paths = paths + "/" + subDirectory;
start = end + 1;
end = directory.indexOf("/", start);
// 检查所有目录是否创建完毕
if (end <= start) {
break;
}
}
}
return success;
}
/**
* 循环创建目录，并且创建完目录后，设置工作目录为当前创建的目录下
*/
private static boolean mkDir(FTPClient ftpClient,String ftpPath) {
if (!ftpClient.isConnected()) {
return false;
}
try {
ftpClient.changeWorkingDirectory("/");
// 将路径中的斜杠统一
char[] chars = ftpPath.toCharArray();
StringBuffer sbStr = new StringBuffer(256);
for (int i = 0; i < chars.length; i++) {
if ('\\' == chars[i]) {
sbStr.append('/');
} else {
sbStr.append(chars[i]);
}
}
ftpPath = sbStr.toString();
if (ftpPath.indexOf('/') == -1) {
// 只有一层目录
ftpClient.makeDirectory(new String(ftpPath.getBytes("GBK"), "iso-8859-1"));
ftpClient.changeWorkingDirectory(new String(ftpPath.getBytes("GBK"), "iso-8859-1"));
} else {
// 多层目录循环创建
String[] paths = ftpPath.split("/");
// String pathTemp = "";
for (int i = 0; i < paths.length; i++) {
ftpClient.makeDirectory(new String(paths[i].getBytes("GBK"), "iso-8859-1"));
ftpClient.changeWorkingDirectory(new String(paths[i].getBytes("GBK"), "iso-8859-1"));
}
}
logger.info(ftpPath+ " 目录创建成功！");
return true;
} catch (Exception e) {
e.printStackTrace();
return false;
}
}
//判断ftp服务器文件是否存在
private static boolean existFile(FTPClient ftpClient, String path) throws IOException {
boolean flag = false;
FTPFile[] ftpFileArr = ftpClient.listFiles(path);
if (ftpFileArr.length > 0) {
flag = true;
}
return flag;
}
//创建目录
private static boolean makeDirectory(FTPClient ftpClient, String dir) {
boolean flag = true;
try {
flag = ftpClient.makeDirectory(dir);
if (flag) {
logger.info("创建文件夹" + dir + " 成功！");
} else {
logger.info("创建文件夹" + dir + " 失败！");
}
} catch (Exception e) {
e.printStackTrace();
}
return flag;
}
/**
* 获取FTP某一特定目录下的所有文件名称
*
* @param ftpClient 已经登陆成功的FTPClient
* @param ftpDirPath FTP上的目标文件路径
*/
public List<String> getFileNameList(FTPClient ftpClient, String ftpDirPath) {
List<String> list = new ArrayList();
try {
// 通过提供的文件路径获取FTPFile对象列表
FTPFile[] files = ftpClient.listFiles(ftpDirPath);
// 遍历文件列表，打印出文件名称
for (int i = 0; i < files.length; i++) {
FTPFile ftpFile = files[i];
// 此处只打印文件，未遍历子目录（如果需要遍历，加上递归逻辑即可）
if (ftpFile.isFile()) {
// log.info(ftpDirPath + ftpFile.getName());
list.add(ftpFile.getName());
}
}
} catch (IOException e) {
logger.error("错误" + e);
}
return list;
}
/**
* 获取到服务器文件夹里面最新创建的文件名称
* @param ftpDirPath 文件路径
* @param ftpClient ftp的连接
* @return fileName
*/
public String getNewFile(FTPClient ftpClient, String ftpDirPath) throws Exception {
// 通过提供的文件路径获取FTPFile对象列表
FTPFile[] files = ftpClient.listFiles(ftpDirPath);
if (files == null) {
return null;
}
Arrays.sort(files, new Comparator<FTPFile>() {
@Override
public int compare(FTPFile f1, FTPFile f2) {
return f1.getTimestamp().compareTo(f2.getTimestamp());
}
public boolean equals(Object obj) {
return true;
}
});
return ftpDirPath + "/" + files[files.length - 1].getName();
}
/**
* 获取时效时间(毫秒)
* @return
*/
public static int getExpireTimeSeconds() {
return AppUtil.getProperty("ftp.expire-time-seconds", Integer.class, 60000);
}
/**
* 获取存储桶
* @param bucket
* @return
*/
public static String bucket(String bucket) {
return StringUtil.isBlank(bucket) ? EnvUtil.getProperty("ftp.bucket", "iform") : bucket;
}
/**
* 获取文件存储路径
* @return
*/
public static String getPath() {
String path = EnvUtil.getProperty("ftp.path", "");
String ym = DateFormatUtil.getNowPart("yyyy/MM");
return StringUtil.isBlank(path) ? "/" + ym : "/" + path + "/" + ym ;
}
public static InputStream mergeChunkFilesAndUploadToFtp(FTPClient ftpClient, FileInfo fileInfo, String[] chunkFiles, String path, String originFileName) throws IOException, RegionConflictException,
ServerException, InvalidBucketNameException, InsufficientDataException, ErrorResponseException,
NoSuchAlgorithmException, InvalidKeyException, InvalidResponseException, XmlParserException, InternalException {
ByteArrayOutputStream outPutStream = new ByteArrayOutputStream();
InputStream fis = null;
logger.info("merge--开始读取分片文件数量:" + chunkFiles.length);
//从FTP 读取所有分片到内存， 并进行合并
for (String fileName : chunkFiles) {
try {
logger.info("merge--开始读分片" + fileName);
fis = downloadFileWithFtpClient(ftpClient, path + File.separator + fileName);
byte[] b = new byte[1024*1024];
int n;
while ((n = fis.read(b)) != -1) {
outPutStream.write(b, 0, n);
}
logger.info("merge--读取完分片" + fileName);
} finally {
if (fis != null) {
fis.close();
//复用FtpClient 需要在读取流的命令里, 执行这个命令，不然无法复用， 需要放在流关闭后面，否则堵塞线程
ftpClient.completePendingCommand();
}
}
}
logger.info("merge--开始上传文件到ftp");
//上传文件到FTP
InputStream byteArrayInputStream = new ByteArrayInputStream(outPutStream.toByteArray());
//在inputStream 关闭前复制数据
byte[] copyByte = copyInputStream(byteArrayInputStream);
InputStream copyInputStream = new ByteArrayInputStream(copyByte);
//这里面会自动关闭 inputstream ftpclient
String filePath = upload(ftpClient, "", "", originFileName, new ByteArrayInputStream(copyByte));
fileInfo.setFilePath(filePath);
logger.info("merge--完成上传文件到ftp");
return copyInputStream;
//上传完成后，异步删除分片文件
//TODO
}
public static byte[] copyInputStream(InputStream input) throws IOException {
ByteArrayOutputStream output = new ByteArrayOutputStream();
byte[] buffer = new byte[8192]; // 选择一个合适的缓冲区大小
int n = 0;
while (-1 != (n = input.read(buffer))) {
output.write(buffer, 0, n);
}
return output.toByteArray();
}
public static String uploadChunkByBytes(FTPClient ftpClient, String bucket, String chunkPath, String filename,
byte[] data) throws InvalidKeyException, ErrorResponseException, IllegalArgumentException,
InsufficientDataException, InternalException, InvalidBucketNameException, InvalidResponseException,
NoSuchAlgorithmException, ServerException, XmlParserException, RegionConflictException, IOException {
String result = null;
ByteArrayInputStream stream = null;
try {
stream = new ByteArrayInputStream(data);
result = uploadChunk(ftpClient, bucket, chunkPath, filename, stream);
} catch (Exception e) {
throw new BaseException(e);
} finally {
if (Objects.nonNull(stream)) {
try {
stream.close();
} catch (IOException e) {
}
}
}
return result;
}
public static String getChunkPath(String bucket, Boolean isChunkTmpPath, String chunkPath) {
String path = bucket(bucket) + getPath();
return path + (isChunkTmpPath?CHUNK_TMP_PATH:"") + chunkPath;
}
}
结合上面提供的上传、下载类，能统一给出优化代码吗 ，这三个类有存在内存泄漏的问题吗
最新发布
本文介绍使用FTPClient进行文件上传时遇到无响应的问题及其解决方案。通过对比主动模式和被动模式的工作原理，阐述如何通过设置被动模式解决该问题。
扫一扫
5062
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
