FastDfS入门到精通

最新推荐文章于 2024-04-28 03:19:33 发布

fengtangjiang

最新推荐文章于 2024-04-28 03:19:33 发布

阅读量111

点赞数

CC 4.0 BY-SA版权

文章标签：数据库 java fastdfs linux spring

本文链接：https://blog.youkuaiyun.com/fengtangjiang/article/details/120192962

一,FastDFS简介

　　FastDFS是用c语言编写的一款开源的分布式文件系统，它是由淘宝资深架构师余庆编写并开源。FastDFS专为互联网量身定制，充分考虑了冗余备份、负载均衡、线性扩容等机制，并注重高可用、高性能等指标，使用FastDFS很容易搭建一套高性能的文件服务器集群提供文件上传、下载等服务。

优势·：

　　适合通用分部式，fastDFS非常适合存储图片等那些小文件，fastDFS不对文件进行分块，所以它就没有分块合并的开销，fastDFS网络通信采用socket，通信速度很快。

二，服务器分为tracker和storage

三，上传文件原理

storage将文件id返回客户端

id中包含的信息

代码实现：

1.工具类：

  1 import org.csource.common.NameValuePair;
  2 import org.csource.fastdfs.*;
  3 import org.slf4j.LoggerFactory;
  4 import org.springframework.core.io.ClassPathResource;
  5 
  6 import java.io.ByteArrayInputStream;
  7 import java.io.IOException;
  8 import java.io.InputStream;
  9 
 10 public class FastDFSClient {
 11     private static org.slf4j.Logger logger = LoggerFactory.getLogger(FastDFSClient.class);
 12 
 13     /***
 14      * 初始化加载FastDFS的TrackerServer配置
 15      */
 16     static {
 17         try {
 18             String filePath = new ClassPathResource("fdfs_client.conf").getFile().getAbsolutePath();;
 19             ClientGlobal.init(filePath);
 20         } catch (Exception e) {
 21             logger.error("FastDFS Client Init Fail!",e);
 22         }
 23     }
 24 
 25     /***
 26      * 文件上传
 27      * @param file
 28      * @return
 29      */
 30     public static String[] upload(FastDFSFile file) {
 31         //获取文件的作者
 32         NameValuePair[] meta_list = new NameValuePair[1];
 33         meta_list[0] = new NameValuePair("author", file.getAuthor());
 34 
 35         //接收返回数据
 36         String[] uploadResults = null;
 37         StorageClient storageClient=null;
 38         try {
 39             //创建StorageClient客户端对象
 40             storageClient = getTrackerClient();
 41 
 42             /***
 43              * 文件上传
 44              * 1)文件字节数组
 45              * 2)文件扩展名
 46              * 3)文件作者
 47              */
 48             uploadResults = storageClient.upload_file(file.getContent(), file.getExt(), meta_list);
 49         } catch (Exception e) {
 50             logger.error("Exception when uploadind the file:" + file.getName(), e);
 51         }
 52 
 53         if (uploadResults == null && storageClient!=null) {
 54             logger.error("upload file fail, error code:" + storageClient.getErrorCode());
 55         }
 56         //获取组名
 57         String groupName = uploadResults[0];
 58         //获取文件存储路径
 59         String remoteFileName = uploadResults[1];
 60         return uploadResults;
 61     }
 62 
 63     /***
 64      * 获取文件信息
 65      * @param groupName:组名
 66      * @param remoteFileName：文件存储完整名
 67      * @return
 68      */
 69     public static FileInfo getFile(String groupName, String remoteFileName) {
 70         try {
 71             StorageClient storageClient = getTrackerClient();
 72             return storageClient.get_file_info(groupName, remoteFileName);
 73         } catch (Exception e) {
 74             logger.error("Exception: Get File from Fast DFS failed", e);
 75         }
 76         return null;
 77     }
 78 
 79     /***
 80      * 文件下载
 81      * @param groupName
 82      * @param remoteFileName
 83      * @return
 84      */
 85     public static InputStream downFile(String groupName, String remoteFileName) {
 86         try {
 87             //创建StorageClient
 88             StorageClient storageClient = getTrackerClient();
 89 
 90             //下载文件
 91             byte[] fileByte = storageClient.download_file(groupName, remoteFileName);
 92             InputStream ins = new ByteArrayInputStream(fileByte);
 93             return ins;
 94         } catch (Exception e) {
 95             logger.error("Exception: Get File from Fast DFS failed", e);
 96         }
 97         return null;
 98     }
 99 
100 
101     /***
102      * 文件删除
103      * @param groupName
104      * @param remoteFileName
105      * @throws Exception
106      */
107     public static void deleteFile(String groupName, String remoteFileName)
108             throws Exception {
109         //创建StorageClient
110         StorageClient storageClient = getTrackerClient();
111 
112         //删除文件
113         int i = storageClient.delete_file(groupName, remoteFileName);
114     }
115 
116 
117     /***
118      * 获取Storage组
119      * @param groupName
120      * @return
121      * @throws IOException
122      */
123     public static StorageServer[] getStoreStorages(String groupName)
124             throws IOException {
125         //创建TrackerClient
126         TrackerClient trackerClient = new TrackerClient();
127         //获取TrackerServer
128         TrackerServer trackerServer = trackerClient.getConnection();
129         //获取Storage组
130         return trackerClient.getStoreStorages(trackerServer, groupName);
131     }
132 
133     /***
134      * 获取Storage信息,IP和端口
135      * @param groupName
136      * @param remoteFileName
137      * @return
138      * @throws IOException
139      */
140     public static ServerInfo[] getFetchStorages(String groupName,
141                                                 String remoteFileName) throws IOException {
142         TrackerClient trackerClient = new TrackerClient();
143         TrackerServer trackerServer = trackerClient.getConnection();
144         return trackerClient.getFetchStorages(trackerServer, groupName, remoteFileName);
145     }
146 
147     /***
148      * 获取Tracker服务地址
149      * @return
150      * @throws IOException
151      */
152     public static String getTrackerUrl() throws IOException {
153         return "http://"+getTrackerServer().getInetSocketAddress().getHostString()+":"+ ClientGlobal.getG_tracker_http_port()+"/";
154     }
155 
156 
157     /***
158      * 获取Storage客户端
159      * @return
160      * @throws IOException
161      */
162     private static StorageClient getTrackerClient() throws IOException {
163         TrackerServer trackerServer = getTrackerServer();
164         StorageClient storageClient = new StorageClient(trackerServer, null);
165         return  storageClient;
166     }
167 
168 
169     /***
170      * 获取Tracker
171      * @return
172      * @throws IOException
173      */
174     private static TrackerServer getTrackerServer() throws IOException {
175         TrackerClient trackerClient = new TrackerClient();
176         TrackerServer trackerServer = trackerClient.getConnection();
177         return  trackerServer;
178     }
179 }

2.启动类

 1 import org.springframework.boot.SpringApplication;
 2 import org.springframework.boot.autoconfigure.SpringBootApplication;
 3 import org.springframework.cloud.netflix.eureka.EnableEurekaClient;
 4 
 5 @SpringBootApplication
 6 @EnableEurekaClient
 7 public class FileApplication {
 8     public static void main(String[] args) {
 9         SpringApplication.run( FileApplication.class );
10     }
11 }

3.FastDFSFile标准类

 1 public class FastDFSFile {
 2 
 3     //文件名字
 4     private String name;
 5     //文件内容
 6     private byte[] content;
 7     //文件扩展名
 8     private String ext;
 9     //文件MD5摘要值
10     private String md5;
11     //文件创建作者
12     private String author;
13 
14     public FastDFSFile(String name, byte[] content, String ext, String height,
15                        String width, String author) {
16         super();
17         this.name = name;
18         this.content = content;
19         this.ext = ext;
20         this.author = author;
21     }
22 
23     public FastDFSFile(String name, byte[] content, String ext) {
24         super();
25         this.name = name;
26         this.content = content;
27         this.ext = ext;
28     }
29 
30     public String getName() {
31         return name;
32     }
33 
34     public void setName(String name) {
35         this.name = name;
36     }
37 
38     public byte[] getContent() {
39         return content;
40     }
41 
42     public void setContent(byte[] content) {
43         this.content = content;
44     }
45 
46     public String getExt() {
47         return ext;
48     }
49 
50     public void setExt(String ext) {
51         this.ext = ext;
52     }
53 
54     public String getMd5() {
55         return md5;
56     }
57 
58     public void setMd5(String md5) {
59         this.md5 = md5;
60     }
61 
62     public String getAuthor() {
63         return author;
64     }
65 
66     public void setAuthor(String author) {
67         this.author = author;
68     }
69 }

4.上传的类

 1 import org.springframework.web.bind.annotation.CrossOrigin;
 2 import org.springframework.web.bind.annotation.PostMapping;
 3 import org.springframework.web.bind.annotation.RequestParam;
 4 import org.springframework.web.bind.annotation.RestController;
 5 import org.springframework.web.multipart.MultipartFile;
 6 
 7 import java.io.IOException;
 8 
 9 @RestController
10 @CrossOrigin
11 public class FileController {
12 
13 
14     @PostMapping("/upload")
15     public String upload( @RequestParam("file") MultipartFile file){
16         try {
17             //1.获取文件名
18             String fileName = file.getOriginalFilename();
19             //2.获取文件内容
20             byte[] bytes = file.getBytes();
21             //3.获取文件扩展名 .
22             String ext = fileName.substring( fileName.lastIndexOf( "." ) );
23             //4.封装文件实体
24             FastDFSFile fastDFSFile=new FastDFSFile( fileName,bytes,ext );
25             //5.文件上传
26             String[] result = FastDFSClient.upload( fastDFSFile );
27             //6.返回结果
28             String path = FastDFSClient.getTrackerUrl()+result[0]+"/"+result[1];
29             return path;
30 
31         } catch (IOException e) {
32             e.printStackTrace();
33             return "";
34         }
35     }
36 
37 }

yml文件

 1 server:
 2   port: 9008
 3 spring:
 4   application:
 5     name: file
 6   servlet:
 7     multipart:
 8       max-file-size: 10MB
 9       max-request-size: 10MB
10   main:
11     allow-bean-definition-overriding: true #当遇到同样名字的时候，是否允许覆盖注册
12 eureka:
13   client:
14     service-url:
15       defaultZone: http://127.0.0.1:6868/eureka
16   instance:
17     prefer-ip-address: true

配置文件

connect_timeout = 60
network_timeout = 60
charset = UTF-8
http.tracker_http_port = 8080
tracker_server = 192.168.200.128:22122

四，文件下载

原理

有tracker根据id中的信息来决定从哪个storage中查询

五，项目级别运用

(一)图片文件上传(对应1-5)

1.文件校验(2)

//文件校验
if (file == null){
    ExceptionCast.cast(CommonCode.INVALIDATE_PARAMS);
}

2.上传文件(3)

 1     
 2         //上传文件
 3         String fileId = this.uploadFileToFastdfs(file);
 4         if (StringUtils.isEmpty(fileId)){
 5             ExceptionCast.cast(FileSystemCode.FS_UPLOADFILE_FILEISNULL);
 6         }
 7 
 8 private String uploadFileToFastdfs(MultipartFile file) {
 9 
10 
11         try {
12 
13             //初始化fastdfs
14             this.initFastdfs();
15 
16             //编写上传功能
17             TrackerClient trackerClient = new TrackerClient();
18             TrackerServer trackerServer = null;
19             trackerServer = trackerClient.getConnection();
20             StorageServer storageServer = trackerClient.getStoreStorage(trackerServer);
21             StorageClient1 storageClient1 = new StorageClient1(trackerServer,storageServer);
22 
23             String originalFilename = file.getOriginalFilename();
24             String extName = originalFilename.substring(originalFilename.lastIndexOf(".")+1);
25 
26             String fileId = storageClient1.upload_file1(file.getBytes(), extName, null);
27             return fileId;
28 
29         } catch (Exception e) {
30             e.printStackTrace();
31         }
32 
33         return null;
34 
35 
36     }
37 
38 
39     private void initFastdfs() {
40         try {
41             ClientGlobal.initByTrackers(tracker_servers);
42             ClientGlobal.setG_network_timeout(network_timeout_in_seconds);
43             ClientGlobal.setG_connect_timeout(connect_timeout_in_seconds);
44             ClientGlobal.setG_charset(charset);
45         } catch (Exception e) {
46             e.printStackTrace();
47         }
48     }

初始化fastDfs1

    private void initFastdfs() {
        try {
            ClientGlobal.initByTrackers(tracker_servers);
            ClientGlobal.setG_network_timeout(network_timeout_in_seconds);
            ClientGlobal.setG_connect_timeout(connect_timeout_in_seconds);
            ClientGlobal.setG_charset(charset);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

，编写上传功能(4返回文件地址)

String fileId = storageClient1.upload_file1(file.getBytes(), extName, null);
      return fileId;

 1     //编写上传功能
 2             TrackerClient trackerClient = new TrackerClient();
 3             TrackerServer trackerServer = null;
 4             trackerServer = trackerClient.getConnection();
 5             StorageServer storageServer = trackerClient.getStoreStorage(trackerServer);
 6             StorageClient1 storageClient1 = new StorageClient1(trackerServer,storageServer);
 7 
 8             String originalFilename = file.getOriginalFilename();
 9             String extName = originalFilename.substring(originalFilename.lastIndexOf(".")+1);
10 
11             String fileId = storageClient1.upload_file1(file.getBytes(), extName, null);
12             return fileId;
13 
14         } catch (Exception e) {
15             e.printStackTrace();
16         }
17 
18         return null;

3.存到mongodb(5，文件信息存到mongo库)

 1  //将文件的id存入mongodb
 2         FileSystem fileSystem = new FileSystem();
 3 
 4         //设置值
 5         //文件id
 6         fileSystem.setFileId(fileId);
 7         //文件在文件系统中的路径
 8         fileSystem.setFilePath(fileId);
 9         //业务标识
10         fileSystem.setBusinesskey(businesskey);
11         //标签
12         fileSystem.setFiletag(filetag);
13         //元数据
14         if(StringUtils.isNotEmpty(metadata)){
15             try {
16                 Map map = JSON.parseObject(metadata, Map.class);
17                 fileSystem.setMetadata(map);
18             } catch (Exception e) {
19                 e.printStackTrace();
20             }
21         }
22         //名称
23         fileSystem.setFileName(file.getOriginalFilename());
24         //大小
25         fileSystem.setFileSize(file.getSize());
26         //文件类型
27         fileSystem.setFileType(file.getContentType());
28 
29         fileSystemRepository.save(fileSystem);
30 
31         //返回结果
32         return new UploadFileResult(CommonCode.SUCCESS,fileSystem);
33     }

(二)图片信息和课程信息绑定，并存到mysql(6-8)

课程管理服务负责将(一)中上传的文件的图片地址，与对应课程id进行绑定并保存在mysql数据库，方便其他子系统使用。

添加方法

查询方法

删除方法

 1     @Override
 2     @PostMapping("/coursepic/add")
 3     public ResponseResult addCoursePic(String courseId, String pic) {
 4         return courseService.addCoursePic(courseId,pic);
 5     }
 6     
 7    
 8     @Override
 9     @PreAuthorize("hasAuthority('course_pic_list')")
10     @GetMapping("/coursepic/list/{courseId}")
11     public CoursePic findCoursePic(@PathVariable("courseId") String courseId) {
12         return courseService.findCoursePic(courseId);
13     }
14     @Override
15     @DeleteMapping("/coursepic/delete")
16     public ResponseResult delCoursePic(String courseId) {
17         return courseService.delCoursePic(courseId);
18     }

servcie

   /**
     * 保存课程图片信息
     * @param courseId
     * @param pic
     * @return
     */
    public ResponseResult addCoursePic(String courseId, String pic) {

        Optional<CoursePic> coursePicOptional = coursePicRepository.findById(courseId);

        CoursePic coursePic = null;

        if (coursePicOptional.isPresent()){
            coursePic = coursePicOptional.get();
        }

        if (coursePic == null){
            coursePic = new CoursePic();
        }

        coursePic.setPic(pic);
        coursePic.setCourseid(courseId);
        coursePicRepository.save(coursePic);
        return new ResponseResult(CommonCode.SUCCESS);
    }

    /**
     * 查询课程图片
     * @param courseId
     * @return
     */
    public CoursePic findCoursePic(String courseId) {

        Optional<CoursePic> coursePicOptional = coursePicRepository.findById(courseId);
        if (coursePicOptional.isPresent()){
            return coursePicOptional.get();
        }

        return null;
    }

    /**
     * 删除课程图片
     * @param courseId 课程id
     * @return  ResponseResult(规范返回类型)
     */
    public ResponseResult delCoursePic(String courseId) {

        coursePicRepository.deleteById(courseId);
        return new ResponseResult(CommonCode.SUCCESS);
    }

dao

import com.xuecheng.filesystem.framework.domain.course.response.CoursePic;
import org.springframework.data.jpa.repository.JpaRepository;

public interface CoursePicRepository extends JpaRepository<CoursePic,String> {
}