基于AmazonS3协议的OSS通用组件,minio8集成(附仓库)

Grandpa_Rick

已于 2023-07-18 15:52:21 修改

阅读量4.4k

点赞数 3

分类专栏： java 文章标签： docker 容器服务器 java linux

于 2022-05-23 16:16:44 首次发布

本文链接：https://blog.youkuaiyun.com/weixin_43558927/article/details/124736243

版权

java 专栏收录该内容

20 篇文章

订阅专栏

基于AmazonS3协议的OSS存储通用组件客户端 + Docker + MinIO8

使用docker + minio8完成业务集成

前言

公司内部新系统开发中, 对于文件的设计考虑使用minio完成文件的上传下载,公司内部用性能啥的在docker里起个单机的minio得了, 因为minio8 与 8 之前API及docker 的某些命令不同, 使用时遇到一些坑

优化

最近看了有个“真”大佬的分享, 对于一个公司来说OSS服务可能使用云或者自建, 对于项目来说不断的更换对应服务商或者minio的sdk来说都是十分不合理的, 由于现在大部分服务商以及minio都是基于Amazon的S3协议, 那么可以通过直接使用Amazon的客户端一套sdk操作所有基于该协议的OSS服务, 考虑到公司所有项目完全基于springboot, 则进一步封装优化该OSS需求, 使用starter方式封装, 该文档此前处理使用的minio sdk说明保留

仓库

oss-common-spring-boot-starter

common-oss-spring-boot-starter

主要依赖, 其他使用到springboot基础依赖, lombok等

		<parent>
	        <groupId>org.springframework.boot</groupId>
	        <artifactId>spring-boot-dependencies</artifactId>
	        <version>2.3.7.RELEASE</version>
	        <relativePath/>
	    </parent>

 		<dependency>
            <groupId>com.amazonaws</groupId>
            <artifactId>aws-java-sdk-s3</artifactId>
            <version>1.12.267</version>
        </dependency>
 		<dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-autoconfigure</artifactId>
            <optional>true</optional>
        </dependency>

		<!--以下非必需-->
        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
        </dependency>
        <dependency>
            <groupId>com.google.guava</groupId>
            <artifactId>guava</artifactId>
            <version>31.1-jre</version>
        </dependency>

META-INF/spring.factories

org.springframework.boot.autoconfigure.EnableAutoConfiguration=\
com.hp.oss.OssAutoConfiguration

OssAutoConfiguration

@Configuration
@EnableConfigurationProperties(OssProperties.class)
public class OssAutoConfiguration {

    @Bean
    @ConditionalOnMissingBean(S3OssClient.class)
    public OssClient ossClient(AmazonS3 amazonS3){
        return new S3OssClient(amazonS3);
    }

    @Bean
    @ConditionalOnMissingBean(AmazonS3.class)
    @ConditionalOnProperty(prefix = "oss",name = "enable",havingValue = "true")
    public AmazonS3 amazonS3(OssProperties ossProperties){
        final long count = Stream.builder()
                .add(ossProperties.getEndpoint())
                .add(ossProperties.getAccessSecret())
                .add(ossProperties.getAccessKey())
                .build()
                .filter(Objects::isNull)
                .count();
        if (count > 0) {
            throw new RuntimeException("OSS配置错误，Endpoint,secret,key配置不能为空，请检查");
        }
        final BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials(ossProperties.getAccessKey(), ossProperties.getAccessSecret());
        final AWSStaticCredentialsProvider awsStaticCredentialsProvider = new AWSStaticCredentialsProvider(basicAWSCredentials);
        return AmazonS3Client.builder()
                .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(ossProperties.getEndpoint(),ossProperties.getRegion()))
                .withCredentials(awsStaticCredentialsProvider)
                .disableChunkedEncoding()
                .withPathStyleAccessEnabled(ossProperties.isPathStyleAccess())
                .build();
    }
}

OssProperties

@ConfigurationProperties(prefix = "oss")
@Setter
@Getter
public class OssProperties {

    private boolean enable = true;
    private String accessKey;
    private String accessSecret;
    /**
     * 如果是服务器MinIO等直接使用 [$schema]://[$ip]:[$port]
     * 外网[$Schema]://[$Bucket].[$Endpoint]/[$Object]*
     * https://help.aliyun.com/document_detail/375241.html*
     */
    private String endpoint;
    /**
     * refres to com.amazonaws.regions.Regions*
     * https://help.aliyun.com/document_detail/31837.htm?spm=a2c4g.11186623.0.0.695178eb0nD6jp*
     */
    private String region;
    private boolean pathStyleAccess = true;
}

定义提供使用的客户端
OssClient

/**
 * OSS客户端
 *
 * @author HP
 * @date 2022/8/25
 */
public interface OssClient {

    void createBucket(String bucketName);
    void bucketPolicy(String bucketName,String policy);
    String getObjectURL(String bucketName,String objectName);
    S3Object getObject(String bucketName,String objectName);
    PutObjectResult putObject(String bucketName, String objectName, InputStream inputStream, long size, String contentType) throws IOException;
    AmazonS3 getS3Client();
    default PutObjectResult putObject(String bucketName,String objectName,InputStream inputStream) throws IOException{
        return putObject(bucketName,objectName,inputStream,inputStream.available(),"application/octet-stream");
    }
}

S3客户端的实现
S3OssClient

/**
 * 亚马逊s3协议客户端
 *
 * @author HP
 * @date 2022/8/25
 */
@RequiredArgsConstructor
public class S3OssClient implements OssClient {

    private final AmazonS3 amazonS3;

    @Override
    public void createBucket(String bucketName) {
        if (amazonS3.doesBucketExistV2(bucketName)) {
            return;
        }
        amazonS3.createBucket(bucketName);
    }

    @Override
    public void bucketPolicy(String bucketName,String policy){
        amazonS3.setBucketPolicy(bucketName, policy);
    }

    @Override
    public String getObjectURL(String bucketName, String objectName) {
        final URL url = amazonS3.getUrl(bucketName, objectName);
        return url.toString();
    }

    @Override
    public S3Object getObject(String bucketName, String objectName) {
        return amazonS3.getObject(bucketName, objectName);
    }

    @Override
    public PutObjectResult putObject(String bucketName, String objectName, InputStream inputStream, long size, String contentType) throws IOException {
        ObjectMetadata metaData = new ObjectMetadata();
        metaData.setContentLength(size);
        metaData.setContentType(contentType);
        final PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, objectName, inputStream, metaData);
        putObjectRequest.getRequestClientOptions().setReadLimit((int) (size + 1));
        return amazonS3.putObject(putObjectRequest);
    }

    @Override
    public AmazonS3 getS3Client() {
        return amazonS3;
    }
}

上述接口中提供了原生的AmazonS3客户端, 可以使用这个客户端做接口未定义的方法, 主要是封装了最常用的方法, 测试也可以通过bucketPolicy接口对minio8的bucketpolicy做自定义修改;

配置文件 yml方式
由于示例代码(仓库)中配置类默认是 enable=false 所以直接引入模块在spring管理的bean中直接注入会报错,找不到bean的问题, 可以自行将配置文件中的enable默认改为true或者记得在配置文件中配置为true

# Minio配置
oss:
  enable: true
  endpoint: http://192.168.0.192:9900
  region:
  accessKey: 1H7LcuGNS2jIkIem
  accessSecret: mLgBgaxuJNbCnYVYdapsOeAZ1g6RhY8K
  bucketName: jsoup-bucket
  pathStyleAccess: true  # 改字段未具体实现功能

`对于accessKey, accessSecret 在 MiniIO中如下配置, 并非使用后台登录/容器启动时配置的账号密码, 其他云平台基本相同(略)
在这里插入图片描述

镜像

docker search minio
#结果 minio/minio

#拉取新版镜像 目前最新的是8版本的 ui啥的比较新
docker pull minio/minio
#拉取旧版镜像 很单一的一个ui控制台的那版
docker pull minio/minio:RELEASE.2021-06-17T00-10-46Z

容器

-d --restart=always --privileged=true看情况加

新版(8x版本)

docker run -p 9900:9000 -p 9990:9090 --name myminio \
  -e "MINIO_ACCESS_KEY=root" \  #至少3位 等于账号
  -e "MINIO_SECRET_KEY=12345678" \  #至少8位 等于密码
  -v /Users/programmer/docker/minIO/data:/data \ #挂载数据目录到宿主机
  -v /Users/programmer/docker/minIO/config:/etc/minio \ #挂载配置目录到宿主机
  minio/minio server --console-address ":9990" --config-dir /etc/minio /data

--console-address ":9090", 新版控制台需要指定具体端口,并且要将端口映射出来,否则是随机分配

#执行结果
WARNING: MINIO_ACCESS_KEY and MINIO_SECRET_KEY are deprecated.
         Please use MINIO_ROOT_USER and MINIO_ROOT_PASSWORD
API: http://172.17.0.2:9000  http://127.0.0.1:9000

Console: http://172.17.0.2:9090 http://127.0.0.1:9090

Documentation: https://docs.min.io
Finished loading IAM sub-system (took 0.0s of 0.0s to load data).

旧版(7x版本)

docker run -p 9000:9000 --name my-minio \
  -e "MINIO_ACCESS_KEY=root" \ #至少3位 等于账号
  -e "MINIO_SECRET_KEY=12345678" \ #至少8位 等于密码
  -v /Users/programmer/docker/minIO/data:/data \ #挂载数据目录到宿主机
  -v /Users/programmer/docker/minIO/config:/etc/minio \ #挂载配置目录到宿主机
  minio/minio:RELEASE.2021-06-17T00-10-46Z server  --config-dir /etc/minio /data

--config-dir /etc/minio 指定了新的配置文件目录, 原本是 ~/.monio/config, 如果按本例类似指定了新的配置文件目录, 挂载时请对应新目录

#执行结果
Endpoint: http://172.17.0.2:9000  http://127.0.0.1:9000

Browser Access:
   http://172.17.0.2:9000  http://127.0.0.1:9000

旧版的控制台和api接口使用同一个端口

java

这里如果使用通用包

@RequiredArgsConstructor
@RestController
public class TestOSS {

    private final OssClient ossClient;

    public static final String POLICY = "{\n" +
            //version这里没去管,直接不改
            "    \"Version\": \"2012-10-17\",\n" +
            "    \"Statement\": [\n" +
            "        {\n" +
            "            \"Effect\": \"Allow\",\n" +
            "            \"Principal\": {\n" +
            "                \"AWS\": [\n" +
            "                    \"*\"\n" +
            "                ]\n" +
            "            },\n" +
            //根据具体action类型增加
            "            \"Action\": [\n" +
            "                \"s3:GetBucketLocation\"\n" +
            "            ],\n" +
            "            \"Resource\": [\n" +
            "                \"arn:aws:s3:::test-bucket\"\n" +
            "            ]\n" +
            "        },\n" +
            "        {\n" +
            "            \"Effect\": \"Allow\",\n" +
            "            \"Principal\": {\n" +
            "                \"AWS\": [\n" +
            "                    \"*\"\n" +
            "                ]\n" +
            "            },\n" +
            //根据具体action类型增加
            "            \"Action\": [\n" +
            //可见action只提供了getObject的权限
            "                \"s3:GetObject\"\n" +
            "            ],\n" +
            "            \"Resource\": [\n" +
            "                \"arn:aws:s3:::test-bucket/*\"\n" +
            "            ]\n" +
            "        }\n" +
            "    ]\n" +
            "}";

    @SneakyThrows
    @PostMapping("/testoss")
    public R testOss(@RequestParam(value = "file") MultipartFile file) {
        ossClient.bucketPolicy("test-bucket", POLICY);
        ossClient.putObject("test-bucket", file.getOriginalFilename(), file.getInputStream());
        final S3Object object = ossClient.getObject("test-bucket", file.getOriginalFilename());
        final String objectURL = ossClient.getObjectURL("test-bucket", file.getOriginalFilename());
        return R.ok(objectURL, object);
    }
}

使用以上API, 即可完成对主流的几个OSS服务的对接, 不需要针对性的引入各方的sdk

以下使用的是MinIO提供的sdk

config类
对于自定义的配置类,个人还是比较推荐这种实现了InitializingBean的写法, 使用的时候代码看着简洁一些

@Getter
@Setter
@Component
@ConfigurationProperties(prefix = "minio")
public class MinioConfig implements InitializingBean {
   private String endpoint;
   private String defaultBucketName;
   private String accessKey;
   private String secretKey;

   public static String END_POINT;
   public static String DEFAULT_BUCKET_NAME;
   public static String ACCESS_KEY;
   public static String SECRET_KEY;

   @Override
   public void afterPropertiesSet() throws Exception {
      END_POINT = endpoint;
      DEFAULT_BUCKET_NAME = defaultBucketName;
      ACCESS_KEY = accessKey;
      SECRET_KEY = secretKey;
   }
}

8之后, API基本上都是使用各种Builder参数, 8之前还是根据需要的参数单独传参.
客户端对象

//新版
minioClient = MinioClient.builder()
                    .endpoint(MinioConfig.END_POINT)
                    .credentials(MinioConfig.ACCESS_KEY, MinioConfig.SECRET_KEY)
                    .build();
//旧版
minioClient = new MinioClient(endpoint, access_key, secret_key);

//创建了客户端对象点取api看看也就清楚了
//例如以文件流形式上传文件
public static boolean upload(String bucketName, String fileName, InputStream inputStream) {
 try {
        makeBucket(bucketName);
        minioClient.putObject(PutObjectArgs.builder().bucket(bucketName).object(fileName).stream(inputStream, inputStream.available(), -1).build());
        return true;
    } catch (Exception e) {
        e.printStackTrace();
        return false;
    }
}

创建永久的文件url

问题:

临时文件url不赘述,和以前一样; minio8之后不提供getObjectUrl()的API了, 其实结果也就是 endPoint+/+bucket+/+filename, 而且getObjectUrl()也得是bucket的policy设置为public时能访问到文件, 如果单纯将策略设置为public, bucket下的文件也可以通过url遍历出来, 不合适.

解决:

7的bucket就给了private和public两种类型, 8之后多了一个custom, 所以正好通过这个custom配置bucket的自定义policy

通过控制台将bucket先设置为public保存,然后再改为custom可以获得public的policy配置, 通过该配置删除部分权限达到目的.

因为bucket跟日期相关, 我选择在每次创建bucket的时候设置此策略为默认策略在这里插入图片描述
代码实现
${bucketName} 需要自行替换为bucket名称

	/**
     * 默认自定义策略，不可以通过url直接查询bucket下所有文件
     */
    private static final String DEFAULT_POLICY = "{\n" +
    		//version这里没去管,直接不改
            "    \"Version\": \"2012-10-17\",\n" +
            "    \"Statement\": [\n" +
            "        {\n" +
            "            \"Effect\": \"Allow\",\n" +
            "            \"Principal\": {\n" +
            "                \"AWS\": [\n" +
            "                    \"*\"\n" +
            "                ]\n" +
            "            },\n" +
            //根据具体action类型增加
            "            \"Action\": [\n" +
            "                \"s3:GetBucketLocation\"\n" +
            "            ],\n" +
            "            \"Resource\": [\n" +
            "                \"arn:aws:s3:::${bucketName}\"\n" +
            "            ]\n" +
            "        },\n" +
            "        {\n" +
            "            \"Effect\": \"Allow\",\n" +
            "            \"Principal\": {\n" +
            "                \"AWS\": [\n" +
            "                    \"*\"\n" +
            "                ]\n" +
            "            },\n" +
            //根据具体action类型增加
            "            \"Action\": [\n" +
            //可见action只提供了getObject的权限
            "                \"s3:GetObject\"\n" +
            "            ],\n" +
            "            \"Resource\": [\n" +
            "                \"arn:aws:s3:::${bucketName}/*\"\n" +
            "            ]\n" +
            "        }\n" +
            "    ]\n" +
            "}";

 	/**
     * 设置bucket策略
     *
     * @param bucketName bucket名称
     */
    public static void bucketPolicy(String bucketName) {
        try {
            minioClient.setBucketPolicy(
                    SetBucketPolicyArgs.builder().bucket(bucketName).config(processPolicy(bucketName)).build());
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

	/**
     * minio 8.x.x后版本不再提供getObjectUrl方法
     * 实际返回就是 endpoint + bucket + filename
     *
     * @param bucketName bucket名称
     * @param fileName   文件名
     * @return 获取文件的url contentType = application/octet-stream
     */
    public static String fileUrl(String bucketName, String fileName) {
        try {
            return MinioConfig.END_POINT + "/" + bucketName + "/" + fileName;
        } catch (Exception e) {
            throw new RuntimeException("获取文件url异常：" + e.getMessage());
        }
    }

当访问到目录时提示无权限, 当访问具体文件时可以通过固定的url直接下载文件
curl 192.168.0.192:9100/2022-05/

<?xml version="1.0" encoding="UTF-8"?>
<Error>
<Code>AccessDenied</Code>
<Message>Access Denied.</Message>
<BucketName>2022-05</BucketName>
<Resource>/2022-05/</Resource>
<RequestId>16F1AD3A9B8424EB</RequestId>
<HostId>fa36963f-b9d4-4dfe-86e3-db649276277d</HostId>
</Error>%