配置类
package com.unicom.microserv.ai.semantic.config;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@Component
public class OSSProperties {
@Value("${endpoints}")
private String endpoint;
@Value("${accessKeyId}")
private String accessKeyId;
@Value("${accessKeySecret}")
private String accessKeySecret;
@Value("${bucketName}")
private String bucketName;
public String getEndpoint() {
return endpoint;
}
public void setEndpoint(String endpoint) {
this.endpoint = endpoint;
}
public String getAccessKeyId() {
return accessKeyId;
}
public void setAccessKeyId(String accessKeyId) {
this.accessKeyId = accessKeyId;
}
public String getAccessKeySecret() {
return accessKeySecret;
}
public void setAccessKeySecret(String accessKeySecret) {
this.accessKeySecret = accessKeySecret;
}
public String getBucketName() {
return bucketName;
}
public void setBucketName(String bucketName) {
this.bucketName = bucketName;
}
}
类名和引用
package com.unicom.microserv.ai.semantic.util;
import com.aliyun.oss.OSS;
import com.aliyun.oss.OSSClientBuilder;
import com.aliyun.oss.model.*;
import com.unicom.microserv.ai.semantic.config.OSSProperties;
import com.unicom.microserv.ai.semantic.model.DialogueRecord;
import org.apache.commons.beanutils.BeanUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.lang.reflect.InvocationTargetException;
import java.net.SocketException;
import java.util.*;
import java.util.stream.Collectors;
@Component
public class OSSUtil {
private static Logger logger = LoggerFactory.getLogger(OSSUtil.class);
@Autowired
private OSSProperties ossProperties;
}
获取oss客户端实例
public OSS getOSSClient(){
OSS ossClient = new OSSClientBuilder().build(
ossProperties.getEndpoint(),
ossProperties.getAccessKeyId(),
ossProperties.getAccessKeySecret());
return ossClient;
}
关闭oss客户端实例
public void shutdown(OSS ossClient){
ossClient.shutdown();
}
上传文本
public void putObject( String ossKey, String content){
OSS ossClient = new OSSClientBuilder().build(ossProperties.getEndpoint(), ossProperties.getAccessKeyId(),
ossProperties.getAccessKeySecret());
PutObjectRequest putObjectRequest = new PutObjectRequest(ossProperties.getBucketName(), ossKey, new ByteArrayInputStream(content.getBytes()));
ossClient.putObject(putObjectRequest);
ossClient.shutdown();
}
追加写入oss,循环调用
public void appendToOss(OSS ossClient, String ossKey, String TsvTitles, List<String> list) throws SocketException {
String content = list.stream().collect(Collectors.joining("\n")) +"\n";
logger.info("数据写入oss start--------->name:{},size:{}", ossKey , list.size());
boolean objectExist = ossClient.doesObjectExist(ossProperties.getBucketName(), ossKey);
long position = 0L;
if ( objectExist ){
ObjectMetadata objectMetadata = ossClient.getObjectMetadata(ossProperties.getBucketName(), ossKey);
position = objectMetadata.getContentLength();
}else{
content = TsvTitles + "\n" + content;
}
ObjectMetadata meta = new ObjectMetadata();
meta.setContentType("text/plain");
AppendObjectRequest appendObjectRequest = new AppendObjectRequest(ossProperties.getBucketName(), ossKey,
new ByteArrayInputStream(content.getBytes()),meta);
appendObjectRequest.setPosition(position);
ossClient.appendObject(appendObjectRequest);
logger.info("数据写入oss end --------->name:{},size:{}", ossKey , list.size());
}
追加写入oss,单次调用
public void appendToOss(String ossKey, String TsvTitles, List<String> list) throws SocketException {
String content = list.stream().collect(Collectors.joining("\n")) +"\n";
logger.info("数据写入oss start--------->name:{},size:{}", ossKey , list.size());
OSS ossClient = new OSSClientBuilder().build(ossProperties.getEndpoint(), ossProperties.getAccessKeyId(),
ossProperties.getAccessKeySecret());
boolean objectExist = ossClient.doesObjectExist(ossProperties.getBucketName(), ossKey);
long position = 0L;
if ( objectExist ){
ObjectMetadata objectMetadata = ossClient.getObjectMetadata(ossProperties.getBucketName(), ossKey);
position = objectMetadata.getContentLength();
}else{
content = TsvTitles + "\n" + content;
}
ObjectMetadata meta = new ObjectMetadata();
meta.setContentType("text/plain");
AppendObjectRequest appendObjectRequest = new AppendObjectRequest(ossProperties.getBucketName(), ossKey,
new ByteArrayInputStream(content.getBytes()),meta);
appendObjectRequest.setPosition(position);
ossClient.appendObject(appendObjectRequest);
ossClient.shutdown();
logger.info("数据写入oss end --------->name:{},size:{}", ossKey , list.size());
}
获取oss文件,并转为 List
public List<String> getOSSList(String ossKey){
logger.info("oss --------->ossKey:{}", ossKey );
ArrayList<String> resultList = new ArrayList<>();
OSS ossClient = new OSSClientBuilder().build(ossProperties.getEndpoint(), ossProperties.getAccessKeyId(),
ossProperties.getAccessKeySecret());
try{
boolean objectExist = ossClient.doesObjectExist(ossProperties.getBucketName(), ossKey);
if ( objectExist ){
OSSObject ossObject = ossClient.getObject(ossProperties.getBucketName(), ossKey);
BufferedReader reader = new BufferedReader(new InputStreamReader(ossObject.getObjectContent()));
while (true) {
String line = reader.readLine();
if (line == null) {
break;
}
resultList.add(line);
}
reader.close();
ossObject.close();
}
} catch (IOException e){
logger.error(e.getMessage());
} finally {
ossClient.shutdown();
}
return resultList;
}
查询目录下文件名称列表
public List<String> getObjectKeyList(OSS ossClient, String KeyPrefix){
List<String> objectList = new ArrayList<>();
String nextMarker = null;
ObjectListing objectListing = null;
do {
ListObjectsRequest listObjectsRequest = new ListObjectsRequest(ossProperties.getBucketName())
.withPrefix(KeyPrefix)
.withMarker(nextMarker);
objectListing = ossClient.listObjects(listObjectsRequest);
if (objectListing.getObjectSummaries().size() > 0) {
for (OSSObjectSummary s : objectListing.getObjectSummaries()) {
objectList.add(s.getKey());
}
}
nextMarker = objectListing.getNextMarker();
} while (objectListing.isTruncated());
return objectList;
}
查询目录下文件名称集合
public Set<String> getObjectKeySet(OSS ossClient, String KeyPrefix){
Set<String> objectKeySet = new HashSet<>();
String nextMarker = null;
ObjectListing objectListing = null;
do {
ListObjectsRequest listObjectsRequest = new ListObjectsRequest(ossProperties.getBucketName())
.withPrefix(KeyPrefix)
.withMarker(nextMarker);
objectListing = ossClient.listObjects(listObjectsRequest);
if (objectListing.getObjectSummaries().size() > 0) {
for (OSSObjectSummary s : objectListing.getObjectSummaries()) {
objectKeySet.add(s.getKey());
}
}
nextMarker = objectListing.getNextMarker();
} while (objectListing.isTruncated());
return objectKeySet;
}
删除文件或目录
public void deleteObject(OSS ossClient, String ossKey){
ossClient.deleteObject(ossProperties.getBucketName(), ossKey);
}
批量删除文件(最多1000条)
public void deleteObjects(OSS ossClient, List<String> keys){
DeleteObjectsResult deleteObjectsResult = ossClient.deleteObjects(new DeleteObjectsRequest(ossProperties.getBucketName()).withKeys(keys));
deleteObjectsResult.getDeletedObjects();
}
批量删除文件,按文件修改时间,删除reserveDayNum天前的文件。
public void deleteObjects(OSS ossClient, String KeyPrefix, int reserveDayNum){
Calendar calendar = Calendar.getInstance();
calendar.add(Calendar.DATE, -reserveDayNum);
Date reserveDate = calendar.getTime();
String nextMarker = null;
ObjectListing objectListing = null;
do {
ListObjectsRequest listObjectsRequest = new ListObjectsRequest(ossProperties.getBucketName())
.withPrefix(KeyPrefix)
.withMarker(nextMarker);
objectListing = ossClient.listObjects(listObjectsRequest);
if (objectListing.getObjectSummaries().size() > 0) {
List<String> keys = new ArrayList<String>();
for (OSSObjectSummary s : objectListing.getObjectSummaries()) {
Date LastModified = s.getLastModified();
if(LastModified.getTime() < reserveDate.getTime()){
keys.add(s.getKey());
logger.info("=====================del key:{}",s.getKey());
}
}
if(keys.size() > 0){
DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(ossProperties.getBucketName()).withKeys(keys);
ossClient.deleteObjects(deleteObjectsRequest);
}
}
nextMarker = objectListing.getNextMarker();
} while (objectListing.isTruncated());
}
获取oss文件,并转为 List
public List<DialogueRecord> getOSSData(OSS ossClient, String ossKey){
List<DialogueRecord> dialogueRecordList = new ArrayList<>();
logger.info("oss --------->ossKey:{}", ossKey );
try{
boolean objectExist = ossClient.doesObjectExist(ossProperties.getBucketName(), ossKey);
if ( objectExist ){
OSSObject ossObject = ossClient.getObject(ossProperties.getBucketName(), ossKey);
BufferedReader reader = new BufferedReader(new InputStreamReader(ossObject.getObjectContent()));
int i = 0;
String titles = "";
while (true) {
String line = reader.readLine();
if (line == null) {
break;
}
i++;
if( i == 1 ){
titles = line;
continue;
}
DialogueRecord dialogueRecord = tsvToObject(titles, line);
dialogueRecordList.add(dialogueRecord);
}
reader.close();
ossObject.close();
}
} catch (IOException e){
logger.error(e.getMessage());
} catch (InvocationTargetException e){
logger.error(e.getMessage());
} catch (IllegalAccessException e){
logger.error(e.getMessage());
}
return dialogueRecordList;
}
private DialogueRecord tsvToObject(String titlesStr, String content)
throws InvocationTargetException, IllegalAccessException {
HashMap dataMap = new HashMap<String, String>();
String[] titles = titlesStr.split("\t",-1);
String[] cols = content.split("\t",-1);
for (int i = 0; i < titles.length ; i++) {
dataMap.put(titles[i],cols[i]);
}
DialogueRecord dialogueRecord = new DialogueRecord();
BeanUtils.populate(dialogueRecord,dataMap);
return dialogueRecord;
}
}