springboot AWS S3的使用案例

转x33g5p2x  于2022-03-29 转载在 Spring  
字(11.3k)|赞(0)|评价(0)|浏览(416)

项目中有将文件存储在S3上，怎么获取，以下是使用案例。

pom.xml

<dependency>
            <groupId>com.amazonaws</groupId>
            <artifactId>aws-java-sdk-s3</artifactId>
            <version>1.12.184</version>
        </dependency>

application.yml

aws-s3:
  bucketName: hsx-test
  region: cn-northwest-1
  hosts: s3.cn-northwest-1.amazonaws.com.cn
  accessKey: XXXX
  secretKey: XXXXXXXXXXX
  amazonS3MaxConnections: 200
  amazonS3SocketTimeout: 10000
  amazonS3MaxErrorRetry: 2
  amazonS3Protocol: http

AwsS3Config.java

/**
 * @program: aws_s3
 * @description: UploadConfig
 * @author: sunyuhua
 * @create: 2022-03-28 10:07
 **/
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import java.util.List;

@ConfigurationProperties(prefix = "aws-s3")
@Data
public class AwsS3Config {

	/**
	 * S3支持集群的链接方式
	 */
	private List<String> hosts;

	private String accessKey;

	private String secretKey;
	/**
	 * 区域
	 */
	private String region;
	/**
	 * 连接协议
	 */
	private String amazonS3Protocol;

	/**
	 * 失败请求重试次数
	 */
	private Integer amazonS3MaxErrorRetry;

	/**
	 * 超时时间
	 */
	private Integer amazonS3SocketTimeout;

	/**
	 * 最大连接数
	 */
	private Integer amazonS3MaxConnections;

	/**
	 * 桶名称
	 */
	private String bucketName;
}

AmazonS3Config.java

package com.fastonetech.aws_s3;

import com.amazonaws.ClientConfiguration;
import com.amazonaws.Protocol;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.CreateBucketRequest;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

import javax.annotation.Resource;

@Configuration
@EnableConfigurationProperties({AwsS3Config.class})
public class AmazonS3Config {

	@Resource
	private AwsS3Config awsS3Config;

	@Bean(name = "amazonS3")
	public AmazonS3 getAmazonS3() {
		String accessKey = awsS3Config.getAccessKey();
		String secretKey = awsS3Config.getSecretKey();
		AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
		ClientConfiguration conf = new ClientConfiguration();
		// 设置AmazonS3使用的最大连接数
		conf.setMaxConnections(awsS3Config.getAmazonS3MaxConnections());
		// 设置socket超时时间
		conf.setSocketTimeout(awsS3Config.getAmazonS3SocketTimeout());
		// 设置失败请求重试次数
		conf.setMaxErrorRetry(awsS3Config.getAmazonS3MaxErrorRetry());
		// 设置协议
		if (!"blank".equals(awsS3Config.getAmazonS3Protocol())) {
			switch (awsS3Config.getAmazonS3Protocol()) {
				case "https":
					conf.setProtocol(Protocol.HTTPS);
					break;
				case "http":
					conf.setProtocol(Protocol.HTTP);
					break;
				default:
					break;
			}
		}
		AmazonS3 s3Client = AmazonS3ClientBuilder.standard()
				.withCredentials(new AWSStaticCredentialsProvider(credentials))
				.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(awsS3Config.getHosts().get(0), awsS3Config.getRegion()))
				.withClientConfiguration(conf)
				.build();
		checkAndCreateBucket(s3Client);
		return s3Client;
	}

	@Bean(name = "transferManager")
	public TransferManager getTransferManager() {
		return TransferManagerBuilder.standard().withS3Client(getAmazonS3()).build();
	}

	/**
	 * 检查桶是否存在，不存在则创建创建
	 *
	 * @param s3Client
	 */
	private void checkAndCreateBucket(AmazonS3 s3Client) {
		boolean exists = s3Client.doesBucketExistV2(awsS3Config.getBucketName());
		if (!exists) {
			CreateBucketRequest request = new CreateBucketRequest(awsS3Config.getBucketName());
			s3Client.createBucket(request);
		}
	}
}

AWSS3Controller.java

import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

import javax.annotation.Resource;
import java.io.IOException;

/**
 * @program: aws_s3
 * @description:
 * @author: sunyuhua
 * @create: 2022-03-28 10:25
 **/
@RestController
@RequestMapping("/s3/")
public class AWSS3Controller {

	@Resource
	private  AWSS3Service S3Service;

	/**
	 * 获取S3上文件的内容
	 * @throws IOException
	 */
	@GetMapping("/getObject")
	public void getObject()  throws IOException  {
		String objectName="dev-nat-log/AWSLogs/085211525492/vpcflowlogs/cn-northwest-1/2022/01/02/085211525492_vpcflowlogs_cn-northwest-1_fl-0a3c82b9af904d152_20220102T0000Z_bf6a5f7a.log.gz";
		 S3Service.getObject(objectName);
	}

	/**
	 * 获取文件的ObjectMetadata
	 * @throws IOException
	 */
	@GetMapping("/getObjectMetadata")
	public void getObjectMetadata() throws  IOException{
		S3Service.getObjectMetadata();
	}

	/**
	 * 获取文件的ObjectSummaries
	 */
	@GetMapping("/getObjectSummaries")
	public void  getObjectSummaries(){
		S3Service.getObjectSummaries();
	}

	/**
	 * 列出指定文件夹下的所有文件 V1版本
	 * @throws IOException
	 */
	@GetMapping("/listObjects")
	public void listObjects() throws IOException {
		S3Service.listObjects();
	}

	/**
	 * 列出指定文件夹下的所有文件 V2版本
	 * @throws IOException
	 */
	@GetMapping("/listObjectsV2")
	public void listObjectsV2() throws IOException{
		S3Service.listObjectsV2();
	}

	/**
	 * 复制文件
	 * @throws IOException
	 */
	@GetMapping("/copyObject")
	public void copyObject() throws  IOException{
		S3Service.copyObject();
	}

	/**
	 * 删除文件
	 * @throws IOException
	 */
	@GetMapping("/deleteObject")
	public void deleteObject() throws IOException{
		S3Service.deleteObject();
	}

}

AWSS3Service.java

import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.DeleteObjectRequest;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.ListObjectsV2Request;
import com.amazonaws.services.s3.model.ListObjectsV2Result;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;

import javax.annotation.Resource;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.zip.GZIPInputStream;

/**
 * @program: aws_s3
 * @description:
 * @author: sunyuhua
 * @create: 2022-03-28 10:22
 **/
@Service
@Slf4j
public class AWSS3Service {

	@Resource
	private AmazonS3 amazonS3;

	public  void getObject(String objectName) throws IOException {
		String bucketName="hsx-test";
		S3Object fileObject = amazonS3.getObject(bucketName,objectName);
		//可以通过getObjectMetadata方法获取对象的ContentType等元数据信息
		String contentType = fileObject.getObjectMetadata().getContentType();
		GZIPInputStream gZIPInputStream = null;
		if(contentType.equals("application/x-gzip")){
			gZIPInputStream = new GZIPInputStream(fileObject.getObjectContent());
			BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(gZIPInputStream));
			String line = null;
			//第一行是标题，跳过第一行
			bufferedReader.readLine();
			while ((line = bufferedReader.readLine()) != null) {
				//转化为UTF-8的编码格式
				line = new String(line.getBytes(StandardCharsets.UTF_8));
				//---此五分钟代表没有流量
				if(line.contains("-")) continue;
				System.out.println("\n" + line);
				System.out.println("******"+(line.split("\\s+")[0].trim())+"******");
				System.out.println(line+"是否是内网"+IpUtils.internalIp((line.split("\\s+")[0])));
				//System.out.println("\n" + line);
			}
			bufferedReader.close();
		}else{
			BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(fileObject.getObjectContent()));
			String line = null;
			while ((line = bufferedReader.readLine()) != null) {
				//转化为UTF-8的编码格式
				//IpUtils.internalIp(line.getBytes(StandardCharsets.UTF_8))
				System.out.println("\n********" +new String(subBytes(line.getBytes(StandardCharsets.UTF_8),0,15)) );
				line = new String(line.getBytes(StandardCharsets.UTF_8));
				//System.out.println("\n" + line);
			}
			bufferedReader.close();
		}
	}

	public static byte[] subBytes(byte[] src, int begin, int count) {
		byte[] bs = new byte[count];
		for (int i=begin;i<begin+count; i++) bs[i-begin] = src[i];
		return bs;
	}

	public void getObjectMetadata(){
		String bucketName="hsx-test";
		String objectName="dev-nat-log/AWSLogs/085211525492/vpcflowlogs/cn-northwest-1/2022/01/02/085211525492_vpcflowlogs_cn-northwest-1_fl-0a3c82b9af904d152_20220102T0000Z_bf6a5f7a.log.gz";
		ObjectMetadata objectMetadata=amazonS3.getObjectMetadata(bucketName,objectName);
		log.info("objectMetadata"+objectMetadata.toString());
	}

	/**
	 * ObjectSummaries
	 */
	public void  getObjectSummaries(){
		String bucketName="hsx-test";
		ObjectListing objectListing = amazonS3.listObjects(bucketName);
		List<S3ObjectSummary> sums = objectListing.getObjectSummaries();
		for (S3ObjectSummary s : sums) {
			System.out.println("\t" + s.getKey());
		}
	}

	public void listObjects() throws IOException {
		String bucketName="hsx-test";
		ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
		listObjectsRequest.setBucketName(bucketName);
		listObjectsRequest.setMaxKeys(20);
		listObjectsRequest.setDelimiter("/");
		listObjectsRequest.setPrefix("dev-nat-log/AWSLogs/085211525492/vpcflowlogs/cn-northwest-1/2022/02/07/");
		ObjectListing listObjects = amazonS3.listObjects(listObjectsRequest);
		do {
			if (listObjects.isTruncated()) {
				ListObjectsRequest request = new ListObjectsRequest();
				request.setBucketName(listObjectsRequest.getBucketName());
				request.setMarker(listObjects.getNextMarker());
				listObjects =  amazonS3.listObjects(request);
				List<S3ObjectSummary> sums = listObjects.getObjectSummaries();
				for (S3ObjectSummary s : sums) {
					if(s.getKey().contains("dev-nat-log/AWSLogs/085211525492/vpcflowlogs/cn-northwest-1/2022/02/07/")){
						//System.out.println("\t" + s.getKey());
						getObject(s.getKey());
					}
				}
			} else {
				break;
			}
		} while (listObjects != null);
	}

	public void listObjectsV2() throws IOException {
		String bucketName="hsx-test";
		ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(bucketName).withMaxKeys(300)
				.withDelimiter("/").withPrefix("dev-nat-log/AWSLogs/085211525492/vpcflowlogs/cn-northwest-1/2022/02/07/");
		ListObjectsV2Result listObjects = amazonS3.listObjectsV2(req);
		do {
			if (listObjects.isTruncated()) {
				ListObjectsV2Request request = new ListObjectsV2Request();
				request.withPrefix(listObjects.getPrefix());
				request.withBucketName(listObjects.getBucketName());
				listObjects =  amazonS3.listObjectsV2(request);
				List<S3ObjectSummary> sums = listObjects.getObjectSummaries();
				for (S3ObjectSummary s : sums) {
					if(s.getKey().contains("dev-nat-log/AWSLogs/085211525492/vpcflowlogs/cn-northwest-1/2022/02/07/")
					&& !s.getKey().contains("back")
					){
						getObject(s.getKey());
					}
				}
			} else {
				break;
			}
		} while (listObjects != null);
	}


	public void copyObject(){
		String from_bucket="hsx-test";
		String from_object_key="dev-nat-log/AWSLogs/085211525492/vpcflowlogs/cn-northwest-1/2022/02/02/085211525492_vpcflowlogs_cn-northwest-1_fl-0a3c82b9af904d152_20220202T0000Z_36cb77fa.log.gz";
		String to_bucket="hsx-test";
		String to_object_key="dev-nat-log/AWSLogs/085211525492/vpcflowlogs/cn-northwest-1/2022/02/02/back/085211525492_vpcflowlogs_cn-northwest-1_fl-0a3c82b9af904d152_20220202T0000Z_36cb77fa.log.gz";
		amazonS3.copyObject(from_bucket, from_object_key, to_bucket, to_object_key);
	}

	public void deleteObject(){
		String from_bucket="hsx-test";
		String from_object_key="dev-nat-log/AWSLogs/085211525492/vpcflowlogs/cn-northwest-1/2022/02/02/085211525492_vpcflowlogs_cn-northwest-1_fl-0a3c82b9af904d152_20220202T0005Z_331483d7.log.gz";
		amazonS3.deleteObject(new DeleteObjectRequest(from_bucket, from_object_key));
		log.info("删除执行完毕");
	}
}

版权说明 : 本文为转载文章, 版权归原作者所有 版权申明

原文链接 : https://blog.csdn.net/sunyuhua_keyboard/article/details/123813325

内容来源于网络，如有侵权，请联系作者删除！

AWS S3

相关文章

    查看更多
    微信公众号

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新文章

    更多
    • 蜀ICP备13028337号-1 大数据知识库 https://www.saoniuhuo.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com