@IPU_REQ_2021@weihf@添加ipu-s3示例

ipu-rest-demo/pom.xml

@@ -105,7 +105,16 @@
			<groupId>com.ai.ipu.server</groupId>
			<artifactId>ipu-restful</artifactId>
		</dependency>
		
		<dependency>
		    <groupId>com.ai.ipu</groupId>
		    <artifactId>ipu-s3</artifactId>
		    <version>${ipu}</version>
		</dependency>
		<dependency>
		    <groupId>com.google.guava</groupId>
		    <artifactId>guava</artifactId>
		    <version>29.0-jre</version>
		</dependency>
        <!-- 这里引入mongo-java-driver的原因是springboot1.5.9缺省使用了mongo-java-driver低版本的驱动，
                                导致ipu-nosql里MongoCredential.createScramSha256Credential报“方法不存在”。
                               由于ipu-rest-libs里引入springboot采用了import方式；

ipu-rest-demo/src/main/java/com/ai/ipu/server/demo/control/dfs/MinIOController.java

@@ -0,0 +1,132 @@
package com.ai.ipu.server.demo.control.dfs;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.ResponseBody;
import com.ai.ipu.data.JMap;
import com.ai.ipu.data.impl.JsonMap;
import com.ai.ipu.server.demo.service.MinIOServiceImpl;
/**
 * 基于ipu-s3组件对MinIO操作范例
 * @author weihf@asiainfo.com
 * @team IPU
 * @date 2021年3月3日
 */
@Controller
@RequestMapping("/minio/")
public class MinIOController {
	private static final String DEFAULT_DFS_NAME = "test";
	private static final String DEFAULT_BUCKET_NAME = "test";
	private static final String LOCAL_FILE_PATH = "/data";
	private static final int DEFAULT_EXPIRE = 300;
	
	@Autowired
	private MinIOServiceImpl service;
	
	@ResponseBody
    @RequestMapping("/createBucket")
    public JMap createBucket(JMap param) throws Exception {
		String bucketName = param.getString("bucketName", DEFAULT_BUCKET_NAME);
		JMap result = new JsonMap();
		result.put("result", service.createBucket(DEFAULT_DFS_NAME, bucketName));
		return result;
	}
	
	@ResponseBody
    @RequestMapping("/deleteBucketWithoutToken")
    public JMap deleteBucketWithoutToken(JMap param) throws Exception {
		String bucketName = param.getString("bucketName", DEFAULT_BUCKET_NAME);
		JMap result = new JsonMap();
		result.put("result", service.deleteBucketWithoutToken(DEFAULT_DFS_NAME, bucketName));
		return result;
	}
	
	@ResponseBody
    @RequestMapping("/deleteBucketWithToken")
    public JMap deleteBucketWithToken(JMap param) throws Exception {
		String bucketName = param.getString("bucketName", DEFAULT_BUCKET_NAME);
		JMap result = new JsonMap();
		result.put("result", service.deleteBucketWithToken(DEFAULT_DFS_NAME, bucketName));
		return result;
	}
	@ResponseBody
    @RequestMapping("/deleteFileWithoutToken")
    public JMap deleteFileWithoutToken(JMap param) throws Exception {
		String bucketName = param.getString("bucketName", DEFAULT_BUCKET_NAME);
		String minioLabel = param.getString("minioLabel");
		JMap result = new JsonMap();
		result.put("result", service.deleteFileWithoutToken(DEFAULT_DFS_NAME, bucketName, minioLabel));
		return result;
	}
	
	@ResponseBody
    @RequestMapping("/deleteFileWithToken")
    public JMap deleteFileWithToken(JMap param) throws Exception {
		String bucketName = param.getString("bucketName", DEFAULT_BUCKET_NAME);
		String minioLabel = param.getString("minioLabel");
		JMap result = new JsonMap();
		result.put("result", service.deleteFileWithToken(DEFAULT_DFS_NAME, bucketName, minioLabel));
		return result;
	}
	
	@ResponseBody
    @RequestMapping("/downloadFile")
    public JMap downloadFile(JMap param) throws Exception {
		String bucketName = param.getString("bucketName", DEFAULT_BUCKET_NAME);
		String minioLabel = param.getString("minioLabel");
		JMap result = new JsonMap();
		result.put("result", service.downloadFile(DEFAULT_DFS_NAME, bucketName, minioLabel, LOCAL_FILE_PATH + "/" + minioLabel));
		return result;
	}
	
	@ResponseBody
    @RequestMapping("/getAllBuckets")
    public JMap getAllBuckets(JMap param) throws Exception {
		JMap result = new JsonMap();
		result.put("result", service.getAllBuckets(DEFAULT_DFS_NAME));
		return result;
	}
	
	@ResponseBody
    @RequestMapping("/getAllFileByPrefix")
    public JMap getAllFileByPrefix(JMap param) throws Exception {
		String bucketName = param.getString("bucketName", DEFAULT_BUCKET_NAME);
		JMap result = new JsonMap();
		result.put("result", service.getAllByPrefix(DEFAULT_DFS_NAME, bucketName));
		return result;
	}
	
	@ResponseBody
    @RequestMapping("/getDownloadUrl")
    public JMap getDownloadUrl(JMap param) throws Exception {
		String bucketName = param.getString("bucketName", DEFAULT_BUCKET_NAME);
		String minioLabel = param.getString("minioLabel");
		int expire = param.getInt("expire", DEFAULT_EXPIRE);
		JMap result = new JsonMap();
		result.put("result", service.getDownloadUrl(DEFAULT_DFS_NAME, bucketName, minioLabel, expire));
		return result;
	}
	
	@ResponseBody
    @RequestMapping("/isBucketExist")
    public JMap isBucketExist(JMap param) throws Exception {
		String bucketName = param.getString("bucketName", DEFAULT_BUCKET_NAME);
		JMap result = new JsonMap();
		result.put("result", service.isBucketExist(DEFAULT_DFS_NAME, bucketName));
		return result;
	}
	
	@ResponseBody
    @RequestMapping("/uploadFile")
    public JMap uploadFile(JMap param) throws Exception {
		String bucketName = param.getString("bucketName", DEFAULT_BUCKET_NAME);
		String localFile = LOCAL_FILE_PATH + "/2.txt";
		JMap result = new JsonMap();
		result.put("result", service.uploadFile(DEFAULT_DFS_NAME, bucketName, localFile, "text/plain"));
		return result;
	}
}

ipu-rest-demo/src/main/java/com/ai/ipu/server/demo/service/MinIOServiceImpl.java

@@ -0,0 +1,165 @@
package com.ai.ipu.server.demo.service;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
import org.springframework.stereotype.Service;
import com.ai.ipu.basic.log.ILogger;
import com.ai.ipu.basic.log.IpuLoggerFactory;
import com.ai.ipu.dfs.s3.IFs;
import com.ai.ipu.dfs.s3.IpuDfsFactory;
import com.ai.ipu.dfs.s3.entity.BucketEntity;
import com.ai.ipu.dfs.s3.entity.DfsEntity;
import io.minio.http.Method;
import io.minio.messages.Bucket;
import io.minio.messages.Item;
@Service
public class MinIOServiceImpl {
	private ILogger log = IpuLoggerFactory.createLogger(MinIOServiceImpl.class);
	private static final String DELETE_TOKEN = "minio";
	
	public boolean isBucketExist(String dfsName, String bucketName) throws Exception {
		DfsEntity dfsEntity = IpuDfsFactory.getDfsEntity(dfsName);
		BucketEntity bucket = new BucketEntity();
		bucket.setDfsName(dfsEntity.getDfsName());
		bucket.setBucketName(bucketName);
		bucket.setRegionName(dfsEntity.getRegionName());
		IFs client = IpuDfsFactory.getFsClient(dfsEntity.getDfsName());
		return client.isBucketExist(bucket);
	}
	
	public boolean createBucket(String dfsName, String bucketName) throws Exception {
		DfsEntity dfsEntity = IpuDfsFactory.getDfsEntity(dfsName);
		BucketEntity bucket = new BucketEntity();
		bucket.setDfsName(dfsEntity.getDfsName());
		bucket.setBucketName(bucketName);
		bucket.setRegionName(dfsEntity.getRegionName());
		IFs client = IpuDfsFactory.getFsClient(dfsEntity.getDfsName());
		return client.createBucket(bucket);
	}
	
	public String getDownloadUrl(String dfsName, String bucketName, String minioLabel, int expire) throws Exception {
		DfsEntity dfsEntity = IpuDfsFactory.getDfsEntity(dfsName);
		BucketEntity bucket = new BucketEntity();
		bucket.setDfsName(dfsEntity.getDfsName());
		bucket.setBucketName(bucketName);
		bucket.setRegionName(dfsEntity.getRegionName());
		IFs client = IpuDfsFactory.getFsClient(dfsEntity.getDfsName());
		
		/* 生成临时下载链接。 */
		return client.getDownloadUrl(bucket, Method.GET, minioLabel, expire);
	}
		
	public boolean downloadFile(String dfsName, String bucketName, String minioLabel, String localFile) throws Exception {
		DfsEntity dfsEntity = IpuDfsFactory.getDfsEntity(dfsName);
		BucketEntity bucket = new BucketEntity();
		bucket.setDfsName(dfsEntity.getDfsName());
		bucket.setBucketName(bucketName);
		bucket.setRegionName(dfsEntity.getRegionName());
		IFs client = IpuDfsFactory.getFsClient(dfsEntity.getDfsName());
		return client.downloadFile(bucket, minioLabel, localFile);
	}
	
	public boolean deleteFileWithoutToken(String dfsName, String bucketName, String minioLabel) throws Exception {
		DfsEntity dfsEntity = IpuDfsFactory.getDfsEntity(dfsName);
		BucketEntity bucket = new BucketEntity();
		bucket.setDfsName(dfsEntity.getDfsName());
		bucket.setBucketName(bucketName);
		bucket.setRegionName(dfsEntity.getRegionName());
		IFs client = IpuDfsFactory.getFsClient(dfsEntity.getDfsName());
		//由于deletetoken错误，删除会抛异常
		return client.deleteFile(bucket, minioLabel, null);
	}
	
	public boolean deleteFileWithToken(String dfsName, String bucketName, String minioLabel) throws Exception {
		DfsEntity dfsEntity = IpuDfsFactory.getDfsEntity(dfsName);
		BucketEntity bucket = new BucketEntity();
		bucket.setDfsName(dfsEntity.getDfsName());
		bucket.setBucketName(bucketName);
		bucket.setRegionName(dfsEntity.getRegionName());
		IFs client = IpuDfsFactory.getFsClient(dfsEntity.getDfsName());
		return client.deleteFile(bucket, minioLabel, DELETE_TOKEN);
	}
	
	public boolean deleteBucketWithoutToken(String dfsName, String bucketName) throws Exception {
		DfsEntity dfsEntity = IpuDfsFactory.getDfsEntity(dfsName);
		BucketEntity bucket = new BucketEntity();
		bucket.setDfsName(dfsEntity.getDfsName());
		bucket.setBucketName(bucketName);
		bucket.setRegionName(dfsEntity.getRegionName());
		IFs client = IpuDfsFactory.getFsClient(dfsEntity.getDfsName());
		return client.deleteBucket(bucket, null);
	}
	
	public boolean deleteBucketWithToken(String dfsName, String bucketName) throws Exception {
		DfsEntity dfsEntity = IpuDfsFactory.getDfsEntity(dfsName);
		BucketEntity bucket = new BucketEntity();
		bucket.setDfsName(dfsEntity.getDfsName());
		bucket.setBucketName(bucketName);
		bucket.setRegionName(dfsEntity.getRegionName());
		IFs client = IpuDfsFactory.getFsClient(dfsEntity.getDfsName());
		return client.deleteBucket(bucket, DELETE_TOKEN);
	}
	
	public boolean uploadFile(String dfsName, String bucketName, String localFile, String contentType) throws Exception {
		DfsEntity dfsEntity = IpuDfsFactory.getDfsEntity(dfsName);
		BucketEntity bucket = new BucketEntity();
		bucket.setDfsName(dfsEntity.getDfsName());
		bucket.setBucketName(bucketName);
		bucket.setRegionName(dfsEntity.getRegionName());
		
		File tmpFile=new File(localFile);
        String minioLabel=tmpFile.getName();
		IFs client = IpuDfsFactory.getFsClient(dfsEntity.getDfsName());
		return client.uploadFile(bucket, localFile, minioLabel, contentType);
	}
	
	public List<String> getAllByPrefix(String dfsName, String bucketName) throws Exception {
		DfsEntity dfsEntity = IpuDfsFactory.getDfsEntity(dfsName);
		BucketEntity bucket = new BucketEntity();
		bucket.setDfsName(dfsEntity.getDfsName());
		bucket.setBucketName(bucketName);
		bucket.setRegionName(dfsEntity.getRegionName());
		IFs client = IpuDfsFactory.getFsClient(dfsEntity.getDfsName());
		List<Item> files = client.getAllByPrefix(bucket, null, 10, false);
		int i=0;
		List<String> results = new ArrayList<String>();
		log.debug("filelist results:");
		for (;i<files.size();i++) {
			log.debug(files.get(i).objectName());
			results.add(files.get(i).objectName());
		}
		return results;
	}
	
	public List<String> getAllBuckets(String dfsName) throws Exception {
		DfsEntity dfsEntity = IpuDfsFactory.getDfsEntity(dfsName);
		BucketEntity bucket = new BucketEntity();
		bucket.setDfsName(dfsEntity.getDfsName());
		IFs client = IpuDfsFactory.getFsClient(dfsEntity.getDfsName());
		List<Bucket> buckets = client.getAllBuckets(bucket);
		int i=0;
		List<String> results = new ArrayList<String>();
		log.debug("buckets results:");
		for (;i<buckets.size();i++) {
			log.debug(buckets.get(i).name());
			results.add(buckets.get(i).name());
		}
		return results;
	}
	
}

ipu-rest-demo/src/main/resources/dev/ipu-s3.xml

@@ -0,0 +1,14 @@
<?xml version = '1.0' encoding = 'UTF-8'?>
<dfs>
	<fs name="test" type="MinIO">
        <config name="DfsUrl" value="http://47.105.160.21:9000"/>
        <config name="AccessKey" value="@DES@RSzkZ6hmDVI="/>
        <config name="SecretKey" value="@DES@ObffDdlirVFZLdcs2DV3cLnXvnxZVc1y"/>
        <config name="RegionName" value="china-beijing"/>  
        <config name="Version" value=""/>
        <config name="DeleteToken" value="@DES@/T/mbyc9wik="/>
        <!-- 是否忽略ssl证书校验，缺省为false，表示需要 校验。此时非CA颁发证书需要通过jvm参数-Dipu.s3.cert=/证书目录/证书设置证书
        true表示不需要校验证书。 -->
        <config name="Insecure" value="true"/>
	</fs>
</dfs>