迅捷开发环境，hadoop测试用例提交。

ipu-hadoop-example/aaa.txt

@@ -0,0 +1,8 @@
111
222 333
444 555 666
777 888
999
0000000000

ipu-hadoop-example/pom.xml

@@ -0,0 +1,57 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
	<modelVersion>4.0.0</modelVersion>
	<parent>
		<groupId>com.ai.ipu</groupId>
		<artifactId>ipu-aggregator</artifactId>
		<version>3.1-SNAPSHOT</version>
	</parent>
	<groupId>com.ai.ipu.example</groupId>
	<artifactId>ipu-hadoop-example</artifactId>
	<version>1.0</version>
	<packaging>jar</packaging>
	<name>ipu-hadoop-example</name>
	<url>http://maven.apache.org</url>
	<properties>
		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
		<ipu>3.1-SNAPSHOT</ipu>
		<jdk>1.8</jdk>
		<junit>4.12</junit>
		<hadoop-client>2.7.3</hadoop-client>
	</properties>
	<dependencies>
		<dependency>
			<groupId>junit</groupId>
			<artifactId>junit</artifactId>
			<version>${junit}</version>
		</dependency>
		<dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-client</artifactId>
			<version>${hadoop-client}</version>
		</dependency>
		<dependency>
			<groupId>com.ai.ipu</groupId>
			<artifactId>ipu-basic</artifactId>
		</dependency>
	</dependencies>
	<build>
		<plugins>
			<plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-surefire-plugin</artifactId>
                <configuration>
                    <skip>true</skip>
                </configuration>
            </plugin>
		</plugins>
	</build>
</project>

ipu-hadoop-example/src/test/java/com/ai/ipu/example/hadoop/HadoopConfig.java

@@ -0,0 +1,67 @@
package com.ai.ipu.example.hadoop;
import com.ai.ipu.basic.file.ResourceBundleUtil;
import com.ai.ipu.basic.log.ILogger;
import com.ai.ipu.basic.log.IpuLoggerFactory;
/**
 * 类描述
 *
 * @Author: lilb3@asiainfo.com
 * @Date: 2019-06-18 17:05
 **/
public class HadoopConfig {
    private static final ILogger LOGGER = IpuLoggerFactory.createLogger(HadoopConfig.class);
    private static final String CONFIG_FILE_PATH = "hadoop";
    /*要连接的Hadoop地址URI， 如hdfs://ip:port*/
    private static String hdfsPath;
    /*是否支持使用Host Name连接Hadoop， true：支持 false：不支持*/
    private static String dfsClientUseDatanodeHostname;
    /*测试创建的Dir的名称*/
    private static String dirName;
    /*测试创建的file的名称， 文件路径为 /mkdir.name/*/
    private static String fileName;
    /*测试写入文件的字符串*/
    private static String writeString;
    /*测试上传到hdfs的文件名，本地需要有对应的文件*/
    private static String uploadLocalFile;
    /*测试下载到本地的文件名*/
    private static String downloadLocalFile;
    public static String getHdfsPath() {
        return hdfsPath;
    }
    public static String getDfsClientUseDatanodeHostname() {
        return dfsClientUseDatanodeHostname;
    }
    public static String getDirName() {
        return dirName;
    }
    public static String getFileName() {
        return fileName;
    }
    public static String getWriteString() {
        return writeString;
    }
    public static String getUploadLocalFile() {
        return uploadLocalFile;
    }
    public static String getDownloadLocalFile() {
        return downloadLocalFile;
    }
    /*加载配置文件*/
    static {
        try {
            ResourceBundleUtil.initialize(CONFIG_FILE_PATH, HadoopConfig.class);
        } catch (Exception e) {
            LOGGER.error(CONFIG_FILE_PATH + "配置文件读取失败", e);
        }
    }
}

ipu-hadoop-example/src/test/java/com/ai/ipu/example/hadoop/HadoopExample.java

@@ -0,0 +1,108 @@
package com.ai.ipu.example.hadoop;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.net.URI;
/**
 * 类描述
 *
 * @Author: lilb3@asiainfo.com
 * @Date: 2019-05-24 17:10
 **/
public class HadoopExample {
    private FileSystem hdfs = null;
    private static final String HDFS_SPLIT_STRING = "/";
    @Before
    public void setBefore() throws Exception {
        Configuration configuration = new Configuration();
        configuration.set("dfs.client.use.datanode.hostname", HadoopConfig.getDfsClientUseDatanodeHostname());
        String hdfsPath = HadoopConfig.getHdfsPath();
        hdfs = FileSystem.get(new URI(hdfsPath), configuration);
    }
    @After
    public void setAfter() throws Exception {
        if (hdfs != null) {
            hdfs.close();
        }
    }
    @Test
    public void mkdir() throws Exception {
        String newDir = HDFS_SPLIT_STRING + HadoopConfig.getDirName();
        boolean result = hdfs.mkdirs(new Path(newDir));
        if (result) {
            System.out.println("Success!");
        } else {
            System.out.println("Failed!");
        }
    }
    @Test
    public void createFile() throws Exception {
        String filePath = HDFS_SPLIT_STRING + HadoopConfig.getDirName() + HDFS_SPLIT_STRING + HadoopConfig.getFileName();
        FSDataOutputStream create = hdfs.create(new Path(filePath));
        create.writeBytes(HadoopConfig.getWriteString());
        create.close();
        System.out.println("Finish!");
    }
    @Test
    public void copyFromLocalFile() throws Exception {
        String localFile = HadoopConfig.getUploadLocalFile();
        String toHDFS = HDFS_SPLIT_STRING + HadoopConfig.getDirName() + HDFS_SPLIT_STRING;
        hdfs.copyFromLocalFile(new Path(localFile), new Path(toHDFS));
        System.out.println("Finish!");
    }
    @Test
    public void copyToLocalFile() throws Exception {
        String hdfsFile = HDFS_SPLIT_STRING + HadoopConfig.getDirName() + HDFS_SPLIT_STRING + HadoopConfig.getFileName();
        String localFile = HadoopConfig.getDownloadLocalFile();
        hdfs.copyToLocalFile(false, new Path(hdfsFile), new Path(localFile));
        System.out.println("Finish!");
    }
    @Test
    public void listFile() throws Exception {
        iteratorListFile(hdfs, new Path(HDFS_SPLIT_STRING));
    }
    @Test
    public void locateFile() throws Exception {
        Path file = new Path(HDFS_SPLIT_STRING + HadoopConfig.getDirName() + HDFS_SPLIT_STRING + HadoopConfig.getFileName());
        FileStatus fileStatus = hdfs.getFileStatus(file);
        BlockLocation[] location = hdfs.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
        for (BlockLocation block : location) {
            String[] hosts = block.getHosts();
            for (String host : hosts) {
                System.out.println("block:" + block + " host:" + host);
            }
        }
    }
    private static void iteratorListFile(FileSystem hdfs, Path path) throws Exception {
        FileStatus[] files = hdfs.listStatus(path);
        for (FileStatus file : files) {
            if (file.isDirectory()) {
                System.out.println(file.getPermission() + " " + file.getOwner()
                        + " " + file.getGroup() + " " + file.getPath());
                iteratorListFile(hdfs, file.getPath());
            } else if (file.isFile()) {
                System.out.println(file.getPermission() + " " + file.getOwner()
                        + " " + file.getGroup() + " " + file.getPath());
            }
        }
    }
}

ipu-hadoop-example/src/test/resources/hadoop.properties

@@ -0,0 +1,14 @@
#要连接的Hadoop地址URI， 如hdfs://ip:port
hdfs.path=hdfs://iZm5e5xe1w25avi0io1f5aZ:9000
#是否支持使用Host Name连接Hadoop， true：支持<此处应设置为true> false：不支持
dfs.client.use.datanode.hostname=true
#测试创建的Dir的名称
dir.name=hdfstest
#测试创建的file的名称， 文件路径为 /mkdir.name/
file.name=touchfile
#测试写入文件的字符串
write.string=abcdefghijklmnopqrstuvwxyz1234567890
#测试上传到hdfs的文件名，本地需要有对应的文件
upload.local.file=aaa.txt
#测试下载之后存在本地的文件名
download.local.file=bbb.txt

ipu-hadoop-example/src/test/resources/log4j.properties

@@ -0,0 +1,8 @@
log4j.rootLogger=error, console
log4j.logger.org.apache.zookeeper=error
log4j.logger.com.ai.ipu.example=debug
log4j.appender.console=org.apache.log4j.ConsoleAppender
log4j.appender.console.target=System.out
log4j.appender.console.layout=org.apache.log4j.PatternLayout
log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %5p [%t] (%F:%L) - %m%n