HdfsClient-在本地开发环境使用Java远程连接操作HDFS

一、pom.xml配置

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.lihejia</groupId>
    <artifactId>HDFSClint</artifactId>
    <version>1.0-SNAPSHOT</version>

    <properties>
        <maven.compiler.source>8</maven.compiler.source>
        <maven.compiler.target>8</maven.compiler.target>
    </properties>

    <dependencies>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
            <version>2.8.1</version>
        </dependency>

        <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-client</artifactId>
        <version>2.8.1</version>
        </dependency>

        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.12</version>
        </dependency>
        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
            <version>1.7.30</version>
        </dependency>
    </dependencies>




</project>

二、HdfsClient代码 

package com.lihejia.hdfs;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.conf.Configuration;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

public class HdfsClient {
    private FileSystem fs;
    @Before
    public void init() throws IOException, URISyntaxException, InterruptedException {
        //连接集群的地址
        URI uri = new URI("hdfs://localhost:9000");

        //创建一个配置文件夹
        Configuration configuration = new Configuration();

        //用户
        String user = "lihejia";

        //获取客户端的对象
        fs = FileSystem.get(uri, configuration, user);

    }

    @After
    public void close() throws IOException {
        //关闭资源
        fs.close();
    }

    @Test
    public void testMkdir() throws IOException {
        //创建文件夹
        fs.mkdirs(new Path("/02jia"));
    }

    @Test
    public void testRemove() throws IOException {
        //删除文件夹
        fs.delete(new Path("/02jia"),true);
    }

    @Test
    public void testCopyFromLocalFile() throws IOException {
        //拷贝本地文件到hdfs
        fs.copyFromLocalFile(new Path("/Users/lihejia/Desktop/Mario0630.txt"),new Path("/"));
    }


}

三、注意点

1、代码中的每个引入的方法记得是hadoop相关的，不要引入错了。

2、URI的地址记得要跟core-site.xml配置文件中保持一致。

譬如配置文件中的hdfs://localhost:9000对应代码中的hdfs的访问地址。

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>


<configuration>
    <property>
        <name>hadoop.tmp.dir</name>
        <value>file:/usr/local/hadoop/tmp</value>
    </property>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://localhost:9000</value>
    </property>
</configuration>

四、HDFS的API接口文件

可以参考文件中的方法根据自己需求使用，代码只是设计简单的创建文件夹、删除文件夹、拷贝文件基本操作，其他操作可以具体参考该配置文件API.

//
// Source code recreated from a .class file by IntelliJ IDEA
// (powered by FernFlower decompiler)
//

package org.apache.hadoop.fs;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import java.io.Closeable;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.lang.ref.ReferenceQueue;
import java.lang.ref.WeakReference;
import java.net.URI;
import java.net.URISyntaxException;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Collection;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.IdentityHashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.ServiceConfigurationError;
import java.util.ServiceLoader;
import java.util.Set;
import java.util.Stack;
import java.util.TreeSet;
import java.util.Map.Entry;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.ContentSummary.Builder;
import org.apache.hadoop.fs.GlobalStorageStatistics.StorageStatisticsProvider;
import org.apache.hadoop.fs.Options.ChecksumOpt;
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.MultipleIOException;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.ClassUtil;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.DataChecksum.Type;
import org.apache.htrace.core.TraceScope;
import org.apache.htrace.core.Tracer;

@Public
@Stable
public abstract class FileSystem extends Configured implements Closeable {
    public static final String FS_DEFAULT_NAME_KEY = "fs.defaultFS";
    public static final String DEFAULT_FS = "file:///";
    public static final Log LOG = LogFactory.getLog(FileSystem.class);
    public static final int SHUTDOWN_HOOK_PRIORITY = 10;
    public static final String TRASH_PREFIX = ".Trash";
    static final FileSystem.Cache CACHE = new FileSystem.Cache();
    private FileSystem.Cache.Key key;
    private static final Map<Class<? extends FileSystem>, FileSystem.Statistics> statisticsTable = new IdentityHashMap();
    protected FileSystem.Statistics statistics;
    private Set<Path> deleteOnExit = new TreeSet();
    boolean resolveSymlinks;
    private static final PathFilter DEFAULT_FILTER = new PathFilter() {
        public boolean accept(Path file) {
            return true;
        }
    };
    private static volatile boolean FILE_SYSTEMS_LOADED = false;
    private static final Map<String, Class<? extends FileSystem>> SERVICE_FILE_SYSTEMS = new HashMap();
    private static boolean symlinksEnabled = false;
    private static Configuration conf = null;

    static void addFileSystemForTesting(URI uri, Configuration conf, FileSystem fs) throws IOException {
        CACHE.map.put(new FileSystem.Cache.Key(uri, conf), fs);
    }

    public static FileSystem get(final URI uri, final Configuration conf, String user) throws IOException, InterruptedException {
        String ticketCachePath = conf.get("hadoop.security.kerberos.ticket.cache.path");
        UserGroupInformation ugi = UserGroupInformation.getBestUGI(ticketCachePath, user);
        return (FileSystem)ugi.doAs(new PrivilegedExceptionAction<FileSystem>() {
            public FileSystem run() throws IOException {
                return FileSystem.get(uri, conf);
            }
        });
    }

    public static FileSystem get(Configuration conf) throws IOException {
        return get(getDefaultUri(conf), conf);
    }

    public static URI getDefaultUri(Configuration conf) {
        return URI.create(fixName(conf.get("fs.defaultFS", "file:///")));
    }

    public static void setDefaultUri(Configuration conf, URI uri) {
        conf.set("fs.defaultFS", uri.toString());
    }

    public static void setDefaultUri(Configuration conf, String uri) {
        setDefaultUri(conf, URI.create(fixName(uri)));
    }

    public void initialize(URI name, Configuration conf) throws IOException {
        String scheme;
        if (name.getScheme() != null && !name.getScheme().isEmpty()) {
            scheme = name.getScheme();
        } else {
            scheme = getDefaultUri(conf).getScheme();
        }

        this.statistics = getStatistics(scheme, this.getClass());
        this.resolveSymlinks = conf.getBoolean("fs.client.resolve.remote.symlinks", true);
    }

    public String getScheme() {
        throw new UnsupportedOperationException("Not implemented by the " + this.getClass().getSimpleName() + " FileSystem implementation");
    }

    public abstract URI getUri();

    protected URI getCanonicalUri() {
        return this.canonicalizeUri(this.getUri());
    }

    protected URI canonicalizeUri(URI uri) {
        if (uri.getPort() == -1 && this.getDefaultPort() > 0) {
            try {
                uri = new URI(uri.getScheme(), uri.getUserInfo(), uri.getHost(), this.getDefaultPort(), uri.getPath(), uri.getQuery(), uri.getFragment());
            } catch (URISyntaxException var3) {
                throw new AssertionError("Valid URI became unparseable: " + uri);
            }
        }

        return uri;
    }

    protected int getDefaultPort() {
        return 0;
    }

    protected static FileSystem getFSofPath(Path absOrFqPath, Configuration conf) throws UnsupportedFileSystemException, IOException {
        absOrFqPath.checkNotSchemeWithRelative();
        absOrFqPath.checkNotRelative();
        return get(absOrFqPath.toUri(), conf);
    }

    @Public
    @Evolving
    public String getCanonicalServiceName() {
        return this.getChildFileSystems() == null ? SecurityUtil.buildDTServiceName(this.getUri(), this.getDefaultPort()) : null;
    }

    /** @deprecated */
    @Deprecated
    public String getName() {
        return this.getUri().toString();
    }

    /** @deprecated */
    @Deprecated
    public static FileSystem getNamed(String name, Configuration conf) throws IOException {
        return get(URI.create(fixName(name)), conf);
    }

    private static String fixName(String name) {
        if (name.equals("local")) {
            LOG.warn("\"local\" is a deprecated filesystem name. Use \"file:///\" instead.");
            name = "file:///";
        } else if (name.indexOf(47) == -1) {
            LOG.warn("\"" + name + "\" is a deprecated filesystem name." + " Use \"hdfs://" + name + "/\" instead.");
            name = "hdfs://" + name;
        }

        return name;
    }

    public static LocalFileSystem getLocal(Configuration conf) throws IOException {
        return (LocalFileSystem)get(LocalFileSystem.NAME, conf);
    }

    public static FileSystem get(URI uri, Configuration conf) throws IOException {
        String scheme = uri.getScheme();
        String authority = uri.getAuthority();
        if (scheme == null && authority == null) {
            return get(conf);
        } else {
            if (scheme != null && authority == null) {
                URI defaultUri = getDefaultUri(conf);
                if (scheme.equals(defaultUri.getScheme()) && defaultUri.getAuthority() != null) {
                    return get(defaultUri, conf);
                }
            }

            String disableCacheName = String.format("fs.%s.impl.disable.cache", scheme);
            return conf.getBoolean(disableCacheName, false) ? createFileSystem(uri, conf) : CACHE.get(uri, conf);
        }
    }

    public static FileSystem newInstance(final URI uri, final Configuration conf, String user) throws IOException, InterruptedException {
        String ticketCachePath = conf.get("hadoop.security.kerberos.ticket.cache.path");
        UserGroupInformation ugi = UserGroupInformation.getBestUGI(ticketCachePath, user);
        return (FileSystem)ugi.doAs(new PrivilegedExceptionAction<FileSystem>() {
            public FileSystem run() throws IOException {
                return FileSystem.newInstance(uri, conf);
            }
        });
    }

    public static FileSystem newInstance(URI uri, Configuration conf) throws IOException {
        String scheme = uri.getScheme();
        String authority = uri.getAuthority();
        if (scheme == null) {
            return newInstance(conf);
        } else {
            if (authority == null) {
                URI defaultUri = getDefaultUri(conf);
                if (scheme.equals(defaultUri.getScheme()) && defaultUri.getAuthority() != null) {
                    return newInstance(defaultUri, conf);
                }
            }

            return CACHE.getUnique(uri, conf);
        }
    }

    public static FileSystem newInstance(Configuration conf) throws IOException {
        return newInstance(getDefaultUri(conf), conf);
    }

    public static LocalFileSystem newInstanceLocal(Configuration conf) throws IOException {
        return (LocalFileSystem)newInstance(LocalFileSystem.NAME, conf);
    }

    public static void closeAll() throws IOException {
        CACHE.closeAll();
    }

    public static void closeAllForUGI(UserGroupInformation ugi) throws IOException {
        CACHE.closeAll(ugi);
    }

    public Path makeQualified(Path path) {
        this.checkPath(path);
        return path.makeQualified(this.getUri(), this.getWorkingDirectory());
    }

    @Private
    public Token<?> getDelegationToken(String renewer) throws IOException {
        return null;
    }

    @Public
    @Evolving
    public Token<?>[] addDelegationTokens(String renewer, Credentials credentials) throws IOException {
        if (credentials == null) {
            credentials = new Credentials();
        }

        List<Token<?>> tokens = new ArrayList();
        this.collectDelegationTokens(renewer, credentials, tokens);
        return (Token[])tokens.toArray(new Token[tokens.size()])