一、pom.xml配置
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.lihejia</groupId>
<artifactId>HDFSClint</artifactId>
<version>1.0-SNAPSHOT</version>
<properties>
<maven.compiler.source>8</maven.compiler.source>
<maven.compiler.target>8</maven.compiler.target>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>2.8.1</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>2.8.1</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.30</version>
</dependency>
</dependencies>
</project>
二、HdfsClient代码
package com.lihejia.hdfs;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.conf.Configuration;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
public class HdfsClient {
private FileSystem fs;
@Before
public void init() throws IOException, URISyntaxException, InterruptedException {
//连接集群的地址
URI uri = new URI("hdfs://localhost:9000");
//创建一个配置文件夹
Configuration configuration = new Configuration();
//用户
String user = "lihejia";
//获取客户端的对象
fs = FileSystem.get(uri, configuration, user);
}
@After
public void close() throws IOException {
//关闭资源
fs.close();
}
@Test
public void testMkdir() throws IOException {
//创建文件夹
fs.mkdirs(new Path("/02jia"));
}
@Test
public void testRemove() throws IOException {
//删除文件夹
fs.delete(new Path("/02jia"),true);
}
@Test
public void testCopyFromLocalFile() throws IOException {
//拷贝本地文件到hdfs
fs.copyFromLocalFile(new Path("/Users/lihejia/Desktop/Mario0630.txt"),new Path("/"));
}
}
三、注意点
1、代码中的每个引入的方法记得是hadoop相关的,不要引入错了。
2、URI的地址记得要跟core-site.xml配置文件中保持一致。
譬如配置文件中的hdfs://localhost:9000对应代码中的hdfs的访问地址。
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<property>
<name>hadoop.tmp.dir</name>
<value>file:/usr/local/hadoop/tmp</value>
</property>
<property>
<name>fs.defaultFS</name>
<value>hdfs://localhost:9000</value>
</property>
</configuration>
四、HDFS的API接口文件
可以参考文件中的方法根据自己需求使用,代码只是设计简单的创建文件夹、删除文件夹、拷贝文件基本操作,其他操作可以具体参考该配置文件API.
//
// Source code recreated from a .class file by IntelliJ IDEA
// (powered by FernFlower decompiler)
//
package org.apache.hadoop.fs;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import java.io.Closeable;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.lang.ref.ReferenceQueue;
import java.lang.ref.WeakReference;
import java.net.URI;
import java.net.URISyntaxException;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Collection;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.IdentityHashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.ServiceConfigurationError;
import java.util.ServiceLoader;
import java.util.Set;
import java.util.Stack;
import java.util.TreeSet;
import java.util.Map.Entry;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.ContentSummary.Builder;
import org.apache.hadoop.fs.GlobalStorageStatistics.StorageStatisticsProvider;
import org.apache.hadoop.fs.Options.ChecksumOpt;
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.MultipleIOException;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.ClassUtil;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.DataChecksum.Type;
import org.apache.htrace.core.TraceScope;
import org.apache.htrace.core.Tracer;
@Public
@Stable
public abstract class FileSystem extends Configured implements Closeable {
public static final String FS_DEFAULT_NAME_KEY = "fs.defaultFS";
public static final String DEFAULT_FS = "file:///";
public static final Log LOG = LogFactory.getLog(FileSystem.class);
public static final int SHUTDOWN_HOOK_PRIORITY = 10;
public static final String TRASH_PREFIX = ".Trash";
static final FileSystem.Cache CACHE = new FileSystem.Cache();
private FileSystem.Cache.Key key;
private static final Map<Class<? extends FileSystem>, FileSystem.Statistics> statisticsTable = new IdentityHashMap();
protected FileSystem.Statistics statistics;
private Set<Path> deleteOnExit = new TreeSet();
boolean resolveSymlinks;
private static final PathFilter DEFAULT_FILTER = new PathFilter() {
public boolean accept(Path file) {
return true;
}
};
private static volatile boolean FILE_SYSTEMS_LOADED = false;
private static final Map<String, Class<? extends FileSystem>> SERVICE_FILE_SYSTEMS = new HashMap();
private static boolean symlinksEnabled = false;
private static Configuration conf = null;
static void addFileSystemForTesting(URI uri, Configuration conf, FileSystem fs) throws IOException {
CACHE.map.put(new FileSystem.Cache.Key(uri, conf), fs);
}
public static FileSystem get(final URI uri, final Configuration conf, String user) throws IOException, InterruptedException {
String ticketCachePath = conf.get("hadoop.security.kerberos.ticket.cache.path");
UserGroupInformation ugi = UserGroupInformation.getBestUGI(ticketCachePath, user);
return (FileSystem)ugi.doAs(new PrivilegedExceptionAction<FileSystem>() {
public FileSystem run() throws IOException {
return FileSystem.get(uri, conf);
}
});
}
public static FileSystem get(Configuration conf) throws IOException {
return get(getDefaultUri(conf), conf);
}
public static URI getDefaultUri(Configuration conf) {
return URI.create(fixName(conf.get("fs.defaultFS", "file:///")));
}
public static void setDefaultUri(Configuration conf, URI uri) {
conf.set("fs.defaultFS", uri.toString());
}
public static void setDefaultUri(Configuration conf, String uri) {
setDefaultUri(conf, URI.create(fixName(uri)));
}
public void initialize(URI name, Configuration conf) throws IOException {
String scheme;
if (name.getScheme() != null && !name.getScheme().isEmpty()) {
scheme = name.getScheme();
} else {
scheme = getDefaultUri(conf).getScheme();
}
this.statistics = getStatistics(scheme, this.getClass());
this.resolveSymlinks = conf.getBoolean("fs.client.resolve.remote.symlinks", true);
}
public String getScheme() {
throw new UnsupportedOperationException("Not implemented by the " + this.getClass().getSimpleName() + " FileSystem implementation");
}
public abstract URI getUri();
protected URI getCanonicalUri() {
return this.canonicalizeUri(this.getUri());
}
protected URI canonicalizeUri(URI uri) {
if (uri.getPort() == -1 && this.getDefaultPort() > 0) {
try {
uri = new URI(uri.getScheme(), uri.getUserInfo(), uri.getHost(), this.getDefaultPort(), uri.getPath(), uri.getQuery(), uri.getFragment());
} catch (URISyntaxException var3) {
throw new AssertionError("Valid URI became unparseable: " + uri);
}
}
return uri;
}
protected int getDefaultPort() {
return 0;
}
protected static FileSystem getFSofPath(Path absOrFqPath, Configuration conf) throws UnsupportedFileSystemException, IOException {
absOrFqPath.checkNotSchemeWithRelative();
absOrFqPath.checkNotRelative();
return get(absOrFqPath.toUri(), conf);
}
@Public
@Evolving
public String getCanonicalServiceName() {
return this.getChildFileSystems() == null ? SecurityUtil.buildDTServiceName(this.getUri(), this.getDefaultPort()) : null;
}
/** @deprecated */
@Deprecated
public String getName() {
return this.getUri().toString();
}
/** @deprecated */
@Deprecated
public static FileSystem getNamed(String name, Configuration conf) throws IOException {
return get(URI.create(fixName(name)), conf);
}
private static String fixName(String name) {
if (name.equals("local")) {
LOG.warn("\"local\" is a deprecated filesystem name. Use \"file:///\" instead.");
name = "file:///";
} else if (name.indexOf(47) == -1) {
LOG.warn("\"" + name + "\" is a deprecated filesystem name." + " Use \"hdfs://" + name + "/\" instead.");
name = "hdfs://" + name;
}
return name;
}
public static LocalFileSystem getLocal(Configuration conf) throws IOException {
return (LocalFileSystem)get(LocalFileSystem.NAME, conf);
}
public static FileSystem get(URI uri, Configuration conf) throws IOException {
String scheme = uri.getScheme();
String authority = uri.getAuthority();
if (scheme == null && authority == null) {
return get(conf);
} else {
if (scheme != null && authority == null) {
URI defaultUri = getDefaultUri(conf);
if (scheme.equals(defaultUri.getScheme()) && defaultUri.getAuthority() != null) {
return get(defaultUri, conf);
}
}
String disableCacheName = String.format("fs.%s.impl.disable.cache", scheme);
return conf.getBoolean(disableCacheName, false) ? createFileSystem(uri, conf) : CACHE.get(uri, conf);
}
}
public static FileSystem newInstance(final URI uri, final Configuration conf, String user) throws IOException, InterruptedException {
String ticketCachePath = conf.get("hadoop.security.kerberos.ticket.cache.path");
UserGroupInformation ugi = UserGroupInformation.getBestUGI(ticketCachePath, user);
return (FileSystem)ugi.doAs(new PrivilegedExceptionAction<FileSystem>() {
public FileSystem run() throws IOException {
return FileSystem.newInstance(uri, conf);
}
});
}
public static FileSystem newInstance(URI uri, Configuration conf) throws IOException {
String scheme = uri.getScheme();
String authority = uri.getAuthority();
if (scheme == null) {
return newInstance(conf);
} else {
if (authority == null) {
URI defaultUri = getDefaultUri(conf);
if (scheme.equals(defaultUri.getScheme()) && defaultUri.getAuthority() != null) {
return newInstance(defaultUri, conf);
}
}
return CACHE.getUnique(uri, conf);
}
}
public static FileSystem newInstance(Configuration conf) throws IOException {
return newInstance(getDefaultUri(conf), conf);
}
public static LocalFileSystem newInstanceLocal(Configuration conf) throws IOException {
return (LocalFileSystem)newInstance(LocalFileSystem.NAME, conf);
}
public static void closeAll() throws IOException {
CACHE.closeAll();
}
public static void closeAllForUGI(UserGroupInformation ugi) throws IOException {
CACHE.closeAll(ugi);
}
public Path makeQualified(Path path) {
this.checkPath(path);
return path.makeQualified(this.getUri(), this.getWorkingDirectory());
}
@Private
public Token<?> getDelegationToken(String renewer) throws IOException {
return null;
}
@Public
@Evolving
public Token<?>[] addDelegationTokens(String renewer, Credentials credentials) throws IOException {
if (credentials == null) {
credentials = new Credentials();
}
List<Token<?>> tokens = new ArrayList();
this.collectDelegationTokens(renewer, credentials, tokens);
return (Token[])tokens.toArray(new Token[tokens.size()])