NET实现Hadoop增删改查

最新推荐文章于 2023-07-17 17:34:16 发布

.NET跨平台

最新推荐文章于 2023-07-17 17:34:16 发布

阅读量1.9k

点赞数

分类专栏：大数据之Hadoop 大数据+云计算文章标签： hadoop

本文链接：https://blog.youkuaiyun.com/WuLex/article/details/78122095

版权

大数据+云计算同时被 2 个专栏收录

17 篇文章

订阅专栏

大数据之Hadoop

15 篇文章

订阅专栏

本文介绍了一个使用 C# 编写的 HDFS (Hadoop 分布式文件系统) 操作类库示例，包括创建、删除目录及文件，获取目录列表等功能，并提供了完整的测试代码。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

packages.config

<?xml version="1.0" encoding="utf-8"?>
<packages>
  <package id="Microsoft.AspNet.WebApi.Client" version="4.0.20505.0" targetFramework="net46" />
  <package id="Microsoft.Data.Edm" version="5.2.0" targetFramework="net46" />
  <package id="Microsoft.Data.OData" version="5.2.0" targetFramework="net46" />
  <package id="Microsoft.Hadoop.WebClient" version="0.12.5126.42915" targetFramework="net46" />
  <package id="Microsoft.Net.Http" version="2.0.20505.0" targetFramework="net46" />
  <package id="Microsoft.WindowsAzure.ConfigurationManager" version="1.8.0.0" targetFramework="net46" />
  <package id="Newtonsoft.Json" version="4.5.11" targetFramework="net46" />
  <package id="System.Spatial" version="5.2.0" targetFramework="net46" />
  <package id="WindowsAzure.Storage" version="2.0.4.1" targetFramework="net46" />
</packages>

HDFSAccess.cs

using Microsoft.Hadoop.WebHDFS;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;

namespace Physical
{
    public sealed class HDFSAccess
    {
        private readonly WebHDFSClient webHDFSClient;

        public HDFSAccess(string uriString, string userName)
        {
            this.webHDFSClient = new WebHDFSClient(new Uri(uriString), userName);
        }

        public List<string> GetDirectories(string path)
        {
            var directoryStatus = this.webHDFSClient.GetDirectoryStatus(path).Result;

            return directoryStatus.Directories.Select(d => d.PathSuffix).ToList();
        }

        public List<string> GetFiles(string path)
        {
            var directoryStatus = this.webHDFSClient.GetDirectoryStatus(path).Result;

            return directoryStatus.Files.Select(d => d.PathSuffix).ToList();
        }

        public bool CreateDirectory(string path)
        {
            // 传入路径不包含根目录时，预设会在根目录「/」底下
            return this.webHDFSClient.CreateDirectory(path).Result;
        }

        public bool DeleteDirectory(string path)
        {
            // 传入路径不包含根目录时，预设会在根目录「/」底下
            return this.webHDFSClient.DeleteDirectory(path).Result;
        }

        public string CreateFile(string localFile, string remotePath)
        {
            // 传入远端路径不包含根目录时，预设会在根目录「/」底下
            return this.webHDFSClient.CreateFile(localFile, remotePath).Result;
        }

        public bool DeleteFile(string path)
        {
            // 传入路径不包含根目录时，预设会在根目录「/」底下
            return this.webHDFSClient.DeleteDirectory(path).Result;
        }

        public HttpResponseMessage OpenFile(string path)
        {
            // 传入路径不包含根目录时，预设会在根目录「/」底下
            return this.webHDFSClient.OpenFile(path).Result;
        }
    }
}

Program.cs

using Physical;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace MyTest
{
    class Program
    {
        // HDFS cluster 客户端进入端点设定于 主机上
        // 预设端点：http://[主机名称]:50070
        // 预设帐号：Wu
        private static HDFSAccess access = new HDFSAccess(@"http://127.0.0.1:50070", "Wu");

        static void Main(string[] args)
        {
            GetDirectoriesTest();
            Console.WriteLine("----------------------------------------------");
            GetFilesTest();
            Console.WriteLine("----------------------------------------------");
            DirectoryTest();
            Console.WriteLine("----------------------------------------------");
            FileTest();
            Console.WriteLine("----------------------------------------------");
            OpenFileTest();
            Console.ReadKey();
        } 

        public void TestCleanup()
        {
            //取得根目录资料夹
            var directories = access.GetDirectories(@"/");

            // 移除预设目录：tmp、user 外的目录
            foreach (var directory in directories)
            {
                if ("tmp".Equals(directory) || "user".Equals(directory))
                {
                    continue;
                }
                else
                {
                    access.DeleteDirectory(directory);
                }
            }

            // 取得根目录档案
            var files = access.GetFiles(@"/");

            // 移除所有档案
            foreach (var file in files)
            {
                access.DeleteFile(file);
            }

            // 移除 OpenFile 转存档案
            File.Delete(Path.Combine(Directory.GetCurrentDirectory(), "Test.jpg"));
        }

        //_传入根目录_预期回传预设目录
        public static void GetDirectoriesTest()
        {
            // 预设根目录下有两个目录：tmp、user
            var expected = new List<string>() { "tmp", "user", };

            var actual = access.GetDirectories(@"/");

            foreach (var item in actual)
            {
                Console.WriteLine(item);
            }

        }

        //_传入根目录_预期回传空集合
        public static void GetFilesTest()
        {
            // 预设根目录下没有档案
            var expected = new List<string>();

            var actual = access.GetFiles(@"/");
            foreach (var item in actual)
            {
                Console.WriteLine(item);
            }

        }

        //_建立zzz目录_预期成功_预期根目录下有zzz目录_删除zzz目录_预期成功_预期根目录下无zzz目录
        public static void DirectoryTest()
        {
            var directoryName = "zzz";

            // 建立zzz目录
            var boolCreateDirectory = access.CreateDirectory(directoryName);

            Console.WriteLine("建立zzz目录_预期成功:"+boolCreateDirectory);

            // 建立zzz目录_预期成功_预期根目录下有zzz目录
            // 预设根目录下有三个目录：tmp、user、zzz
            var expectedCreateDirectory = new List<string>() { "tmp", "user", directoryName, };

            var actualCreateDirectory = access.GetDirectories(@"/");


            foreach (var item in actualCreateDirectory)
            {
                Console.WriteLine(item);
            }

            Console.WriteLine("********************************************");
            // 删除zzz目录
            var boolDeleteDirectory = access.DeleteDirectory(directoryName);

            Console.WriteLine("删除zzz目录_预期成功:" + boolDeleteDirectory);


            // 删除zzz目录_预期成功_预期根目录下无zzz目录
            // 预设根目录下有两个目录：tmp、user
            var expectedDeleteDirectory = new List<string>() { "tmp", "user", };

            var actualDeleteDirectory = access.GetDirectories(@"/");

            foreach (var item in actualDeleteDirectory)
            {
                Console.WriteLine(item);
            }
        }

        //_建立Test档案_预期根目录下有Test档案_删除Test档案_预期成功_预期根目录下无Test档案
        public static void FileTest()
        {
            var localFile = Path.Combine(Directory.GetCurrentDirectory(), "TestFolder", "Test.jpg");

            var remotePath = "Test.jpg";

            // 建立Test档案
            var boolCreateFile = access.CreateFile(localFile, remotePath);

            // 建立Test档案_预期根目录下有Test档案
            var expectedCreateFile = new List<string>() { remotePath, };

            var actualCreateFile = access.GetFiles(@"/");

            foreach (var item in actualCreateFile)
            {
                Console.WriteLine(item);
            }
            Console.WriteLine("********************************************");
            // 删除Test档案
            var boolDeleteFile = access.DeleteDirectory(remotePath);

            Console.WriteLine("删除Test档案_预期成功:"+boolDeleteFile);

            // 删除Test档案_预期成功_预期根目录下无Test档案
            var expectedDeleteFile = new List<string>();

            var actualDeleteFile = access.GetFiles(@"/");

            foreach (var item in actualDeleteFile)
            {
                Console.WriteLine(item);
            }
        }

        //_建立Test档案_预期根目录下有Test档案_取得Test档案_预期成功_预期回传Test档案Stream并转存成功
        public static void OpenFileTest()
        {
            var localFile = Path.Combine(Directory.GetCurrentDirectory(), "TestFolder", "Test.jpg");
            var remotePath = "Test.jpg";
            var saveFile = Path.Combine(Directory.GetCurrentDirectory(), "Test.jpg");

            Console.WriteLine("saveFile:" + saveFile);
            Console.WriteLine("********************************************");
            // 建立Test档案
            var boolCreateFile = access.CreateFile(localFile, remotePath);

            // 建立Test档案_预期根目录下有Test档案
            var expectedCreateFile = new List<string>() { remotePath, };

            var actualCreateFile = access.GetFiles(@"/");

            foreach (var item in actualCreateFile)
            {
                Console.WriteLine(item);
            }


            Console.WriteLine("********************************************");
            // 取得Test档案
            var response = access.OpenFile(remotePath);

            // 取得Test档案_预期成功
            response.EnsureSuccessStatusCode();

            // 取得Test档案_预期成功_预期回传Test档案Stream并转存成功
            using (var fs = File.Create(saveFile))
            {
                response.Content.CopyToAsync(fs).Wait();
            }

            Console.WriteLine(File.Exists(saveFile));

        }
    }
}

运行结果如图:

这里写图片描述