大数据学习10

spring整合hadoop

maven：

<!--hadoop依赖-->
<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-client -->
<dependency>
  <groupId>org.apache.hadoop</groupId>
  <artifactId>hadoop-client</artifactId>
  <version>2.6.0</version>
</dependency>
<!--spring整合hadoop-->
<dependency>
  <groupId>org.springframework.data</groupId>
  <artifactId>spring-data-hadoop</artifactId>
  <version>2.5.0.RELEASE</version>
</dependency>

xml：

<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
       xmlns:hdp="http://www.springframework.org/schema/hadoop"
       xmlns:context="http://www.springframework.org/schema/context"
       xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
        http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd
        http://www.springframework.org/schema/hadoop http://www.springframework.org/schema/hadoop/spring-hadoop.xsd">
    <hdp:configuration id="hadoopConfiguration">
        fs.defaultFS=${fs.defaultFS.uri}
    </hdp:configuration>
    <hdp:file-system id="fileSystem" configuration-ref="hadoopConfiguration" user="lc"/>
    <context:property-placeholder location="application.properties"/>
</beans>

代码：

package com.lc.hadoop.hdfsAPP;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;

public class springForHadoop {
    Configuration configuration = null;
    FileSystem fileSystem = null;
    ApplicationContext applicationContext = null;
    @Test
    public void mkdirs() throws Exception{
        fileSystem.mkdirs(new Path("/lcnewpath"));
    }
    @Test
    public void create() throws Exception{
        if(fileSystem.exists(new Path("/lcnewpath"))){
            fileSystem.delete(new Path("/lcnewpath"),true);
        }
        FSDataOutputStream fsDataOutputStream = fileSystem.create(new Path("/lcnewpath"));
        fsDataOutputStream.write("你好啊".getBytes());
    }
    @Test
    public void open() throws Exception{
        FSDataInputStream fsDataInputStream = fileSystem.open(new Path("/lcnewpath"));
        IOUtils.copyBytes(fsDataInputStream,System.out,1024);
    }
    @Before
    public void setUp(){
        applicationContext = new ClassPathXmlApplicationContext("spring-beans.xml");
        //获取fileSystem
        fileSystem = (FileSystem) applicationContext.getBean("fileSystem");
    }
    @After
    public void tearDown(){
        fileSystem = null;
    }
}