当前位置： 移动技术网 > IT编程>数据库>其他数据库 > hadoop3自学入门笔记(3)-java 操作hdfs

hadoop3自学入门笔记(3)-java 操作hdfs

2020年03月09日  | 移动技术网IT编程  | 我要评论

1.core-site.xml

<configuration>
  <property>
        <name>fs.defaultfs</name>
        <value>hdfs://192.168.3.61:9820</value>
    </property>
<property>
        <name>hadoop.tmp.dir</name>
        <value>/opt/hadoopdata</value>
    </property>
</configuration>

2.pom.xml

<?xml version="1.0" encoding="utf-8"?>

<project xmlns="http://maven.apache.org/pom/4.0.0" xmlns:xsi="http://www.w3.org/2001/xmlschema-instance"
  xsi:schemalocation="http://maven.apache.org/pom/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelversion>4.0.0</modelversion>

  <groupid>com.qmkj</groupid>
  <artifactid>hdfsclienttest</artifactid>
  <version>0.1</version>

  <name>hdfsclienttest</name>
  <!-- fixme change it to the project's website -->
  <url>http://www.example.com</url>

  <properties>
    <project.build.sourceencoding>utf-8</project.build.sourceencoding>
    <maven.compiler.source>1.7</maven.compiler.source>
    <maven.compiler.target>1.7</maven.compiler.target>
  </properties>

  <dependencies>
    <dependency>
      <groupid>junit</groupid>
      <artifactid>junit</artifactid>
      <version>4.11</version>
      <scope>test</scope>
    </dependency>
    <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-hdfs-client -->
    <dependency>
      <groupid>org.apache.hadoop</groupid>
      <artifactid>hadoop-hdfs-client</artifactid>
      <version>3.2.1</version>
      <scope>provided</scope>
    </dependency>
    <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common -->
    <dependency>
      <groupid>org.apache.hadoop</groupid>
      <artifactid>hadoop-common</artifactid>
      <version>3.2.1</version>
    </dependency>
    <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-hdfs -->
    <dependency>
      <groupid>org.apache.hadoop</groupid>
      <artifactid>hadoop-hdfs</artifactid>
      <version>3.2.1</version>
    </dependency>

  </dependencies>

  <build>
    <pluginmanagement><!-- lock down plugins versions to avoid using maven defaults (may be moved to parent pom) -->
      <plugins>
        <!-- clean lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#clean_lifecycle -->
        <plugin>
          <artifactid>maven-clean-plugin</artifactid>
          <version>3.1.0</version>
        </plugin>
        <!-- default lifecycle, jar packaging: see https://maven.apache.org/ref/current/maven-core/default-bindings.html#plugin_bindings_for_jar_packaging -->
        <plugin>
          <artifactid>maven-resources-plugin</artifactid>
          <version>3.0.2</version>
        </plugin>
        <plugin>
          <artifactid>maven-compiler-plugin</artifactid>
          <version>3.8.0</version>
        </plugin>
        <plugin>
          <artifactid>maven-surefire-plugin</artifactid>
          <version>2.22.1</version>
        </plugin>
        <plugin>
          <artifactid>maven-jar-plugin</artifactid>
          <version>3.0.2</version>
        </plugin>
        <plugin>
          <artifactid>maven-install-plugin</artifactid>
          <version>2.5.2</version>
        </plugin>
        <plugin>
          <artifactid>maven-deploy-plugin</artifactid>
          <version>2.8.2</version>
        </plugin>
        <!-- site lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#site_lifecycle -->
        <plugin>
          <artifactid>maven-site-plugin</artifactid>
          <version>3.7.1</version>
        </plugin>
        <plugin>
          <artifactid>maven-project-info-reports-plugin</artifactid>
          <version>3.0.0</version>
        </plugin>
      </plugins>
    </pluginmanagement>
  </build>
</project>

3.测试代码

package com.qmkj;


import org.apache.hadoop.conf.configuration;
import org.apache.hadoop.fs.*;
import org.junit.before;
import org.junit.test;

import java.io.filenotfoundexception;
import java.io.ioexception;
import java.net.uri;


/**
 * unit test for simple app.
 */
public class apptest {
    filesystem fs = null;

    @before
    public void init() throws exception {

        configuration conf = new configuration();
        //设立的设置url请注意，设置core-site.xml中配置fs.defaultfs的地址
        fs = filesystem.get(new uri("hdfs://192.168.3.61:9820"), conf, "root");

    }

    @test
    public void testadd() throws exception {
        fs.copyfromlocalfile(new path("d:\\kk_movies\\kk 2020-02-20 20-45-55.mp4"), new path("/zhanglei"));
        fs.close();
    }

    /**
     * 从hdfs中复制文件到本地文件系统
     *
     * @throws ioexception
     * @throws illegalargumentexception
     */
    @test
    public void testdownloadfiletolocal() throws illegalargumentexception, ioexception {

        // fs.copytolocalfile(new path("/mysql-connector-java-5.1.28.jar"), new
        // path("d:/"));
        fs.copytolocalfile(false, new path("test.txt"), new path("e:/"), true);
        fs.close();

    }

    /**
     * 目录操作
     *
     * @throws illegalargumentexception
     * @throws ioexception
     */
    @test
    public void testmkdiranddeleteandrename() throws illegalargumentexception, ioexception {

        // 创建目录
        fs.mkdirs(new path("/zhanglei/b1/c1"));

        // 删除文件夹 ，如果是非空文件夹，参数2必须给值true ，删除所有子文件夹
        fs.delete(new path("/b1"), true);

        // 重命名文件或文件夹
        fs.rename(new path("/zhanglei"), new path("/qmkj"));

    }

    /**
     * 查看目录信息，只显示文件
     *
     * @throws ioexception
     * @throws illegalargumentexception
     * @throws filenotfoundexception
     */
    @test
    public void testlistfiles() throws filenotfoundexception, illegalargumentexception, ioexception {


        remoteiterator<locatedfilestatus> listfiles = fs.listfiles(new path("/"), true);

        while (listfiles.hasnext()) {

            locatedfilestatus filestatus = listfiles.next();

            system.out.println(filestatus.getpath().getname());
            system.out.println(filestatus.getblocksize());
            system.out.println(filestatus.getpermission());
            system.out.println(filestatus.getlen());
            blocklocation[] blocklocations = filestatus.getblocklocations();
            for (blocklocation bl : blocklocations) {
                system.out.println("block-length:" + bl.getlength() + "--" + "block-offset:" + bl.getoffset());
                string[] hosts = bl.gethosts();
                for (string host : hosts) {
                    system.out.println(host);
                }

            }

            system.out.println("--------------打印的分割线--------------");

        }

    }

    /**
     * 查看文件及文件夹信息
     *
     * @throws ioexception
     * @throws illegalargumentexception
     * @throws filenotfoundexception
     */
    @test
    public void testlistall() throws filenotfoundexception, illegalargumentexception, ioexception {
        //可以右击方法名，run 测试一下。
        filestatus[] liststatus = fs.liststatus(new path("/"));

        string flag = "";
        for (filestatus fstatus : liststatus) {

            if (fstatus.isfile()) {
                flag = "f-- ";
            } else {
                flag = "d-- ";
            }
            system.out.println(flag + fstatus.getpath().getname());
            system.out.println(fstatus.getpermission());

        }

    }

}

testdownloadfiletolocal 这里测试请注意，本地也要装hdfs才可以

公众号【lovepythoncn】

您可能感兴趣的文章:

如对本文有疑问， 点击进行留言回复！！

相关文章:

验证码：
最近更新的文章
大家感兴趣的文章
移动技术网