This article mainly studies the access method of HDFS's Java API. The specific code is shown below and has detailed comments.
The pace has been a bit fast recently. When you have time, you can wrap this up
import java.io.IOException;import java.net.URI;import java.net.URISyntaxException;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.BlockLocation;import org.apache.hadoop.fs.FileStatus;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.FileUtil;import org.apache.hadoop.fs.Path;import org.apache.hadoop.hdfs.DistributedFileSystem;import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
/** * Get HDFS file system* @return * @throws IOException * @throws URISyntaxException */public static FileSystem getFileSystem() throws IOException, URISyntaxException{//read config fileConfiguration conf = new Configuration();//Return to the default file system//If run under the Hadoop cluster, use this method to directly obtain the default file system//FileSystem fs = FileSystem.get(conf);//Specified file system address URI uri = new URI("hdfs://hy:9000");//Return the specified file system//If you are testing locally, you need to use this method to get the file system FileSystem fs = FileSystem.get(uri, conf);return fs;}/** * Create file directory* @throws Exception */public static void mkdir() throws Exception{//Get file system FileSystem fs = getFileSystem();//Create file directory fs.mkdirs(new Path("hdfs://hy:9000/hy/weibo"));//Release resource fs.close();}/** * Delete file or file directory* @throws Exception */public static void rmdir() throws Exception{//get file system FileSystem fs = getFileSystem();//Delete file or file directory fs.delete(new Path("hdfs://hy:9000/hy/weibo"), true);//Release resource fs.close();}/** * Get all files in the directory* @throws Exception */public static void listAllFile() throws Exception{//get file system FileSystem fs = getFileSystem();//List the directory contents FileStatus[] status = fs.listStatus(new Path("hdfs://hy:9000/hy/"));//Get all file paths in the directory Path[] listedPaths = FileUtil.stat2Paths(status);//Loop each file for (Path path : listedPaths) {System.out.println(path);}//Release the resource fs.close();}/** * Upload the file to HDFS * @throws Exception */public static void copyToHDFS() throws Exception{//Get file object FileSystem fs = getFileSystem();//The source file path is the path under Linux Path srcPath = new Path("/home/hadoop/temp.jar");//If you need to test in Windows, you need to change it to the path under Windows, for example, E://temp.jarPath srcPath = new Path("E://temp.jar");//Destination path Path dstPath = new Path("hdfs://hy:9000/hy/weibo");//Implement file upload fs.copyFromLocalFile(srcPath, dstPath);//Release the resource fs.close();}/** * Download the file from HDFS* @throws Exception */public static void getFile() throws Exception{//Get file system FileSystem fs = getFileSystem();//Source file path Path srcPath = new Path("hdfs://hy:9000/hy/weibo/temp.jar");//Default path, default is Linux //If tested under Windows, you need to change to the path under Windows, such as C://User/andy/Desktop/Path dstPath = new Path("D://");//Download the file fs.copyToLocalFile(srcPath, dstPath);//Release the resource fs.close();}/** * Get information about HDFS cluster points* @throws Exception */public static void getHDFSNodes() throws Exception{//Get file system FileSystem fs = getFileSystem();//Get distributed file system DistributedFileSystem hdfs = (DistributedFileSystem)fs;//Get all nodes DatanodeInfo[] dataNodeStats = hdfs.getDataNodeStats();//Loop traversal for (int i = 0; i < dataNodeStats.length; i++) {System.out.println("DataNote_" + i + "_Name:" + dataNodeStats[i].getHostName());}//Release the resource fs.close();}/** * Find the location of a file in the HDFS cluster* @throws Exception */public static void getFileLocal() throws Exception{//Get file system FileSystem fs = getFileSystem();//File path Path path = new Path("hdfs://hy:9000/hy/weibo/temp.jar");//Get file directory FileStatus fileStatus = fs.getFileStatus(path);//Get the file block location list BlockLocation[] blockLocations = fs.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());//Loop output block information for (int i = 0; i < blockLocations.length; i++) {String[] hosts = blockLocations[i].getHosts();System.out.println("block_" + i + "_location:" + hosts[0]);}//Release the resource fs.close();}The above is the entire content of this article about the access method of HDFS Java API. I hope it will be helpful to everyone. Interested friends can continue to refer to other related topics on this site. If there are any shortcomings, please leave a message to point it out. Thank you friends for your support for this site!