HDFS Client API

In this post, let us explore Hadoop file system basic APIs

• Create a directory in hdfs
• Copy a file from local files system to hdfs
• Read the hdfs file
• Delete the hdfs file

import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

public class HDFSClient {

    public static Configuration conf = new Configuration();
   
    static{
        conf.addResource(new Path("/home/impadmin/sws/hadoop-0.20.2/conf/core-site.xml"));
    }
   
    public static void createHdfsfile(String fromLocalFile,String hdfsFile) throws IOException {
        FileSystem hdfs = FileSystem.get(conf);
        Path localDir = new Path(fromLocalFile);
        Path hdfsDir = new Path(hdfsFile);
        hdfs.copyFromLocalFile(localDir, hdfsDir);
    }

    public static void readFile(String file) {
        FileSystem fileSystem;
        try {
           fileSystem = FileSystem.get(conf);
            Path path = new Path(file);
        if (!fileSystem.exists(path)) {
            System.out.println("File " + file + " does not exists");
            return;
        }
        FSDataInputStream in = fileSystem.open(path);
        String filename = file.substring(file.lastIndexOf('/') + 1,
                file.length());
        OutputStream out = new BufferedOutputStream(new FileOutputStream(
                new File(filename)));
        byte[] b = new byte[1024];
        int numBytes = 0;
        while ((numBytes = in.read(b)) > 0) {
            out.write(b, 0, numBytes);
        }
        String s = new String(b);
        System.out.println(s);
        in.close();
        out.close();
        fileSystem.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    public static void deleteFile(String file) throws IOException {
        FileSystem fileSystem = FileSystem.get(conf);
        Path path = new Path(file);
        if (!fileSystem.exists(path)) {
            System.out.println("File " + file + " does not exists");
            return;
        }
        boolean success = fileSystem.delete(new Path(file), true);
        fileSystem.close();
    }

    public static void mkdir(String dir) throws IOException {
        FileSystem fileSystem = FileSystem.get(conf);
        Path path = new Path(dir);
        if (fileSystem.exists(path)) {
            System.out.println("Dir " + dir + " already exists...");
            return;
        }
        boolean success = fileSystem.mkdirs(path);
        fileSystem.close();
    }

    public static void main(String[] args) throws IOException {
        //Create a Directory in HDFS
        HDFSClient.mkdir("hdfs://localhost:9000/user/impadmin/testDir");
       
        String srcFile = "file:///home/impadmin/testdata/Order";
        String destFile = "hdfs://localhost:9000/user/impadmin/testDir";
      
       //Copy File from local file system to hdfs
        HDFSClient.createHdfsfile(srcFile,destFile);

        //Read the file
        HDFSClient.readFile("hdfs://localhost:9000/user/impadmin/testDir/Order");
       
        //Delete The File
        HDFSClient.deleteFile("hdfs://localhost:9000/user/impadmin/testDir/Order");
        }
}

For any query please drop a comment......