- Import dependencies
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>2.7.3</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>2.7.3</version>
</dependency>
- Download winutils.exe and configure environment variables
GitHub - srccodes/hadoop-common-2.2.0-bin: hadoop-common-2.2.0/bin
HADOOP_HOME: The directory where the file is located\hadoop-common-2.2.0-bin-master\bin
- Add hdfs configuration information
hdfs:
path: hdfs://localhost:9000
username: hadoop01
- Write code
package com.iproinfo.refinedanalysis.service;
import org.apache.hadoop.fs.BlockLocation;
import org.springframework.web.multipart.MultipartFile;
import java.util.List;
import java.util.Map;
public interface HdfsService {
boolean mkdir(String path) throws Exception;
boolean existFile(String path) throws Exception;
List<Map<String, Object>> readPathInfo(String path) throws Exception;
void createFile(String path, MultipartFile file) throws Exception;
String readFile(String path) throws Exception;
List<Map<String, String>> listFile(String path) throws Exception;
boolean renameFile(String oldName, String newName) throws Exception;
boolean deleteFile(String path) throws Exception;
void uploadFile(String path, String uploadPath) throws Exception;
void downloadFile(String path, String downloadPath) throws Exception;
void copyFile(String sourcePath, String targetPath) throws Exception;
byte[] openFileToBytes(String path) throws Exception;
<T extends Object> T openFileToObject(String path, Class<T> clazz) throws Exception;
BlockLocation[] getFileBlockLocations(String path) throws Exception;
}
package com.iproinfo.refinedanalysis.service.impl;
import com.iproinfo.refinedanalysis.service.HdfsService;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.web.multipart.MultipartFile;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.URI;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* @describe:
* @PageLocation:
* @Auther xzl
* @Date 2023/9/6 14:41
*/
public class HdfsServiceImpl implements HdfsService {
@Value("${hdfs.path}")
private String hdfsPath;
@Value("${hdfs.username}")
private String hdfsName;
private final int bufferSize = 1024 * 1024 * 64;
/**
* Get HDFS configuration information
* @return
*/
private Configuration getConfiguration() {
Configuration configuration = new Configuration();
configuration.set("fs.defaultFS", hdfsPath);
return configuration;
}
/**
* Get HDFS file system object
* @return
* @throwsException
*/
public FileSystem getFileSystem() throws Exception {
// The client has a user identity when operating HDFS. By default, the HDFS client API will obtain a parameter from the JVM as its user identity.
// DHADOOP_USER_NAME=hadoop
// You can also pass it in through parameters when constructing the client fs object.
FileSystem fileSystem = FileSystem.get(new URI(hdfsPath), getConfiguration(), hdfsName);
return fileSystem;
}
/**
* Create a folder in HDFS
* @param path
* @return
* @throwsException
*/
@Override
public boolean mkdir(String path) throws Exception {
if (StringUtils.isEmpty(path)) {
return false;
}
if (existFile(path)) {
return true;
}
FileSystem fs = getFileSystem();
// Target path
Path srcPath = new Path(path);
boolean isOk = fs.mkdirs(srcPath);
fs.close();
return isOk;
}
/**
* Determine whether HDFS file exists
* @param path
* @return
* @throwsException
*/
@Override
public boolean existFile(String path) throws Exception {
if (StringUtils.isEmpty(path)) {
return false;
}
FileSystem fs = getFileSystem();
Path srcPath = new Path(path);
boolean isExists = fs.exists(srcPath);
return isExists;
}
/**
* Read HDFS directory information
* @param path
* @return
* @throwsException
*/
@Override
public List<Map<String, Object>> readPathInfo(String path) throws Exception {
if (StringUtils.isEmpty(path)) {
return null;
}
if (!existFile(path)) {
return null;
}
FileSystem fs = getFileSystem();
// Target path
Path newPath = new Path(path);
FileStatus[] statusList = fs.listStatus(newPath);
List<Map<String, Object>> list = new ArrayList<>();
if (null != statusList & amp; & amp; statusList.length > 0) {
for (FileStatus fileStatus : statusList) {
Map<String, Object> map = new HashMap<>();
map.put("filePath", fileStatus.getPath());
map.put("fileStatus", fileStatus.toString());
list.add(map);
}
return list;
} else {
return null;
}
}
/**
* HDFS create file
* @param path
* @param file
* @throwsException
*/
@Override
public void createFile(String path, MultipartFile file) throws Exception {
if (StringUtils.isEmpty(path) || null == file.getBytes()) {
return;
}
String fileName = file.getOriginalFilename();
FileSystem fs = getFileSystem();
// The default is the current directory when uploading, and the directory of files will be automatically spliced later.
Path newPath = new Path(path + "/" + fileName);
//Open an output stream
FSDataOutputStream outputStream = fs.create(newPath);
outputStream.write(file.getBytes());
outputStream.close();
fs.close();
}
/**
* Read HDFS file content
* @param path
* @return
* @throwsException
*/
@Override
public String readFile(String path) throws Exception {
if (StringUtils.isEmpty(path)) {
return null;
}
if (!existFile(path)) {
return null;
}
FileSystem fs = getFileSystem();
// Target path
Path srcPath = new Path(path);
FSDataInputStream inputStream = null;
try {
inputStream = fs.open(srcPath);
// Prevent Chinese garbled characters
BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
String lineTxt = "";
StringBuffer sb = new StringBuffer();
while ((lineTxt = reader.readLine()) != null) {
sb.append(lineTxt);
}
return sb.toString();
} finally {
inputStream.close();
fs.close();
}
}
/**
* Read HDFS file list
* @param path
* @return
* @throwsException
*/
@Override
public List<Map<String, String>> listFile(String path) throws Exception {
if (StringUtils.isEmpty(path)) {
return null;
}
if (!existFile(path)) {
return null;
}
FileSystem fs = getFileSystem();
// Target path
Path srcPath = new Path(path);
// Find all files recursively
RemoteIterator<LocatedFileStatus> filesList = fs.listFiles(srcPath, true);
List<Map<String, String>> returnList = new ArrayList<>();
while (filesList.hasNext()) {
LocatedFileStatus next = filesList.next();
String fileName = next.getPath().getName();
Path filePath = next.getPath();
Map<String, String> map = new HashMap<>();
map.put("fileName", fileName);
map.put("filePath", filePath.toString());
returnList.add(map);
}
fs.close();
return returnList;
}
/**
* HDFS rename files
* @param oldName
* @param newName
* @return
* @throwsException
*/
@Override
public boolean renameFile(String oldName, String newName) throws Exception {
if (StringUtils.isEmpty(oldName) || StringUtils.isEmpty(newName)) {
return false;
}
FileSystem fs = getFileSystem();
// Original file target path
Path oldPath = new Path(oldName);
//Rename target path
Path newPath = new Path(newName);
boolean isOk = fs.rename(oldPath, newPath);
fs.close();
return isOk;
}
/**
* Delete HDFS files
* @param path
* @return
* @throwsException
*/
@Override
public boolean deleteFile(String path) throws Exception {
if (StringUtils.isEmpty(path)) {
return false;
}
if (!existFile(path)) {
return false;
}
FileSystem fs = getFileSystem();
Path srcPath = new Path(path);
boolean isOk = fs.deleteOnExit(srcPath);
fs.close();
return isOk;
}
/**
* Upload HDFS files
* @param path
* @param uploadPath
* @throwsException
*/
@Override
public void uploadFile(String path, String uploadPath) throws Exception {
if (StringUtils.isEmpty(path) || StringUtils.isEmpty(uploadPath)) {
return;
}
FileSystem fs = getFileSystem();
//Upload path
Path clientPath = new Path(path);
// Target path
Path serverPath = new Path(uploadPath);
// Call the file copy method of the file system. The first parameter is whether to delete the original file. True means deletion. The default is false.
fs.copyFromLocalFile(false, clientPath, serverPath);
fs.close();
}
/**
* Download HDFS file
* @param path
* @param downloadPath
* @throwsException
*/
@Override
public void downloadFile(String path, String downloadPath) throws Exception {
if (StringUtils.isEmpty(path) || StringUtils.isEmpty(downloadPath)) {
return;
}
FileSystem fs = getFileSystem();
//Upload path
Path clientPath = new Path(path);
// Target path
Path serverPath = new Path(downloadPath);
// Call the file copy method of the file system. The first parameter is whether to delete the original file. True means deletion. The default is false.
fs.copyToLocalFile(false, clientPath, serverPath);
fs.close();
}
/**
* HDFS file copy
* @param sourcePath
* @param targetPath
* @throwsException
*/
@Override
public void copyFile(String sourcePath, String targetPath) throws Exception {
if (StringUtils.isEmpty(sourcePath) || StringUtils.isEmpty(targetPath)) {
return;
}
FileSystem fs = getFileSystem();
//Original file path
Path oldPath = new Path(sourcePath);
// Target path
Path newPath = new Path(targetPath);
FSDataInputStream inputStream = null;
FSDataOutputStream outputStream = null;
try {
inputStream = fs.open(oldPath);
outputStream = fs.create(newPath);
IOUtils.copyBytes(inputStream, outputStream, bufferSize, false);
} finally {
inputStream.close();
outputStream.close();
fs.close();
}
}
/**
* Open the file on HDFS and return the byte array
* @param path
* @return
* @throwsException
*/
@Override
public byte[] openFileToBytes(String path) throws Exception {
if (StringUtils.isEmpty(path)) {
return null;
}
if (!existFile(path)) {
return null;
}
FileSystem fs = getFileSystem();
// Target path
Path srcPath = new Path(path);
try {
FSDataInputStream inputStream = fs.open(srcPath);
// return IOUtils.readFullyToByteArray(inputStream);
return null;
} finally {
fs.close();
}
}
/**
* Open the file on HDFS and return the java object
* @param path
* @return
* @throwsException
*/
@Override
public <T extends Object> T openFileToObject(String path, Class<T> clazz) throws Exception {
if (StringUtils.isEmpty(path)) {
return null;
}
if (!existFile(path)) {
return null;
}
String jsonStr = readFile(path);
// return JsonUtil.fromObject(jsonStr, clazz);
return null;
}
/**
* Get the cluster location of a file in HDFS
* @param path
* @return
* @throwsException
*/
@Override
public BlockLocation[] getFileBlockLocations(String path) throws Exception {
if (StringUtils.isEmpty(path)) {
return null;
}
if (!existFile(path)) {
return null;
}
FileSystem fs = getFileSystem();
// Target path
Path srcPath = new Path(path);
FileStatus fileStatus = fs.getFileStatus(srcPath);
return fs.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
}
}