文章目录
- POM依赖
- HDFS API
- 上传本地数据到HDFS系统
- 删除指定路径下的文件
- 创建文件夹及文件,并向文件中写入数据
- 获取指定文件的日期
- 重命名目录
- 获取指定路径所有的文件
POM依赖
<repositories>
<repository>
<id>cloudera</id>
<url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
</repository>
</repositories>
<dependencies>
<dependency>
<groupId>org.apache.Hadoop</groupId>
<artifactId>Hadoop-client</artifactId>
<version>2.6.0-mr1-cdh5.14.0</version>
</dependency>
<dependency>
<groupId>org.apache.Hadoop</groupId>
<artifactId>Hadoop-common</artifactId>
<version>2.6.0-cdh5.14.0</version>
</dependency>
<dependency>
<groupId>org.apache.Hadoop</groupId>
<artifactId>Hadoop-hdfs</artifactId>
<version>2.6.0-cdh5.14.0</version>
</dependency>
<dependency>
<groupId>org.apache.Hadoop</groupId>
<artifactId>Hadoop-mapreduce-client-core</artifactId>
<version>2.6.0-cdh5.14.0</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
<version>RELEASE</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.0</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
<encoding>UTF-8</encoding>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.4.3</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<minimizeJar>true</minimizeJar>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
HDFS API
上传本地数据到HDFS系统
copyFromLocalFile()
@Test
public void copyFromLocalFile() throws Exception {
Configuration configuration = new Configuration();
FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.100.21/"), configuration);
fileSystem.copyFromLocalFile(new Path("D://123.txt"), new Path("/"));
fileSystem.close();
}
删除指定路径下的文件
delete()
public void deleteFile() throws Exception {
Configuration configuration = new Configuration();
FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.100.21/"), configuration);
Path rmPath = new Path("/abc/a.txt");
boolean delete = fileSystem.delete(rmPath, true);
System.out.println(delete ? "文件删除成功" : "文件删除失败");
fileSystem.close();
}
创建文件夹及文件,并向文件中写入数据
mkdir()
create()
public void createData() throws Exception {
Configuration configuration = new Configuration();
FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.100.21"), configuration);
FSDataOutputStream fsDataOutputStream = null;
Path dir = new Path("/abc");
if (!fileSystem.exists(dir)) {
fileSystem.mkdirs(dir);
}
Path file = new Path(dir, "/a.txt");
if (!fileSystem.exists(file)) {
fsDataOutputStream = fileSystem.create(file, true, 4096, new Progressable() {
@Override
public void progress() {
System.out.println("显示创建进度(可有可无)");
}
});
}
byte[] bytes = "向文件中写入的数据内容".getBytes();
fsDataOutputStream.write(bytes, 0, bytes.length);
fsDataOutputStream.flush();
fsDataOutputStream.close();
fileSystem.close();
}
获取指定文件的日期
getFileStatus()
getModificationTime()
public void getTime() throws Exception {
Configuration configuration = new Configuration();
FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.100.21/"), configuration);
FileStatus fileStatus = fileSystem.getFileStatus(new Path("/a.txt"));
long modificationTime = fileStatus.getModificationTime();
System.out.println(modificationTime);
fileSystem.close();
}
重命名目录
rename()
public void rename() throws Exception {
Configuration configuration = new Configuration();
FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.100.21/"), configuration);
Path oldPath = new Path("/abc");
Path newPath = new Path("/123");
boolean rename = fileSystem.rename(oldPath, newPath);
System.out.println(rename ? "重命名成功" : "重命名失败");
fileSystem.close();
}
获取指定路径所有的文件
listStatus()
public void listStatus() throws Exception {
Configuration configuration = new Configuration();
FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.100.21/"), configuration);
FileStatus[] fileStatuses = fileSystem.listStatus(new Path("/"));
for (FileStatus fileStatus : fileStatuses) {
System.out.println(fileStatus.getPath().toString());
}
fileSystem.close();
}
本文内容由网友自发贡献,版权归原作者所有,本站不承担相应法律责任。如您发现有涉嫌抄袭侵权的内容,请联系:hwhale#tublm.com(使用前将#替换为@)