Hadoop压缩和解压缩文件大数据BLU111的博客-

Hadoop压缩和解压缩文件

修改Hadoop_WordCount单词统计工程

1. 创建CompressionTest类

package com.blu.compress; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.io.compress.CompressionCodecFactory; import org.apache.hadoop.io.compress.CompressionInputStream; import org.apache.hadoop.io.compress.CompressionOutputStream; import org.junit.Test; public class CompressionTest { /**   * 压缩文件   *    * @author BLU   */ @Test public void compress_test() throws Exception { //文件输入流（要压缩的文件）   FileInputStream fis = new FileInputStream(new File("D:\data\money.txt")); //创建Configuration   Configuration conf = new Configuration(); //获得CompressionCodecFactory   CompressionCodecFactory factory = new CompressionCodecFactory(conf); //根据类名获得不同压缩方式的CompressionCodec的对象   CompressionCodec  codec = factory.getCodecByClassName("org.apache.hadoop.io.compress.GzipCodec"); //文件输出流   FileOutputStream fos = new FileOutputStream(new File("D:\data\compress_test.txt" + codec.getDefaultExtension())); //文件输出流转换为压缩文件输出流   CompressionOutputStream cos = codec.createOutputStream(fos); //输出   IOUtils.copyBytes(fis, cos, conf); //关闭流   IOUtils.closeStream(cos);   IOUtils.closeStream(fos);   IOUtils.closeStream(fis); } /**   * 解压文件   *    * @author BLU   */ @Test public void decompress_test() throws Exception { //要解压的文件   String path = "D:\data\compress_test.txt.gz"; //创建Configuration   Configuration conf = new Configuration(); //获得CompressionCodecFactory   CompressionCodecFactory factory = new CompressionCodecFactory(conf);   CompressionCodec codec = factory.getCodec(new Path(path)); if(codec != null) { //获取解压缩的输入流    FileInputStream fis = new FileInputStream(new File(path));    CompressionInputStream cis = codec.createInputStream(fis); //输出流FileOutputStream    FileOutputStream fos = new FileOutputStream("D:\data\decompress.txt"); //输出和关闭流    IOUtils.copyBytes(cis, fos, conf);    IOUtils.closeStream(fos);    IOUtils.closeStream(cis);    IOUtils.closeStream(fis); }else {    System.out.println("不支持的解压类型"); } } } 

2. 运行compress_test()压缩方法，在D:data目录下生成了compress_test.txt.gz压缩包文件
3. 再运行decompress_test()解压缩方法，生成了decompress.txt文件