简单的使用 flate,gzip,lzo,zlib,snappy 库压缩
compresser.go 简易压缩
import (
"Examples/compress/utils"
"compress/flate"
"compress/gzip"
"compress/zlib"
"fmt"
"github.com/cyberdelia/lzo"
"github.com/golang/snappy"
"io"
"os"
"time"
)
type CompressMode int
const (
Unknow CompressMode= iota
FlateMode // falte
GzipMode // gzip
LzoMode // lzo
SnappyMode // snappy
ZlibMode // zlib
)
type Compresser struct {
inFile *os.File
ouFile *os.File
absPath string
originSize int64
writer io.Writer
modeStr string
}
func New(input string) (*Compresser,error) {
fi, err := os.Stat(input)
if err != nil{
return nil,err
}
if fi.IsDir() {
return nil,fmt.Errorf("error read: because %s is dir",input)
}
c := &Compresser{originSize: fi.Size(),absPath:fi.Name()}
c.inFile, err = os.Open(input)
return c,err
}
func (c *Compresser) UseGzipWriter(out string) (io.Writer,error){
file,err := os.OpenFile(out,os.O_WRONLY|os.O_CREATE|os.O_TRUNC,0666)
if err != nil {
return nil,err
}
c.ouFile=file
w,err := gzip.NewWriterLevel(file,gzip.BestSpeed)
return w,err
}
func (c *Compresser) UseLzoWriter(out string) (io.Writer,error){
file,err := os.OpenFile(out,os.O_WRONLY|os.O_CREATE|os.O_TRUNC,0666)
if err != nil {
return nil,err
}
c.ouFile=file
w,err := lzo.NewWriterLevel(file,lzo.BestSpeed)
return w,err
}
func (c *Compresser) UseSnappyWriter(out string) (io.Writer,error){
file,err := os.OpenFile(out,os.O_WRONLY|os.O_CREATE|os.O_TRUNC,0666)
if err != nil {
return nil,err
}
c.ouFile=file
return snappy.NewBufferedWriter(file),nil
}
func (c *Compresser) UseZlibWriter(out string) (io.Writer,error){
file,err := os.OpenFile(out,os.O_WRONLY|os.O_CREATE|os.O_TRUNC,0666)
if err != nil {
return nil,err
}
c.ouFile=file
w,err := zlib.NewWriterLevel(file,zlib.BestSpeed)
return w,err
}
func (c *Compresser) UseFlateWriter(out string) (io.Writer,error){
file,err := os.OpenFile(out,os.O_WRONLY|os.O_CREATE|os.O_TRUNC,0666)
if err != nil {
return nil,err
}
c.ouFile=file
w,err := flate.NewWriter(file,flate.BestSpeed)
return w,err
}
func (c *Compresser) WithMode(mode CompressMode ) ( *Compresser,error){
var writer io.Writer
var err error
var modStr string
switch mode {
case GzipMode:
modStr="GZIP"
writer,err = c.UseGzipWriter(fmt.Sprintf("%s.gz",c.inFile.Name()))
case LzoMode:
modStr="LZO"
writer,err = c.UseLzoWriter(fmt.Sprintf("%s.lzo",c.inFile.Name()))
case SnappyMode:
modStr="SNAPPY"
writer,err = c.UseSnappyWriter(fmt.Sprintf("%s.snap",c.inFile.Name()))
case ZlibMode:
modStr="ZLIB"
writer,err = c.UseZlibWriter(fmt.Sprintf("%s.zlib",c.inFile.Name()))
case FlateMode:
modStr="FLATE"
writer,err = c.UseFlateWriter(fmt.Sprintf("%s.flate",c.inFile.Name()))
}
c.writer = writer
c.modeStr=modStr
return c,err
}
// 启动压缩 并计算时间,压缩前后文件大小
func (c *Compresser) Compress(log bool) {
if log{
startTime := time.Now()
defer c.ouFile.Close()
defer c.inFile.Close()
defer func() {
nano := time.Since(startTime)
fi,_ := c.ouFile.Stat()
fmt.Printf("使用%s压缩, 原始文件大小:%s, 压缩后大小:%s, 耗时:%v\n",
c.modeStr,utils.Bytes(uint64(c.originSize)),utils.Bytes(uint64(fi.Size())),nano )
}()
}
//buf := make([]byte,1024*100)
//_,err := io.CopyBuffer(c.writer,c.inFile,buf)
_,err := io.Copy(c.writer,c.inFile)
if err != nil {
panic(err)
}
}
byte_fmt.go 文件大小格式化
import (
"fmt"
"math"
)
const (
Byte = 1 << (iota * 10)
KiByte
MiByte
GiByte
TiByte
PiByte
EiByte
)
// SI Sizes.
const (
IByte = 1
KByte = IByte * 1000
MByte = KByte * 1000
GByte = MByte * 1000
TByte = GByte * 1000
PByte = TByte * 1000
EByte = PByte * 1000
)
var bytesSizeTable = map[string]uint64{
"b": Byte,
"kib": KiByte,
"kb": KByte,
"mib": MiByte,
"mb": MByte,
"gib": GiByte,
"gb": GByte,
"tib": TiByte,
"tb": TByte,
"pib": PiByte,
"pb": PByte,
"eib": EiByte,
"eb": EByte,
// Without suffix
"": Byte,
"ki": KiByte,
"k": KByte,
"mi": MiByte,
"m": MByte,
"gi": GiByte,
"g": GByte,
"ti": TiByte,
"t": TByte,
"pi": PiByte,
"p": PByte,
"ei": EiByte,
"e": EByte,
}
func logn(n, b float64) float64 {
return math.Log(n) / math.Log(b)
}
func humanateBytes(s uint64, base float64, sizes []string) string {
if s < 10 {
return fmt.Sprintf("%d B", s)
}
e := math.Floor(logn(float64(s), base))
suffix := sizes[int(e)]
val := math.Floor(float64(s)/math.Pow(base, e)*10+0.5) / 10
f := "%.0f %s"
if val < 10 {
f = "%.1f %s"
}
return fmt.Sprintf(f, val, suffix)
}
// Bytes produces a human readable representation of an SI size.
//
// See also: ParseBytes.
//
// Bytes(82854982) -> 83 MB
func Bytes(s uint64) string {
sizes := []string{"B", "kB", "MB", "GB", "TB", "PB", "EB"}
return humanateBytes(s, 1000, sizes)
}
// IBytes produces a human readable representation of an IEC size.
//
// See also: ParseBytes.
//
// IBytes(82854982) -> 79 MiB
func IBytes(s uint64) string {
sizes := []string{"B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB"}
return humanateBytes(s, 1024, sizes)
}
main.go 主程序
import (
"sync"
)
func main(){
wg := sync.WaitGroup{}
mode := []CompressMode{GzipMode,FlateMode,SnappyMode,LzoMode,ZlibMode}
for i := 0;i< len(mode);i++{
wg.Add(1)
go func(i int) {
c,err := New("testdata/fiction.txt")
if err != nil {
panic(err)
}
c,err = c.WithMode(mode[i])
if err != nil {
panic(err)
}
c.Compress(true)
wg.Done()
}(i)
}
wg.Wait()
}
result 结果
使用SNAPPY压缩, 原始文件大小:19 MB, 压缩后大小:12 MB, 耗时:75ms
使用LZO压缩, 原始文件大小:19 MB, 压缩后大小:12 MB, 耗时:113ms
使用GZIP压缩, 原始文件大小:19 MB, 压缩后大小:9.2 MB, 耗时:481ms
使用FLATE压缩, 原始文件大小:19 MB, 压缩后大小:9.2 MB, 耗时:482ms
使用ZLIB压缩, 原始文件大小:19 MB, 压缩后大小:9.2 MB, 耗时:486ms
