用C++写扩展改善Python性能

Tags: Boost.Python, PyBind11, md5sum, python md5 文件

CPython众所周知是慢的，尤其是大量的循环… 山哥最近想用Python搞个文件系统同步的工具，希望用MD5 Checksum来比对两个文件是不是一毛一样。

原先是直接用Python读取文件，然后计算MD5的Hex串。发现操作100M以上的文件实在是太慢了，1G以上的文件要喝完一杯水都出不来结果，这个实在接受不了！上代码。。

# -*- coding:utf-8 -*-

import hashlib
import os


md5_file(filename):
    if not os.path.isfile(filename):
        return None
    md5 = hashlib.md5()
    f = open(filename, 'rb')
    while True:
        b = f.read(8192)
        if not b:
            break
        md5.update(b)
    f.close()
    return md5.hexdigest()


if __name__ == '__main__':
    print(md5_file('D:\\迅雷下载\\ubuntu-16.04.2-desktop-amd64.iso'))

改善方案之一，用Linux/Unix系统工具

那我们能不能直接用系统工具 MD5Sum来做呢？当然可以！但是Windows下面是没有的，要下载一个，挺麻烦的，不怕麻烦的可以这样做，性能自然是杠杠的！

# -*- coding:utf-8 -*-
import subprocess
import locale

#此处取得系统的encoding在后面用，如果不加以处理，中文的文件名可能会出问题，尤其是万恶的Windows
SYS_ENCODING = locale.getpreferredencoding()

def md5sum(file):
    cmd = "md5sum " + file
    fp = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    (s, e) = fp.communicate()
    result = (s.decode(encoding=SYS_ENCODING).split(' ')[0]).strip()
    e = (e.decode(encoding=SYS_ENCODING)).strip()
    return result, e


if __name__ == '__main__':
    checksum, error = md5sum('D:\\迅雷下载\\ubuntu-16.04.2-desktop-amd64.iso')
    print(checksum)

改善方案之二，用Cython…

试了一下，直接用Cython对性能并没有多少提升，如果用Cython的方式大改的话，代码丑的一塌糊涂！所以就不想玩了。。Byebye Cython！

改善方案之三，用C/C++写

Python被设计于容易与C/C++整合，Python的哲学就是“有问题找C哥哥，没问题找我”！市面上那么多产品可以做到用C扩展，那么问题来了，到底“C++写Python扩展哪家强”呢？经过多方综合，我是偏向于Boost.Python的，因为它不用修改C的代码，加一个Export的MACRO说明就可以了！但是Hold on! 现在竟然有人说Boost.Python由于要考虑各种老旧傻B的编译器，变得很臃肿，已经有人把它用C++11标准重新实现，瘦身之后成功晋升白富美！那就是Pybind11 （名字很蹉但也很亲民直接，跟二丫阿凤这种名字一个水平）！于是山哥按捺住激动的心情，把Boost.Python和Pybind11都试了一下。果然，Boost.Python要事先编译一个400K大小的Dll，那么您的程序以后到哪里都要带着这个拖油瓶！而Pybind11只需要加入头文件就可以了，编译之后只有包含你业务逻辑的 Pyd（Windows） 或者 SO文件 （Unix/Linux）

Export的代码

//export.cpp

#include <pybind11/pybind11.h>

#include "md5.h"
#include <iostream>

namespace py = pybind11;


#define BUF_SIZE 8192

// 用wstring是为了中文问题，你懂的。。
string md5sum(const std::wstring &filename)
{
    //setlocale(LC_ALL, "zh_CN.utf8");
    std::cout << filename.c_str() << std::endl;
    ifstream in(filename.c_str(), std::ios::binary);
    //PyCode
    if (!in)
        return "";

    MD5 md5;
    std::streamsize length;
    char buffer[BUF_SIZE];
    while (!in.eof()) {
        in.read(buffer, BUF_SIZE);
        length = in.gcount();
        if (length > 0)
            md5.update(buffer, length);
    }
    in.close();
    return md5.toString();
}

// 这个是module的名字，编译后在Windows下要起名叫md5sum.pyd
PYBIND11_PLUGIN(md5sum)
{
    py::module m("md5sum", "md5sum c++ binding for python");
    m.def("digest_file", &md5sum, "A function which digest a file and returns md5 hex string");
    return m.ptr();
}

测试的程序如下，速度果然快到飞起！

# -*- coding:utf-8 -*-
# coding:utf-8
import md5sum
res = md5sum.digest_file('D:\\迅雷下载\\ubuntu-16.04.2-desktop-amd64.iso')
print("result is " + res)

关于编译。在Mac或者Linux下面太容易就不说了，Windows下，用Visual Studio 2017，是这么编译的：

• 新建空白的Win32 dll 工程。
• 加入包含目录 C:\Python36\include和C:\Commands\CPP\pybind11-stable\include
• 加入库目录 C:\Python36\libs
• 编译的结果取名为md5sum扩展名为.pyd

好了！大功告成！
附上MD5的C++源码

// md5.h
#ifndef MD5_H  
#define MD5_H  

#include <string>  
#include <fstream>  

/* Type define */
typedef unsigned char byte;
typedef unsigned int uint32;

using std::string;
using std::ifstream;

/* MD5 declaration. */
class MD5 {
public:
    MD5();
    MD5(const void *input, size_t length);
    MD5(const string &str);
    MD5(ifstream &in);
    void update(const void *input, size_t length);
    void update(const string &str);
    void update(ifstream &in);
    const byte* digest();
    string toString();
    void reset();
private:
    void update(const byte *input, size_t length);
    void final();
    void transform(const byte block[64]);
    void encode(const uint32 *input, byte *output, size_t length);
    void decode(const byte *input, uint32 *output, size_t length);
    string bytesToHexString(const byte *input, size_t length);

    /* class uncopyable */
    MD5(const MD5&);
    MD5& operator=(const MD5&);
private:
    uint32 _state[4];   /* state (ABCD) */
    uint32 _count[2];   /* number of bits, modulo 2^64 (low-order word first) */
    byte _buffer[64];   /* input buffer */
    byte _digest[16];   /* message digest */
    bool _finished;     /* calculate finished ? */

    static const byte PADDING[64];  /* padding for calculate */
    static const char HEX[16];
    static const size_t BUFFER_SIZE = 1024;
};

#endif/*MD5_H*/ #pragma once

//md5.cpp

#include "md5.h"  

using namespace std;

/* Constants for MD5Transform routine. */
#define S11 7  
#define S12 12  
#define S13 17  
#define S14 22  
#define S21 5  
#define S22 9  
#define S23 14  
#define S24 20  
#define S31 4  
#define S32 11  
#define S33 16  
#define S34 23  
#define S41 6  
#define S42 10  
#define S43 15  
#define S44 21  


/* F, G, H and I are basic MD5 functions.
*/
#define F(x, y, z) (((x) & (y)) | ((~x) & (z)))  
#define G(x, y, z) (((x) & (z)) | ((y) & (~z)))  
#define H(x, y, z) ((x) ^ (y) ^ (z))  
#define I(x, y, z) ((y) ^ ((x) | (~z)))  

/* ROTATE_LEFT rotates x left n bits.
*/
#define ROTATE_LEFT(x, n) (((x) << (n)) | ((x) >> (32-(n))))  

/* FF, GG, HH, and II transformations for rounds 1, 2, 3, and 4.
Rotation is separate from addition to prevent recomputation.
*/
#define FF(a, b, c, d, x, s, ac) { \
    (a) += F ((b), (c), (d)) + (x) + ac; \
    (a) = ROTATE_LEFT ((a), (s)); \
    (a) += (b); \
}  
#define GG(a, b, c, d, x, s, ac) { \
    (a) += G ((b), (c), (d)) + (x) + ac; \
    (a) = ROTATE_LEFT ((a), (s)); \
    (a) += (b); \
}  
#define HH(a, b, c, d, x, s, ac) { \
    (a) += H ((b), (c), (d)) + (x) + ac; \
    (a) = ROTATE_LEFT ((a), (s)); \
    (a) += (b); \
}  
#define II(a, b, c, d, x, s, ac) { \
    (a) += I ((b), (c), (d)) + (x) + ac; \
    (a) = ROTATE_LEFT ((a), (s)); \
    (a) += (b); \
}  


const byte MD5::PADDING[64] = { 0x80 };
const char MD5::HEX[16] = {
    '0', '1', '2', '3',
    '4', '5', '6', '7',
    '8', '9', 'a', 'b',
    'c', 'd', 'e', 'f'
};

/* Default construct. */
MD5::MD5() {
    reset();
}

/* Construct a MD5 object with a input buffer. */
MD5::MD5(const void *input, size_t length) {
    reset();
    update(input, length);
}

/* Construct a MD5 object with a string. */
MD5::MD5(const string &str) {
    reset();
    update(str);
}

/* Construct a MD5 object with a file. */
MD5::MD5(ifstream &in) {
    reset();
    update(in);
}

/* Return the message-digest */
const byte* MD5::digest() {
    if (!_finished) {
        _finished = true;
        final();
    }
    return _digest;
}

/* Reset the calculate state */
void MD5::reset() {

    _finished = false;
    /* reset number of bits. */
    _count[0] = _count[1] = 0;
    /* Load magic initialization constants. */
    _state[0] = 0x67452301;
    _state[1] = 0xefcdab89;
    _state[2] = 0x98badcfe;
    _state[3] = 0x10325476;
}

/* Updating the context with a input buffer. */
void MD5::update(const void *input, size_t length) {
    update((const byte*)input, length);
}

/* Updating the context with a string. */
void MD5::update(const string &str) {
    update((const byte*)str.c_str(), str.length());
}

/* Updating the context with a file. */
void MD5::update(ifstream &in) {

    if (!in)
        return;

    std::streamsize length;
    char buffer[BUFFER_SIZE];
    while (!in.eof()) {
        in.read(buffer, BUFFER_SIZE);
        length = in.gcount();
        if (length > 0)
            update(buffer, length);
    }
    in.close();
}

/* MD5 block update operation. Continues an MD5 message-digest
operation, processing another message block, and updating the
context.
*/
void MD5::update(const byte *input, size_t length) {

    uint32 i, index, partLen;

    _finished = false;

    /* Compute number of bytes mod 64 */
    index = (uint32)((_count[0] >> 3) & 0x3f);

    /* update number of bits */
    if ((_count[0] += ((uint32)length << 3)) < ((uint32)length << 3))
        _count[1]++;
    _count[1] += ((uint32)length >> 29);

    partLen = 64 - index;

    /* transform as many times as possible. */
    if (length >= partLen) {

        memcpy(&_buffer[index], input, partLen);
        transform(_buffer);

        for (i = partLen; i + 63 < length; i += 64)
            transform(&input[i]);
        index = 0;

    }
    else {
        i = 0;
    }

    /* Buffer remaining input */
    memcpy(&_buffer[index], &input[i], length - i);
}

/* MD5 finalization. Ends an MD5 message-_digest operation, writing the
the message _digest and zeroizing the context.
*/
void MD5::final() {

    byte bits[8];
    uint32 oldState[4];
    uint32 oldCount[2];
    uint32 index, padLen;

    /* Save current state and count. */
    memcpy(oldState, _state, 16);
    memcpy(oldCount, _count, 8);

    /* Save number of bits */
    encode(_count, bits, 8);

    /* Pad out to 56 mod 64. */
    index = (uint32)((_count[0] >> 3) & 0x3f);
    padLen = (index < 56) ? (56 - index) : (120 - index);
    update(PADDING, padLen);

    /* Append length (before padding) */
    update(bits, 8);

    /* Store state in digest */
    encode(_state, _digest, 16);

    /* Restore current state and count. */
    memcpy(_state, oldState, 16);
    memcpy(_count, oldCount, 8);
}

/* MD5 basic transformation. Transforms _state based on block. */
void MD5::transform(const byte block[64]) {

    uint32 a = _state[0], b = _state[1], c = _state[2], d = _state[3], x[16];

    decode(block, x, 64);

    /* Round 1 */
    FF(a, b, c, d, x[0], S11, 0xd76aa478); /* 1 */
    FF(d, a, b, c, x[1], S12, 0xe8c7b756); /* 2 */
    FF(c, d, a, b, x[2], S13, 0x242070db); /* 3 */
    FF(b, c, d, a, x[3], S14, 0xc1bdceee); /* 4 */
    FF(a, b, c, d, x[4], S11, 0xf57c0faf); /* 5 */
    FF(d, a, b, c, x[5], S12, 0x4787c62a); /* 6 */
    FF(c, d, a, b, x[6], S13, 0xa8304613); /* 7 */
    FF(b, c, d, a, x[7], S14, 0xfd469501); /* 8 */
    FF(a, b, c, d, x[8], S11, 0x698098d8); /* 9 */
    FF(d, a, b, c, x[9], S12, 0x8b44f7af); /* 10 */
    FF(c, d, a, b, x[10], S13, 0xffff5bb1); /* 11 */
    FF(b, c, d, a, x[11], S14, 0x895cd7be); /* 12 */
    FF(a, b, c, d, x[12], S11, 0x6b901122); /* 13 */
    FF(d, a, b, c, x[13], S12, 0xfd987193); /* 14 */
    FF(c, d, a, b, x[14], S13, 0xa679438e); /* 15 */
    FF(b, c, d, a, x[15], S14, 0x49b40821); /* 16 */

                                            /* Round 2 */
    GG(a, b, c, d, x[1], S21, 0xf61e2562); /* 17 */
    GG(d, a, b, c, x[6], S22, 0xc040b340); /* 18 */
    GG(c, d, a, b, x[11], S23, 0x265e5a51); /* 19 */
    GG(b, c, d, a, x[0], S24, 0xe9b6c7aa); /* 20 */
    GG(a, b, c, d, x[5], S21, 0xd62f105d); /* 21 */
    GG(d, a, b, c, x[10], S22, 0x2441453); /* 22 */
    GG(c, d, a, b, x[15], S23, 0xd8a1e681); /* 23 */
    GG(b, c, d, a, x[4], S24, 0xe7d3fbc8); /* 24 */
    GG(a, b, c, d, x[9], S21, 0x21e1cde6); /* 25 */
    GG(d, a, b, c, x[14], S22, 0xc33707d6); /* 26 */
    GG(c, d, a, b, x[3], S23, 0xf4d50d87); /* 27 */
    GG(b, c, d, a, x[8], S24, 0x455a14ed); /* 28 */
    GG(a, b, c, d, x[13], S21, 0xa9e3e905); /* 29 */
    GG(d, a, b, c, x[2], S22, 0xfcefa3f8); /* 30 */
    GG(c, d, a, b, x[7], S23, 0x676f02d9); /* 31 */
    GG(b, c, d, a, x[12], S24, 0x8d2a4c8a); /* 32 */

                                            /* Round 3 */
    HH(a, b, c, d, x[5], S31, 0xfffa3942); /* 33 */
    HH(d, a, b, c, x[8], S32, 0x8771f681); /* 34 */
    HH(c, d, a, b, x[11], S33, 0x6d9d6122); /* 35 */
    HH(b, c, d, a, x[14], S34, 0xfde5380c); /* 36 */
    HH(a, b, c, d, x[1], S31, 0xa4beea44); /* 37 */
    HH(d, a, b, c, x[4], S32, 0x4bdecfa9); /* 38 */
    HH(c, d, a, b, x[7], S33, 0xf6bb4b60); /* 39 */
    HH(b, c, d, a, x[10], S34, 0xbebfbc70); /* 40 */
    HH(a, b, c, d, x[13], S31, 0x289b7ec6); /* 41 */
    HH(d, a, b, c, x[0], S32, 0xeaa127fa); /* 42 */
    HH(c, d, a, b, x[3], S33, 0xd4ef3085); /* 43 */
    HH(b, c, d, a, x[6], S34, 0x4881d05); /* 44 */
    HH(a, b, c, d, x[9], S31, 0xd9d4d039); /* 45 */
    HH(d, a, b, c, x[12], S32, 0xe6db99e5); /* 46 */
    HH(c, d, a, b, x[15], S33, 0x1fa27cf8); /* 47 */
    HH(b, c, d, a, x[2], S34, 0xc4ac5665); /* 48 */

                                           /* Round 4 */
    II(a, b, c, d, x[0], S41, 0xf4292244); /* 49 */
    II(d, a, b, c, x[7], S42, 0x432aff97); /* 50 */
    II(c, d, a, b, x[14], S43, 0xab9423a7); /* 51 */
    II(b, c, d, a, x[5], S44, 0xfc93a039); /* 52 */
    II(a, b, c, d, x[12], S41, 0x655b59c3); /* 53 */
    II(d, a, b, c, x[3], S42, 0x8f0ccc92); /* 54 */
    II(c, d, a, b, x[10], S43, 0xffeff47d); /* 55 */
    II(b, c, d, a, x[1], S44, 0x85845dd1); /* 56 */
    II(a, b, c, d, x[8], S41, 0x6fa87e4f); /* 57 */
    II(d, a, b, c, x[15], S42, 0xfe2ce6e0); /* 58 */
    II(c, d, a, b, x[6], S43, 0xa3014314); /* 59 */
    II(b, c, d, a, x[13], S44, 0x4e0811a1); /* 60 */
    II(a, b, c, d, x[4], S41, 0xf7537e82); /* 61 */
    II(d, a, b, c, x[11], S42, 0xbd3af235); /* 62 */
    II(c, d, a, b, x[2], S43, 0x2ad7d2bb); /* 63 */
    II(b, c, d, a, x[9], S44, 0xeb86d391); /* 64 */

    _state[0] += a;
    _state[1] += b;
    _state[2] += c;
    _state[3] += d;
}

/* Encodes input (ulong) into output (byte). Assumes length is
a multiple of 4.
*/
void MD5::encode(const uint32 *input, byte *output, size_t length) {

    for (size_t i = 0, j = 0; j<length; i++, j += 4) {
        output[j] = (byte)(input[i] & 0xff);
        output[j + 1] = (byte)((input[i] >> 8) & 0xff);
        output[j + 2] = (byte)((input[i] >> 16) & 0xff);
        output[j + 3] = (byte)((input[i] >> 24) & 0xff);
    }
}

/* Decodes input (byte) into output (ulong). Assumes length is
a multiple of 4.
*/
void MD5::decode(const byte *input, uint32 *output, size_t length) {

    for (size_t i = 0, j = 0; j<length; i++, j += 4) {
        output[i] = ((uint32)input[j]) | (((uint32)input[j + 1]) << 8) |
            (((uint32)input[j + 2]) << 16) | (((uint32)input[j + 3]) << 24);
    }
}

/* Convert byte array to hex string. */
string MD5::bytesToHexString(const byte *input, size_t length) {
    string str;
    str.reserve(length << 1);
    for (size_t i = 0; i < length; i++) {
        int t = input[i];
        int a = t / 16;
        int b = t % 16;
        str.append(1, HEX[a]);
        str.append(1, HEX[b]);
    }
    return str;
}

/* Convert digest to string value */
string MD5::toString() {
    return bytesToHexString(digest(), 16);
}