Scikit 中的乐趣
# 来源：NumPy Cookbook 2e Ch10
加载示例数据集
from __future__ import print_function
from sklearn import datasets
# datasets.load_? 用于加载不同的数据集
print filter(lambda s: s.startswith('load_'), dir(datasets))
'''
['load_boston', 'load_breast_cancer', 'load_diabetes', 'load_digits', 'load_files', 'load_iris', 'load_lfw_pairs', 'load_lfw_people', 'load_linnerud', 'load_mlcomp', 'load_sample_image', 'load_sample_images', 'load_svmlight_file', 'load_svmlight_files']
'''
# 这里加载波士顿房价数据集
# 波士顿房价数据集是连续模型
boston_prices = datasets.load_boston()
# 对于离散型数据集来说，data 是属性，target 是标签
# 对于连续型数据集来说，data 是自变量，target 是因变量
# data 是二维数组，行为记录，列为属性/自变量
print("Data shape", boston_prices.data.shape)
# Data shape (506, 13)
print("Data max=%s min=%s" % (boston_prices.data.max(), boston_prices. data.min()))
# Data max=711.0 min=0.0
# target 是标签/因变量的一维数组
print("Target shape", boston_prices.target.shape)
# Target shape (506,)
print("Target max=%s min=%s" % (boston_prices.target.max(), boston_ prices.target.min()))
# Target max=50.0 min=5.0
道琼斯股票聚类
# 2011 到 2012
start = datetime.datetime(2011, 01, 01)
end = datetime.datetime(2012, 01, 01)
# 这里是股票代码
symbols = ["AA", "AXP", "BA", "BAC", "CAT",
"CSCO", "CVX", "DD", "DIS", "GE", "HD",
"HPQ", "IBM", "INTC", "JNJ", "JPM",
"KO", "MCD", "MMM", "MRK", "MSFT", "PFE",
"PG", "T", "TRV", "UTX", "VZ", "WMT", "XOM"]
# 下载每只股票 2011 ~ 2012 的所有数据
quotes = []
for symbol in symbols:
try:
quotes.append(finance.quotes_historical_yahoo(symbol, start, end, asobject=True))
except urllib2.HTTPError as e:
print(symbol, e)
# 每只股票只取收盘价
close = np.array([q.close for q in quotes]).astype(np.float)
print(close.shape)
# (29, 252)
# 计算每只股票的对数收益
logreturns = np.diff(np.log(close))
print(logreturns.shape)
# (29, 251)
# 计算对数收益的平方和
logreturns_norms = np.sum(logreturns ** 2, axis=1)
# np.dot(logreturns, logreturns.T) 的矩阵
# 每项是 logret[i] · logret[j]
# logreturns_norms[:, np.newaxis]
# 每项是 sqsum[i]
# logreturns_norms[np. newaxis, :]
# 每项是 sqsum[j]
# S 的每一项就是 logret[i] 和 logret[j] 的欧氏距离
S = - logreturns_norms[:, np.newaxis] - logreturns_norms[np. newaxis, :] + 2 * np.dot(logreturns, logreturns.T)
# 使用 AP 算法进行聚类
# AffinityPropagation 用于创建聚类器
# 向 fit 传入距离矩阵可以对其聚类
# 用于聚类的属性是每个向量到其它向量的距离
aff_pro = sklearn.cluster.AffinityPropagation().fit(S)
# labels_ 获取聚类结果
labels = aff_pro.labels_
# 打印每只股票的类别
for symbol, label in zip(symbols, labels):
print('%s in Cluster %d' % (symbol, label))
'''
AA in Cluster 0
AXP in Cluster 6
BA in Cluster 6
BAC in Cluster 1
CAT in Cluster 6
CSCO in Cluster 2
CVX in Cluster 7
DD in Cluster 6
DIS in Cluster 6
GE in Cluster 6
HD in Cluster 5
HPQ in Cluster 3
IBM in Cluster 5
INTC in Cluster 6
JNJ in Cluster 5
JPM in Cluster 4
KO in Cluster 5
MCD in Cluster 5
MMM in Cluster 6
MRK in Cluster 5
MSFT in Cluster 5
PFE in Cluster 7
PG in Cluster 5
T in Cluster 5
TRV in Cluster 5
UTX in Cluster 6
VZ in Cluster 5
WMT in Cluster 5
XOM in Cluster 7
使用 statsmodels 执行正态性测试
from __future__ import print_function
import datetime
import numpy as np
from matplotlib import finance
from statsmodels.stats.adnorm import normal_ad
# 下载 2011 到 2012 的收盘价数据
start = datetime.datetime(2011, 01, 01)
end = datetime.datetime(2012, 01, 01)
quotes = finance.quotes_historical_yahoo('AAPL', start, end, asobject=True)
close = np.array(quotes.close).astype(np.float)
print(close.shape)
# (252,)
# 对对数收益执行正态性测试
# 也就是是否满足正态分布
# normal_ad 使用 Anderson-Darling 测试
# 请见 http://en.wikipedia.org/wiki/Anderson%E2%80%93Darling_test
print(normal_ad(np.diff(np.log(close))))
# (0.57103805516803163, 0.13725944999430437)
# p-value，也就是概率为 0.13
角点检测
from skimage.feature import corner_peaks
from skimage.color import rgb2gray
# 加载示例图片（亭子那张）
dataset = load_sample_images()
img = dataset.images[0]
# 将 RGB 图像转成灰度
gray_img = rgb2gray(img)
# 使用 Harris 角点检测器
# http://en.wikipedia.org/wiki/Corner_detection
harris_coords = corner_peaks(corner_harris(gray_img))
# harris_coords 第一列是 y，第二列是 x
y, x = np.transpose(harris_coords)
plt.axis('off')
# 绘制图像和角点
plt.imshow(img)
plt.plot(x, y, 'ro')
plt.show()
边界检测
from sklearn.datasets import load_sample_images
import matplotlib.pyplot as plt
import skimage.feature
# 加载示例图片（亭子那张）
dataset = load_sample_images()
img = dataset.images[0]
# 使用 Canny 过滤器检测边界
# 基于高斯分布的标准差
# http://en.wikipedia.org/wiki/Edge_detection
edges = skimage.feature.canny(img[..., 0])
# 绘制图像
plt.axis('off')
plt.imshow(edges)
plt.show()
