一、线性判别的原理
import warnings
warnings.filterwarnings('ignore')
import numpy as np
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn import datasets
X,y = datasets.load_iris(True)
X[:5]
array([[5.1, 3.5, 1.4, 0.2],
[4.9, 3. , 1.4, 0.2],
[4.7, 3.2, 1.3, 0.2],
[4.6, 3.1, 1.5, 0.2],
[5. , 3.6, 1.4, 0.2]])
# 矩阵运算，投影，原来的值，不同了呀
'''n_components : int, optional (default=None)
Number of components (<= min(n_classes - 1, n_features)) for
dimensionality reduction. If None, will be set to
min(n_classes - 1, n_features).'''
# n_features = 4
# n_classes = 3
# <= min(3 - 1, 4) = 2)
lda = LinearDiscriminantAnalysis(solver= 'eigen',n_components=2)
lda.fit(X,y)
X_lda = lda.transform(X)
X_lda[:5]
array([[6.01716893, 7.03257409],
[5.0745834 , 5.9344564 ],
[5.43939015, 6.46102462],
[4.75589325, 6.05166375],
[6.08839432, 7.24878907]])
自己手动计算
第一步计算总的散度矩阵，协方差 St ：scatter total
St = np.cov(X,rowvar=False,bias = True)
St
array([[ 0.68112222, -0.04215111, 1.26582 , 0.51282889],
[-0.04215111, 0.18871289, -0.32745867, -0.12082844],
[ 1.26582 , -0.32745867, 3.09550267, 1.286972 ],
[ 0.51282889, -0.12082844, 1.286972 , 0.57713289]])
第二步 计算类内的散度矩阵Sw ：scatter within
Sw = np.zeros(shape = (4,4))
for i in range(3):
index = y == i
Sw += np.cov(X[index],rowvar=False,bias = True)/3 # 一类数据
Sw
array([[0.259708 , 0.09086667, 0.164164 , 0.03763333],
[0.09086667, 0.11308 , 0.05413867, 0.032056 ],
[0.164164 , 0.05413867, 0.181484 , 0.041812 ],
[0.03763333, 0.032056 , 0.041812 , 0.041044 ]])
第三步，计算类间的散度矩阵，Sb ，scatter between
Sb = St - Sw
Sb
array([[ 0.42141422, -0.13301778, 1.101656 , 0.47519556],
[-0.13301778, 0.07563289, -0.38159733, -0.15288444],
[ 1.101656 , -0.38159733, 2.91401867, 1.24516 ],
[ 0.47519556, -0.15288444, 1.24516 , 0.53608889]])
# scipy 是高级的科学计算工具包，相对numpy来说，更多的，高级方法
from scipy import linalg
第四步，计算特征值和特征向量，考虑类间散度矩阵，类内的散度矩阵，排序
values,vectors = linalg.eigh(Sb,Sw)
vect =vectors[:,values.argsort()[::-1]]
第五步，筛选特征向量，进行矩阵运算
result = X.dot(vect[:,[0,1]])
result[:5]
array([[6.01716893, 7.03257409],
[5.0745834 , 5.9344564 ],
[5.43939015, 6.46102462],
[4.75589325, 6.05166375],
[6.08839432, 7.24878907]])
X_lda[:5]
array([[6.01716893, 7.03257409],
[5.0745834 , 5.9344564 ],
[5.43939015, 6.46102462],
[4.75589325, 6.05166375],
[6.08839432, 7.24878907]])
二、代码实例：
import warnings
warnings.filterwarnings('ignore')
import numpy as np
# LDA 线性判别
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.decomposition import PCA
from sklearn import datasets
from sklearn.neighbors import KNeighborsClassifier
from sklearn.model_selection import train_test_split
# 上课示例代码，工作数据会复杂一些
# 官网提供，经典案例，学以致用！
X,y = datasets.load_iris(True)
X_train,X_test,y_train,y_test = train_test_split(X,y,test_size = 0.2,random_state = 512)
相通的
老师两种一种是：知识传播者
另一种老师：授人以渔
奇异值（非满秩矩阵），特征值（满秩矩阵），线性代数中概念
A = X - X.mean(axis = 0)
B = np.cov(A,rowvar=False)# B是一个方阵，四行四列
B
array([[ 0.68569351, -0.042434 , 1.27431544, 0.51627069],
[-0.042434 , 0.18997942, -0.32965638, -0.12163937],
[ 1.27431544, -0.32965638, 3.11627785, 1.2956094 ],
[ 0.51627069, -0.12163937, 1.2956094 , 0.58100626]])
# B 是一个方阵，4*4
# 矩阵有秩 = 4，满秩的
# B矩阵的秩 < 4 ,非满秩矩阵，奇异矩阵
np.linalg.matrix_rank(B)
4
# 矩阵C，奇异矩阵
C = np.array([[1,2,3,4],[2,4,6,8],[2,6,1,3],[-1,2,0,6]])
display(C)
np.linalg.matrix_rank(C) # 奇异矩阵
array([[ 1, 2, 3, 4],
[ 2, 4, 6, 8],
[ 2, 6, 1, 3],
[-1, 2, 0, 6]])
3
# x + y = 3
# 2x + y = 4
# x = 1 y = 2
X1 = np.array([[1,1],[2,1]])
np.linalg.matrix_rank(X1)#满秩矩阵，方程有解！！！唯一解！
2
# x+ y = 3
# 2x + 2y = 6
# x,y ?
X2 = np.array([[1,1],[2,2]])
np.linalg.matrix_rank(X2)#非满秩，方程无解（无穷多个解，无解）！！！奇异矩阵
1
对矩阵B(鸢尾花，属性协方差)，特征值和特征向量以及SVD奇异值分解
B是满秩矩阵
# linalg ---->linear(线性) algebra(代数)
# 特征值
l,v = np.linalg.eig(B)
display(l,v)
array([4.22824171, 0.24267075, 0.0782095 , 0.02383509])
array([[ 0.36138659, -0.65658877, -0.58202985, 0.31548719],
[-0.08452251, -0.73016143, 0.59791083, -0.3197231 ],
[ 0.85667061, 0.17337266, 0.07623608, -0.47983899],
[ 0.3582892 , 0.07548102, 0.54583143, 0.75365743]])
验证特征值特征和特征向量
B.dot(v)
array([[ 1.52802986, -0.15933489, -0.04552026, 0.00751967],
[-0.35738162, -0.17718882, 0.04676231, -0.00762063],
[ 3.62221038, 0.04207247, 0.00596239, -0.01143701],
[ 1.51493333, 0.01831704, 0.0426892 , 0.01796349]])
l[0]*(v[:,0])
array([ 1.52802986, -0.35738162, 3.62221038, 1.51493333])
l[1]*v[:,1]
array([-0.15933489, -0.17718882, 0.04207247, 0.01831704])
l[2]*v[:,2]
array([-0.04552026, 0.04676231, 0.00596239, 0.0426892 ])
l[3]*v[:,3]
array([ 0.00751967, -0.00762063, -0.01143701, 0.01796349])
验证奇异值分解
# X3不是方阵
X3 = np.random.randint(0,10,size = (3,5)) # X3.shape = m*n m = 3,n = 5
X3
array([[5, 6, 1, 3, 3],
[8, 9, 4, 1, 8],
[8, 8, 8, 6, 0]])
# 奇异值分解
U,sigma,V = np.linalg.svd(X3)
display(U,sigma,V)
array([[-0.394634 , 0.14426999, 0.90744156],
[-0.65332272, 0.65038133, -0.38752233],
[-0.64609088, -0.74578167, -0.16240772]])
array([21.76187304, 7.3624042 , 2.49316791])
array([[-0.56835544, -0.61651107, -0.37573291, -0.26255874, -0.29457408],
[-0.00568467, 0.102249 , -0.43741935, -0.46065099, 0.76549187],
[ 0.05525797, 0.26379556, -0.7788924 , 0.54563353, -0.15155575],
[ 0.13517964, -0.65062123, 0.09163103, 0.56508074, 0.48031863],
[-0.80969921, 0.34141348, 0.228953 , 0.31911029, 0.27124375]])
sigma2 = np.eye(N = 3,M = 5)
for i in range(3):
sigma2[i,i] = sigma[i]
sigma2
array([[21.76187304, 0. , 0. , 0. , 0. ],
[ 0. , 7.3624042 , 0. , 0. , 0. ],
[ 0. , 0. , 2.49316791, 0. , 0. ]])
X3
array([[5, 6, 1, 3, 3],
[8, 9, 4, 1, 8],
[8, 8, 8, 6, 0]])
U.dot(sigma2).dot(V)
array([[ 5.00000000e+00, 6.00000000e+00, 1.00000000e+00,
3.00000000e+00, 3.00000000e+00],
[ 8.00000000e+00, 9.00000000e+00, 4.00000000e+00,
1.00000000e+00, 8.00000000e+00],
[ 8.00000000e+00, 8.00000000e+00, 8.00000000e+00,
6.00000000e+00, -1.45763702e-15]])
使用LDA线性判别，进行数据的降维
lda = LinearDiscriminantAnalysis(solver='eigen', n_components = 2)
lda.fit(X_train,y_train)#有监督
X_train_lda = lda.transform(X_train)#训练数据
X_test_lda = lda.transform(X_test)
X_lda.shape
(150, 2)
knn = KNeighborsClassifier()
knn.fit(X_train_lda,y_train)
knn.score(X_test_lda,y_test)
0.9666666666666667
pca = PCA(n_components=2)
pca.fit(X_train)# 无监督学习，没有目标值y
X_train_pca = pca.transform(X_train)
X_test_pca = pca.transform(X_test)
X_train_pca.shape
(120, 2)
knn = KNeighborsClassifier()
knn.fit(X_train_pca,y_train)
knn.score(X_test_pca,y_test)
0.9
