目录
1、import python package and analysis data（导入和初步分析数据）
Analyze by pivoting features（数据表格）
Analyze by visualizing data （可视化-将数据表格的猜想可视化）
Correcting by dropping features （删除部分特征-比如缺失值严重）
Creating new feature extracting from existing（包括文本处理）
Completing a numerical continuous feature (对连续值特征的处理)
Create new feature combining existing features（通过现有特征分析产生新特征（alone））
Completing a categorical feature（对分类特征的处理）
Converting categorical feature to numeric（将字符分类改为数值形式）
Quick completing and converting a numeric feature
4、Model, predict and solve（建模、预测）
1、import python package and analysis data（导入和初步分析数据）
首先对数据进行分析：观察出数据中的二分类、多分类、连续数值、离散的数值（但不是分类问题）、文本。
# 读取数据、合并
train_df = pd.read_csv('../input/train.csv')
test_df = pd.read_csv('../input/test.csv')
combine = [train_df, test_df]
# 检视数据1（观察是否混合类型【订单号】、是否错误值【名称拼写】）
train_df.head()
# 检视数据2（观察数据类型【整数、字符串】、空值数量）
train_df.info()
test_df.info()
# 检视数据3 （观察每个特征的分布情况）
train_df.describe() #数值型
train_df.describe(include=['O']) #对象型
2、Analyze data（分析数据）
Analyze by pivoting features（数据表格）
# 看各自变量与因变量的关系（分布-选出相关性强的若干种并作出假设-比如婴儿的存活率更高）
train_df[["Sex", "Survived"]].groupby(['Sex'], as_index=False).mean().sort_values(by='Survived', ascending=False)
Analyze by visualizing data （可视化-将数据表格的猜想可视化）
#关联数值特征（验证婴儿存活率的假设）
g = sns.FacetGrid(train_df, col='Survived')
g.map(plt.hist, 'Age', bins=20)
# 关联数值特征和序数特征
grid = sns.FacetGrid(train_df, col='Survived', row='Pclass', size=2.2, aspect=1.6)
grid.map(plt.hist, 'Age', alpha=.5, bins=20)
grid.add_legend()
# 关联分类特征
grid = sns.FacetGrid(train_df, row='Embarked', size=2.2, aspect=1.6)
grid.map(sns.pointplot, 'Pclass', 'Survived', 'Sex', palette='deep')
grid.add_legend()
# 关联分类和数值特征
grid = sns.FacetGrid(train_df, row='Embarked', col='Survived', size=2.2, aspect=1.6)
grid.map(sns.barplot, 'Sex', 'Fare', alpha=.5, ci=None)
grid.add_legend()
3、Wrangle data（处理错误数据）
主要包括：删除特征、新增特征、特征变换（连续值分段、分类特征数值化）
Correcting by dropping features （删除部分特征-比如缺失值严重）
train_df = train_df.drop(['Ticket', 'Cabin'], axis=1)
test_df = test_df.drop(['Ticket', 'Cabin'], axis=1)
combine = [train_df, test_df]
Creating new feature extracting from existing（包括文本处理）
for dataset in combine:
dataset['Title'] = dataset.Name.str.extract(' ([A-Za-z]+)\.', expand=False)
pd.crosstab(train_df['Title'], train_df['Sex']) #交叉表，按指定的行列统计分组频数；sum(sex) group by Title
# 将对象特征转换为序数特征
title_mapping = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}
for dataset in combine:
dataset['Title'] = dataset['Title'].map(title_mapping)
dataset['Title'] = dataset['Title'].fillna(0)
Completing a numerical continuous feature (对连续值特征的处理)
# 填补空值的三种方法，此处选第二种
1、平均值+-方差之间的随机数
2、中值
3、中值+-方差之间的随机数
for dataset in combine:
for i in range(0, 2):
for j in range(0, 3):
guess_df = dataset[(dataset['Sex'] == i) & \
(dataset['Pclass'] == j+1)]['Age'].dropna()
age_guess = guess_df.median()
# Convert random age float to nearest .5 age
guess_ages[i,j] = int( age_guess/0.5 + 0.5 ) * 0.5
for i in range(0, 2):
for j in range(0, 3):
dataset.loc[ (dataset.Age.isnull()) & (dataset.Sex == i) & (dataset.Pclass == j+1),\
'Age'] = guess_ages[i,j]
dataset['Age'] = dataset['Age'].astype(int)
# 连续数值特征分段
train_df['AgeBand'] = pd.cut(train_df['Age'], 5)# 按数值值等分，区别 qcut()按数值个数等分
train_df[['AgeBand', 'Survived']].groupby(['AgeBand'], as_index=False).mean().sort_values(by='AgeBand', ascending=True)
for dataset in combine:
dataset.loc[ dataset['Age'] <= 16, 'Age'] = 0
dataset.loc[(dataset['Age'] > 16) & (dataset['Age'] <= 32), 'Age'] = 1
dataset.loc[(dataset['Age'] > 32) & (dataset['Age'] <= 48), 'Age'] = 2
dataset.loc[(dataset['Age'] > 48) & (dataset['Age'] <= 64), 'Age'] = 3
dataset.loc[ dataset['Age'] > 64, 'Age']
train_df.head()
Create new feature combining existing features（通过现有特征分析产生新特征（alone））
# 将两个数值特征相加合并
for dataset in combine:
dataset['FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1
Completing a categorical feature（对分类特征的处理）
#填补缺失值（只有两个，所以按最常用的填补）
freq_port = train_df.Embarked.dropna().mode()[0] # 最常见值
for dataset in combine:
dataset['Embarked'] = dataset['Embarked'].fillna(freq_port)
train_df[['Embarked', 'Survived']].groupby(['Embarked'], as_index=False).mean().sort_values(by='Survived', ascending=False)
Converting categorical feature to numeric（将字符分类改为数值形式）
for dataset in combine:
dataset['Embarked'] = dataset['Embarked'].map( {'S': 0, 'C': 1, 'Q': 2} ).astype(int)
Quick completing and converting a numeric feature
test_df['Fare'].fillna(test_df['Fare'].dropna().median(), inplace=True)# 缺失较少，取中值
4、Model, predict and solve（建模、预测）
# 数据准备
X_train = train_df.drop("Survived", axis=1)
Y_train = train_df["Survived"]
X_test = test_df.drop("PassengerId", axis=1).copy()
X_train.shape, Y_train.shape, X_test.shape
# Logistic Regression
logreg = LogisticRegression()
logreg.fit(X_train, Y_train)
Y_pred = logreg.predict(X_test)
acc_log = round(logreg.score(X_train, Y_train) * 100, 2)
----acc_log =80.359999999999999
# 验证各自变量与因变量之间的相关性
coeff_df = pd.DataFrame(train_df.columns.delete(0))
coeff_df.columns = ['Feature']
coeff_df["Correlation"] = pd.Series(logreg.coef_[0])
coeff_df.sort_values(by='Correlation', ascending=False)
# Support Vector Machines
svc = SVC()
svc.fit(X_train, Y_train)
Y_pred = svc.predict(X_test)
acc_svc = round(svc.score(X_train, Y_train) * 100, 2)
----acc_svc =83.840000000000003
# KNN
knn = KNeighborsClassifier(n_neighbors = 3)
knn.fit(X_train, Y_train)
Y_pred = knn.predict(X_test)
acc_knn = round(knn.score(X_train, Y_train) * 100, 2)
----acc_knn =84.739999999999995
# Gaussian Naive Bayes
gaussian = GaussianNB()
gaussian.fit(X_train, Y_train)
Y_pred = gaussian.predict(X_test)
acc_gaussian = round(gaussian.score(X_train, Y_train) * 100, 2)
----acc_gaussian =72.280000000000001
# Perceptron
perceptron = Perceptron()
perceptron.fit(X_train, Y_train)
Y_pred = perceptron.predict(X_test)
acc_perceptron = round(perceptron.score(X_train, Y_train) * 100, 2)
----acc_perceptron =78.0
# Linear SVC
linear_svc = LinearSVC()
linear_svc.fit(X_train, Y_train)
Y_pred = linear_svc.predict(X_test)
acc_linear_svc = round(linear_svc.score(X_train, Y_train) * 100, 2)
----acc_linear_svc =79.120000000000005
# Stochastic Gradient Descent
sgd = SGDClassifier()
sgd.fit(X_train, Y_train)
Y_pred = sgd.predict(X_test)
acc_sgd = round(sgd.score(X_train, Y_train) * 100, 2)
acc_sgd =77.670000000000002
# Decision Tree
decision_tree = DecisionTreeClassifier()
decision_tree.fit(X_train, Y_train)
Y_pred = decision_tree.predict(X_test)
acc_decision_tree = round(decision_tree.score(X_train, Y_train) * 100, 2)
----acc_decision_tree =86.760000000000005
# Random Forest
random_forest = RandomForestClassifier(n_estimators=100)
random_forest.fit(X_train, Y_train)
Y_pred = random_forest.predict(X_test)
random_forest.score(X_train, Y_train)
acc_random_forest = round(random_forest.score(X_train, Y_train) * 100, 2)
----acc_random_forest =86.760000000000005
# Model evaluation
models = pd.DataFrame({
'Model': ['Support Vector Machines', 'KNN', 'Logistic Regression',
'Random Forest', 'Naive Bayes', 'Perceptron',
'Stochastic Gradient Decent', 'Linear SVC',
'Decision Tree'],
'Score': [acc_svc, acc_knn, acc_log,
acc_random_forest, acc_gaussian, acc_perceptron,
acc_sgd, acc_linear_svc, acc_decision_tree]})
models.sort_values(by='Score', ascending=False)
# 提交
submission = pd.DataFrame({
"PassengerId": test_df["PassengerId"],
"Survived": Y_pred
})
# submission.to_csv('../output/submission.csv', index=False)
