Predicting house prices in Ames, IA

import pandas as pd
import numpy as np

train = pd.read_csv('train.csv')
test = pd.read_csv('test.csv')

print("Shape of Original DataFrame: {}".format(train.shape))

# dropping the features with around half of value, 
# since we cannot legitimately infer what the missing values might stand for and therefore impute them
train = train.drop(['PoolQC', 'MiscFeature', 'Alley', 'Fence', 'FireplaceQu'], axis=1)

# dropping features of Garage and BSMT group, since they are interchangeable
train = train.drop(['GarageCond', 'GarageFinish', 'GarageQual', 'GarageType', 'GarageYrBlt'], axis=1)
train = train.drop(['BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2', 'BsmtQual'], axis=1)

# dropping the relatively/subjectively unimportant
train = train.drop(['MasVnrArea', 'MasVnrType'], axis=1)
train = train.drop(train.loc[train['Electrical'].isnull()].index)

# LotFrontage: NA most likely means no lot frontage
train.loc[:, "LotFrontage"] = train.loc[:, "LotFrontage"].fillna(0)

print("Shape of DataFrame After Dropping All Rows with Missing Values: {}".format(train.shape))

import matplotlib.pyplot as plt
%matplotlib inline
plt.hist(train.SalePrice, color='gold')
plt.show()

y = np.log(train['SalePrice'].ravel()).copy()

from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LinearRegression, Lasso
from sklearn.model_selection import cross_val_score, cross_val_predict
from sklearn.decomposition import PCA
from sklearn.feature_selection import RFE

# CONVENTIONAL
quality = train[['OverallQual', 'KitchenQual', 'ExterQual', 'LowQualFinSF', 'HeatingQC']]
footage = train[['TotalBsmtSF', 'GrLivArea', 'GarageArea', 'PoolArea', 'LotArea']]
neighborhood = train[['MSSubClass', 'LandContour', 'Neighborhood', 'BldgType']]
age = train[['YearBuilt', 'YearRemodAdd', 'YrSold']]
property_attr = train[['Foundation', 'Exterior1st', 'Utilities']]
rooms = train[['FullBath', 'BedroomAbvGr', 'Fireplaces', 'GarageCars']]
conventional = pd.get_dummies(pd.concat([quality, footage, neighborhood, age, property_attr, rooms], axis=1))

# ALL
allfeatures = pd.get_dummies(train[[col for col in train.columns if col !='SalePrice']].copy(), drop_first=True) 

# CORRELATED
top24 = ss.fit_transform(pd.get_dummies(train[['OverallQual', 'GrLivArea', 'TotalBsmtSF', 'GarageCars', 'GarageArea', 
                    '1stFlrSF', 'FullBath', 'TotRmsAbvGrd', 'YearBuilt', 'YearRemodAdd', 
                    'Fireplaces', 'BsmtFinSF1', 'LotFrontage', 'SaleCondition', 'WoodDeckSF', 
                    '2ndFlrSF', 'OpenPorchSF', 'HalfBath', 'LotArea', 'BsmtFullBath', 
                    'BsmtUnfSF', 'BedroomAbvGr', 'ScreenPorch', 'PoolArea']]))

ss = StandardScaler()
allfeatures_scaled = ss.fit_transform(allfeatures)
conventional_scaled = ss.fit_transform(conventional)
top24_scaled = ss.fit_transform(top24)

# PCA
pca = PCA(n_components=24)
pca_features = pca.fit_transform(allfeatures_scaled)

# RFE
rfe = RFE(lr, n_features_to_select=48) 
rfe.fit(allfeatures_scaled, y)
rfe_features = allfeatures.iloc[:, rfe.support_]

# Lasso
from sklearn.feature_selection import SelectFromModel
lasso = Lasso()
lasso.fit(allfeatures, y)
lasso_features = SelectFromModel(lasso, prefit=True).transform(allfeatures_scaled)
print("%s variables of %s were removed"%(sum(lasso.coef_ == 0), len(lasso.coef_)),'\n')

# To see columns chosen by Lasso
for col, coef in zip(allfeatures.columns, lasso.coef_):
        if coef != 0.0:
            print('Use', col, coef)

# Baseline model predicting the mean
from sklearn.metrics import r2_score
train['baseline'] = y.mean()
print('Score of predicting the mean:', r2_score(train['SalePrice'], train['baseline']))

# Just the Linear Regression model
lr = LinearRegression()
print('Score of LR using all features:', cross_val_score(lr, allfeatures_scaled, y, cv=5).mean())
print('Score of LR using conventional features:', cross_val_score(lr, conventional_scaled, y, cv=5).mean())
print('Score LR via .corr:', cross_val_score(lr, top24_scaled, y, cv=5).mean())

print('Score via PCA:', cross_val_score(lr, pca_features, y, cv=5).mean())

print('Score via Lasso:', cross_val_score(lr, lasso_features, y, cv=5).mean())

print('Score via RFE:', cross_val_score(lr, rfe_features, y, cv=5).mean())

from sklearn.preprocessing import Imputer
from sklearn.pipeline import Pipeline

X = allfeatures

imputing = ('imputing', Imputer(missing_values=0, strategy='mean', axis=0))
scaling = ('scaling', StandardScaler())
rfe = ('rfe', RFE(lr, n_features_to_select=24, step=5))
pca = ('pca', PCA(n_components=24))
lrp = ('lrp', LinearRegression())
lasso = ('lasso', Lasso())

lr_pipeline = Pipeline([imputing, scaling, lrp])
lr_pipeline.fit(conventional, y)

rfe_pipeline = Pipeline([imputing, scaling, rfe]) # Create the pipeline: pipeline
rfe_pipeline.fit(X, y) # Fit the pipeline to the train set
         
pca_pipeline = Pipeline([imputing, scaling, pca, lrp])
pca_pipeline.fit(X, y)

lasso_pipeline = Pipeline([imputing, scaling, lasso]) # Create the pipeline: pipeline
lasso_features = SelectFromModel(Lasso().fit(X, y), prefit=True).transform(X)

print('LR score, conv:', cross_val_score(lr_pipeline, conventional, y, cv=5).mean())
print('LR score, top24:', cross_val_score(lr_pipeline, top24, y, cv=5).mean())
print('PCA score:', cross_val_score(pca_pipeline, X, y, cv=5).mean())
print('RFE score:', cross_val_score(rfe_pipeline, X, y).mean())
print('Lasso score:', cross_val_score(lr, lasso_features, y, cv=5).mean())