In [4]:

    

library(data.table)
library(xgboost)
library(Metrics)
library(Matrix)
library(mice)
library(dplyr)
library('ggplot2')
library('ggthemes')
library('scales')
library(bit64)  # may be needed for fread() for bit64::integer64 type properly displayed
library(moments)  # imports skewness
library(caret)
library(gridExtra)
library(cowplot)
library(lattice)
# library(Hmisc)  # not working
# library(Formula)
# library(VIM)

HousePrices <- setClass(
              # Set name of class
              "HousePrices",
              
              # Define slots
              slots = c(
                        df = "data.frame",
                        df_test = "data.frame",
                        # df_train_test_merged = "numeric"
                        numerical_feature_names = "character",
                        non_numerical_feature_names = "character",
                        feature_names_num = "character",
                        is_one_hot_encoder = "logical",
                        is_with_log1p_SalePrice = "logical"
                        
                        ),
              
              # Set default values
              prototype=list(
                    df = fread('/home/mizio/Documents/Kaggle/HousePrices/train.csv', showProgress = T, data.table = FALSE),
                    # df = read.csv('/home/mizio/Documents/Kaggle/HousePrices/train.csv', stringsAsFactors = T),
                    df_test = fread('/home/mizio/Documents/Kaggle/HousePrices/test.csv', showProgress = T, data.table = FALSE),
                    # df_test = read.csv('/home/mizio/Documents/Kaggle/HousePrices/test.csv', stringsAsFactors = T)
                    # y_train = df$SalePrice
                    numerical_feature_names = c(),
                    non_numerical_feature_names = c(),
                    feature_names_num = c(),
                    is_one_hot_encoder = F,
                    is_with_log1p_SalePrice = F
                    
                    # Merge training and test data together
                    # df_train_test_merged = merge_train_and_test_dataframe(df, df_test)
                    )
              )

setGeneric(name="merge_train_and_test_dataframe",
                                           def=function(theObject, df, df_test)
                                           {
                                                    standardGeneric("merge_train_and_test_dataframe")
                                           }
                                           )

setMethod(f="merge_train_and_test_dataframe",
                                          signature="HousePrices",
                                          definition=function(theObject, df, df_test)
                                          {
                                                    # Remove Id feature and SalePrice (stored in y_train)
                                                    df$Id <- NULL
                                                    df$SalePrice <- NULL
                                                    # Id_column = test_data$Id
                                                    df_test$Id <- NULL
                                                    return(rbind(df, df_test))
                                          }
                                          )

setGeneric(name="drop_variable_before_preparation",
                                         def=function(theObject, df)
                                         {
                                                    standardGeneric("drop_variable_before_preparation")
                                         }
                                         )

setMethod(f="drop_variable_before_preparation",
                                        signature="HousePrices",
                                        definition=function(theObject, df)
                                        {
                                                    # Drop features that have certain procentage of missing values considering the training data and test, 
                                                    # since they will undergo imputation together.
                                                    # print(colSums(is.na(df)))
                                                    number_of_missing_values_in_features <- colSums(is.na(df))
                                                    features_with_many_missing_values <- character(0)
                                                    features_in_df <- names(df)
                                                    for(feature in features_in_df)
                                                      {
                                                      if(number_of_missing_values_in_features[[feature]] >= 0.3*nrow(df))
                                                        {
                                                        features_with_many_missing_values <- append(features_with_many_missing_values, feature)
                                                        }
                                                      }
                                                    print(features_with_many_missing_values)
                                                    df <- df[, !(names(df) %in% features_with_many_missing_values)]
                                                    return(df)
                                        }
                                        )

setGeneric(name="clean_data",
                       def=function(theObject, df)
                       {
                                  standardGeneric("clean_data")
                       }
                       )

setMethod(f="clean_data",
                      signature="HousePrices",
                      definition=function(theObject, df)
                      {
                                  if(sum(is.na(df)) > 0)
                                  {
                                    is_with_MICE <- F
                                    if(is_with_MICE)
                                    {
                                      # Imputation with MICE
                                      set.seed(0)
                                      df <- as.data.frame(df)
                                      df_imputed <- complete(mice(df))  # method='rf'))
                                    } else
                                    {
                                      # Remove all rows with NA values
                                      df_imputed <- df[complete.cases(df),]
                                    }
                                    
                                  }
                                  return(df_imputed)
                      }
                      )

setGeneric(name="numerical_feature_logical",
                                          def=function(theObject, df)
                                          {
                                                      standardGeneric("numerical_feature_logical")
                                          }
                                          )

setMethod(f="numerical_feature_logical",
                                      signature="HousePrices",
                                      definition=function(theObject, df)
                                      {
                                                  # Numeric data types in R: 'numeric', 'integer'
                                                  numerical_features <- data.frame(logical(dim(df)[2]), row.names = names(df))
                                                  for(feature in rownames(numerical_features))
                                                  {
                                                    # if(class(df[, get(feature)]) == "numeric" || class(df[, get(feature)]) == "integer")
                                                    if(class(df[, feature]) == "numeric" || class(df[, feature]) == "integer")
                                                    {
                                                      numerical_features[feature,] = T
                                                    }
                                                  }
                                                  # Feature names with True
                                                  # browser()
                                                  return(numerical_features)
                                      }
                                      )

setGeneric(name="extract_numerical_features",
                                       def=function(theObject, numerical_features)
                                       {
                                                  standardGeneric("extract_numerical_features")
                                       }
                                       )

setMethod(f="extract_numerical_features",
                                      signature="HousePrices",
                                      definition=function(theObject, numerical_features)
                                      {
                                                  mask_index <- which(numerical_features$logical.dim.df..2..)
                                                  return(rownames(numerical_features)[mask_index])
                                      }
                                      )

setGeneric(name="extract_non_numerical_features",
                                             def=function(theObject, numerical_features)
                                             {
                                                        standardGeneric("extract_non_numerical_features")
                                             }
                                             )

setMethod(f="extract_non_numerical_features",
                                          signature="HousePrices",
                                          definition=function(theObject, numerical_features)
                                          {
                                                        mask_index <- which(numerical_features$logical.dim.df..2.. == F)
                                                        return(rownames(numerical_features)[mask_index])
                                          }
                                          )

setGeneric(name="encode_labels_in_numeric_format",
           def=function(theObject, df, feature)
           {
                      standardGeneric("encode_labels_in_numeric_format")
           }
           )
setMethod(f="encode_labels_in_numeric_format",
          signature="HousePrices",
          definition=function(theObject, df, feature)
          {
            # Encode categorical features as integers
            # browser()
            feature_name_num <- paste0(feature, 'Num', collapse='')
            levels <- sort(unique(df[[feature]]))
            df[[feature_name_num]] <- as.integer(factor(df[[feature]], levels=levels))
            return(df)
          }
          )

setGeneric(name="one_hot_encoder",
                             def=function(theObject, df, feature)
                             {
                                        standardGeneric("one_hot_encoder")
                             }
                             )

setMethod(f="one_hot_encoder",
                          signature="HousePrices",
                          definition=function(theObject, df, feature)
                          {
                            levels <- sort(unique(df[[feature]]))
                            levels_num <- lapply(levels, function(x) paste0(feature, x, collapse=''))
                            feature_num <- paste0(feature, 'Num', collapse='')
                            ith <- 1
                            # browser()
                            for(level in levels_num)
                            {
                              df[level] <- as.integer(df[feature_num] == ith)
                              ith <- ith + 1
                            }
                            
                            # df_all_dummy <- acm.disjonctif(df[feature])
                            # df[feature] <- NULL
                            # df <- cbind(df, df_all_dummy)
                            return(df)
                            
                          }
                          )

setGeneric(name="feature_mapping_to_numerical_values",
                                                   def=function(theObject, df)
                                                   {
                                                              standardGeneric("feature_mapping_to_numerical_values")
                                                   }
                                                   )

setMethod(f="feature_mapping_to_numerical_values",
                                              signature="HousePrices",
                                              definition=function(theObject, df)
                                              {
                                                ith <- 1
                                                for(feature in theObject@non_numerical_feature_names)
                                                {
                                                  df <- encode_labels_in_numeric_format(theObject, df, feature)
                                                  
                                                  if(theObject@is_one_hot_encoder)
                                                  {
                                                    df <- one_hot_encoder(theObject, df, feature)
                                                  }
                                                }
                                                return(df)
                                              }
                                              )

setGeneric(name="feature_names_num",
           def=function(theObject, df)
           {
             standardGeneric("feature_names_num")
           }
           )

setMethod(f="feature_names_num",
          signature="HousePrices",
          definition=function(theObject, df)
          {
            feature_names_num <- vector("character", length=length(theObject@non_numerical_feature_names))
            ith <- 1
            for(feature in theObject@non_numerical_feature_names)
            {
              feature_name_num <- paste0(feature, 'Num', collapse='')
              feature_names_num[ith] <- feature_name_num
              ith = ith + 1
            }
            return(feature_names_num)
          }
          )

setGeneric(name="drop_features_num",
           def=function(theObject, df)
           {
             standardGeneric("drop_features_num")
           }
           )

setMethod(f="drop_features_num",
          signature="HousePrices",
          definition=function(theObject, df)
          {
            df <- df[, !(names(df) %in% theObject@feature_names_num)]
            return(df)
          }
          )

setGeneric(name="get_is_one_hot_encoder",
                                     def=function(theObject)
                                     {
                                                standarGeneric("get_is_one_hot_encoder")
                                     }
                                     )

setMethod(f="get_is_one_hot_encoder",
                                  signature="HousePrices",
                                  definition=function(theObject)
                                  {
                                    return(theObject@is_one_hot_encoder)
                                  }
                                  )

setGeneric(name="set_is_one_hot_encoder",
                                     def=function(theObject, is_one_hot_encoder)
                                     {
                                       standarGeneric("set_is_one_hot_encoder")
                                     }
                                     )

setMethod(f="set_is_one_hot_encoder",
                                  signature="HousePrices",
                                  definition=function(theObject, is_one_hot_encoder)
                                  {
                                    theObject@is_one_hot_encoder <- is_one_hot_encoder
                                    return(theObject)
                                  }
                                  )

setGeneric(name="skew_correction",
           def=function(theObject, df)
           {
             standardGeneric("skew_correction")
           }
           )

setMethod(f="skew_correction",
          signature=c("HousePrices", "data.frame"),
          definition=function(theObject, df)
          {
            skewed_feats <- lapply(df, function(x) skewness(x[!is.na(x)]))  # compute skewness
            skewed_feats <- skewed_feats[skewed_feats > 0.75]
            skewed_feats = names(skewed_feats)
            df[, skewed_feats] <- log1p(df[skewed_feats])
            return(df)
          }
          )

setMethod(f="skew_correction",
          signature=c("HousePrices", "integer"),
          definition=function(theObject, df)
          {
            # browser()
            is_skewed_feat <- skewness(df) > 0.75  # compute skewness
            if(is_skewed_feat)
            {
              df <- log1p(df)
            }
            return(df)
          }
          )

setGeneric(name="feature_engineering",
           def=function(theObject, df)
           {
             standardGeneric("feature_engineering")
           }
           )

setMethod(f="feature_engineering",
          signature="HousePrices",
          definition=function(theObject, df)
          {
            is_skewness_correction_for_all_features <-  1
            if(is_skewness_correction_for_all_features)
            {
              # Correcting for skewness
              # Treat all numerical variables that were not one-hot encoded
              numerical_feature_names_of_non_modified_df <- theObject@numerical_feature_names
              if(!(theObject@is_one_hot_encoder))
              {
                numerical_feature_names_of_non_modified_df <- c(theObject@feature_names_num, 
                                                                numerical_feature_names_of_non_modified_df)
              }
              relevant_features <- numerical_feature_names_of_non_modified_df
              df[, relevant_features] <- skew_correction(theObject, df[, relevant_features])
            } else
            {
                # Only scale down sale price and leave other numerical features standardized.
                if(any(names(df) %in% 'SalePrice'))
                {
                  df$SalePrice <- log1p(df$SalePrice)
                }
            }
            return(df)
          }
          )

setGeneric(name="feature_scaling",
           def=function(theObject, df)
           {
             standardGeneric("feature_scaling")
           }
           )

setMethod(f="feature_scaling",
          signature="HousePrices",
          definition=function(theObject, df)
          {
            is_scaling_for_all_features <-  1
            if(is_scaling_for_all_features)
            {
              # Correcting for skewness
              # Treat all numerical variables that were not one-hot encoded
              numerical_feature_names_of_non_modified_df <- theObject@numerical_feature_names
              if(!(theObject@is_one_hot_encoder))
              {
                numerical_feature_names_of_non_modified_df <- c(theObject@feature_names_num, 
                                                                numerical_feature_names_of_non_modified_df)
              }
              relevant_features <- numerical_feature_names_of_non_modified_df
              df[, relevant_features] <- scale(df[, relevant_features])
            } else
            {
              # Only scale down sale price and leave other numerical features standardized.
              if(any(names(df) %in% 'SalePrice'))
              {
                df$SalePrice <- scale(df$SalePrice)
              }
            }
            return(df)
          }
          )

setGeneric(name="plot_histogram",
           def=function(theObject, df, feature_col_index)
           {
             standardGeneric("plot_histogram")
           }
           )

setMethod(f="plot_histogram",
          signature="HousePrices",
          definition=function(theObject, df, feature_col_index)
          {
            return(ggplot(data=df[feature_col_index], aes(x=factor(df[[feature_col_index]]))) + 
                     stat_count() + xlab(colnames(df[feature_col_index])) + theme_light() + theme(axis.text.x=element_text(angle=90, hjust=1)))
          }
          )

setGeneric(name="plot_function",
           def=function(theObject, df, function_to_plot, feature_column_indices, ncol=2)
           {
             standardGeneric("plot_function")
           }
           )

setMethod(f="plot_function",
          signature="HousePrices",
          definition=function(theObject, df, function_to_plot, feature_column_indices, ncol=2)
          {
            feature_plots <- list()
            for(ite in feature_column_indices)
            {
              feature_plot <- function_to_plot(theObject, df=df, feature_col_index=ite)
              feature_plots <- c(feature_plots, list(feature_plot))
            }
            do.call("grid.arrange", c(feature_plots, ncol=ncol))
          }
          )

setGeneric(name="plot_density",
           def=function(theObject, df, feature_col_index, target_column_name)
           {
             standardGeneric("plot_density")
           }
           )

setMethod(f="plot_density",
          signature="HousePrices",
          definition=function(theObject, df, feature_col_index, target_column_name)
          {
            # Todo: check if commented version has error
            # df_to_plot <- data.frame(x=df[[feature_col_index]], SalePrice=df[, target_column_name])
            # feature_plot <- ggplot(data=df_to_plot) + geom_line(aes(x=df[[feature_col_index]]), stat='Sale price density', size=1, alpha=1.0) +
            #   xlab(paste0((colnames(df[[feature_col_index]])), '\n', 'Skewness: ', round(skewness(df[[feature_col_index]], na.rm=T), 2))) +
            #   theme_light()
            feature_plot <- ggplot(data=df_to_plot) + geom_line(aes(x=x), stat='Sale price density', size=1, alpha=1.0) +
              xlab(paste0((colnames(df[[feature_col_index]])), '\n', 'Skewness: ', round(skewness(df[[feature_col_index]], na.rm=T), 2))) +
              theme_light()
            return(feature_plot)
          }
          )

setGeneric(name="prepare_data",
                          def=function(theObject, df)
                          {
                                      standardGeneric("prepare_data")
                          }
                          )

setMethod(f="prepare_data", 
                        signature="HousePrices",
                        definition=function(theObject, df)
                        {
                          df <- drop_variable_before_preparation(theObject, df)
                          # browser()
                          numerical_feature_log <- numerical_feature_logical(theObject, df)
                          theObject@non_numerical_feature_names <- extract_non_numerical_features(theObject, numerical_feature_log)
                          theObject@numerical_feature_names <- extract_numerical_features(theObject, numerical_feature_log)
                          # theObject@is_one_hot_encoder <- T
                          # theObject@is_with_log1p_SalePrice <- T
                          
                          is_not_import_data <- 1
                          if(is_not_import_data)
                          {
                            # browser()
                            theObject@feature_names_num <- feature_names_num(theObject, df)
                            df <- feature_mapping_to_numerical_values(theObject, df)
                            if(theObject@is_one_hot_encoder)
                            {
                              df <- drop_features_num(theObject, df)
                            }
                            df <- feature_engineering(theObject, df)
                            df <- clean_data(theObject, df)
                            df <- feature_scaling(theObject, df)
                            # browser()
                          }
                          return(df)
                        }
                        )

# Todo: implement get and set in encapsulated form for public variables in constructor
# using ex. theObject@class_var (obs. '@' works on instance like '.' in python)

if(T)
  {
  options(error=recover, show.error.locations=TRUE, warn=2)
  
  # Create instance of class
  house_prices <- HousePrices()  # , is_with_log1p_SalePrice=T)
  house_prices@is_one_hot_encoder <- F
  house_prices@is_with_log1p_SalePrice <- T

  # house_prices <- new("HousePrices", is_one_hot_encoder=T)#, is_with_log1p_SalePrice=T)
  is.object(house_prices)
  isS4(house_prices)
  
  # Create object to load data
  df <- slot(house_prices, "df") 
  df_test <- slot(house_prices, "df_test")
  y_train <- df$SalePrice
  
  ## Prepare data
  # Merge training and test data together
  train_test_merged <- merge_train_and_test_dataframe(house_prices, df, df_test)
  # Number of rows in training data for later splitting
  rows_in_train <- nrow(df)
  features_in_train <- names(df)
  train_test_merged_prepared <- prepare_data(house_prices, train_test_merged)
  y_train_prepared <- skew_correction(house_prices, y_train)
  
  # Extracting numerical feature columns
  train_test_merged_prepared_numerical_feature_log <- numerical_feature_logical(house_prices, train_test_merged_prepared)
  train_test_merged_prepared_numerical_features <- extract_numerical_features(house_prices, train_test_merged_prepared_numerical_feature_log)
  # Extracting non numerical feature columns
  train_test_merged_prepared_non_numerical_features <- extract_non_numerical_features(house_prices, train_test_merged_prepared_numerical_feature_log)
  train_test_merged_prepared_only_categorical <- train_test_merged_prepared[, train_test_merged_prepared_non_numerical_features]
  train_test_merged_prepared <- train_test_merged_prepared[, train_test_merged_prepared_numerical_features]

  # Splitting merged data set
  x_train <- train_test_merged_prepared[1:rows_in_train,]
  x_test <- train_test_merged_prepared[(rows_in_train + 1):nrow(train_test_merged_prepared),]
  
  # Casting all types to numeric type (also integer)
  x_train[] <- lapply(x_train, as.numeric)
  x_test[] <- lapply(x_test, as.numeric)
  
  is_explore_data <- T
  if(is_explore_data)
  {
    # Information of df before preperation
    print(sort(sapply(train_test_merged, function(x) sum(is.na(x))), decreasing = TRUE))
    # print(colSums(is.na(train_test_merged)))
    print(summary(train_test_merged))
    
    # Check df after preparation
    print('After preperation =================')
    print(colSums(is.na(train_test_merged_prepared)))
    print(summary(train_test_merged_prepared))
    
    print(cat('Train has', dim(df)[1], 'rows and', dim(df)[2], 'columns.'))
    print(cat('Test has', dim(df_test)[1], 'rows and', dim(df_test)[2], ' columns.'))
    
    # The percentage of data missing in train.
    sum(is.na(df)) / (nrow(df) *ncol(df))
    
    # The percentage of data missing in test.
    sum(is.na(df_test)) / (nrow(df_test) * ncol(df_test))
    
    # Check if any duplicate rows
    cat("The number of row duplicates:", nrow(df) - nrow(unique(df)))
    
    # Check how distributions change after imputation
    # Plot LotFrontage price distributions
    par(mfrow=c(1,2))
    hist(train_test_merged$LotFrontage, freq=F, main='LotFrontage: Original Data', 
         col='darkgreen')
    hist(train_test_merged_prepared$LotFrontage, freq=F, main='LotFrontage: Mice Imputed Data', 
         col='lightgreen')
    
    # Check how distributions change after imputation
    # Plot LotFrontage price distributions
    par(mfrow=c(1,2))
    hist(y_train, freq=F, main='Sale Price: Original Data', 
         col='darkgreen')
    hist(y_train_prepared, freq=F, main='Sale Price: skewness corrected Data', 
         col='lightgreen')
    
    # Plot GarageYrBlt price distributions
    par(mfrow=c(1,2))
    hist(train_test_merged$GarageYrBlt, freq=F, main='GarageYrBlt: Original Data', 
         col='darkgreen')
    hist(train_test_merged_prepared$GarageYrBlt, freq=F, main='GarageYrBlt: Mice Imputed Data', 
         col='lightgreen')
    
    is_clustering <- 0
    if(is_clustering)
    {
      hel =1
      # train_test_merged_prepared_clustered <- varclus(x_train, similarity="spearman", minlev=0.05)
      # train_test_merged_prepared_clustered <- varclus(as.matrix(train_test_merged_prepared), similarity="spearman", minlev=0.05)
      # plot(train_test_merged_prepared_clustered, cex=0.5)
    }
    
    
    is_categorical_feature_plots <- T
    if(is_categorical_feature_plots)
    {
      # Todo: implement categorical feature plots
      
      #Bar plots
      # Categorical features are in total 38
      plot_function(house_prices, df[, train_test_merged_prepared_non_numerical_features], plot_histogram, 1:4, 2)
      plot_function(house_prices, df[, train_test_merged_prepared_non_numerical_features], plot_histogram, 5:8, 2)
      plot_function(house_prices, df[, train_test_merged_prepared_non_numerical_features], plot_histogram, 9:13, 2)
      plot_function(house_prices, df[, train_test_merged_prepared_non_numerical_features], plot_histogram, 14:17, 2)
      plot_function(house_prices, df[, train_test_merged_prepared_non_numerical_features], plot_histogram, 18:21, 2)
    }
    
  }
  
  
  is_test_functions <- F
  if(is_test_functions)
  {
    # Testing functions
    # numerical_feature_log <- numerical_feature_logical(house_prices, df)
    # df_num <- extract_numerical_features(house_prices, numerical_feature_log)
  }
  
  is_make_prediction <- T
  if(is_make_prediction)
  {
    
    is_xgb_cv <- T
    if(is_xgb_cv)
    {
      # --- xgboost ---
      save_path <- '/home/mizio/Documents/Kaggle/HousePrices/submission/'
      dtrain <- xgb.DMatrix(as.matrix(x_train), label=y_train_prepared)
      dtest <- xgb.DMatrix(as.matrix(x_test))
      
      # Saving xgb.DMatrix
      xgb.DMatrix.save(dtrain, paste0(save_path, "dtrain.buffer", collapse=''))
      xgb.DMatrix.save(dtest, paste0(save_path, "dtest.buffer", collapse=''))
      # Loading xgb.DMatrix
      # dtrain2 <- xgb.DMatrix(paste0(save_path, "dtrain.buffer", collapse=''))
      # dtest2 <-  xgb.DMatrix(paste0(save_path, "dtest.buffer", collapse=''))
      
      # Params aggressive and prone to overfitting
      # xgb_params <- list(
      #   seed = 0,
      #   colsample_bytree = 0.8,
      #   silent = 1,
      #   subsample = 0.6,
      #   # learning_rate = 0.01,
      #   eta = 0.06,  # low value means it is more robust to overfitting
      #   # objective = 'reg:linear',
      #   max_depth = 15,
      #   num_parallel_tree = 1,
      #   # alpha = 1,
      #   gamma = 0,
      #   min_child_weight = 0.1,
      #   eval_metric = 'rmse',
      #   booster = 'gbtree'
      #   # booster = 'gblinear'
      # )
      
      # Params conservative
      xgb_params <- list(
        seed = 0,
        colsample_bytree = 0.8,
        silent = 1,
        subsample = 0.6,
        # learning_rate = 0.01,
        eta = 0.02,  # low value means it is more robust to overfitting
        # objective = 'reg:linear',
        max_depth = 15,
        num_parallel_tree = 1,
        # alpha = 1,
        gamma = 0,
        min_child_weight = 1,
        eval_metric = 'rmse',
        booster = 'gbtree'
        # booster = 'gblinear'
        # verbose_eval=10
      )
      
      xgb_cv <- xgb.cv(xgb_params, dtrain, nrounds=1000, nfold=5, stratified=F, early_stopping_rounds=100, verbose=2)
      
      best_nrounds <- xgb_cv$best_ntreelimit
      
      # Measure learning progress while building the model
      set.seed(3410)
      train_partition_index <- createDataPartition(y_train_prepared, p=0.8, list=F, times=1)
      # dtrain_bench = dtrain[1:as.integer(dim(dtrain)[1]*0.8)]
      # dtest_bench = dtrain[(as.integer(dim(dtrain)[1]*0.8) + 1):dim(dtrain)[1]]
      dtrain_bench = dtrain[train_partition_index,]
      test_partition_index <- vector("logical", length=length(y_train_prepared))
      test_partition_index[train_partition_index] <- T
      test_partition_index <- which(!test_partition_index)
      dtest_bench = dtrain[test_partition_index,]
      watchlist <- list(train=dtrain_bench, test=dtest_bench)
      
      gbdt <- xgb.train(xgb_params, dtrain, nrounds=as.integer(best_nrounds))
      # gbdt <- xgb.train(xgb_params, dtrain_bench, watchlist=watchlist, nrounds=as.integer(best_nrounds))
      output_xgb_cv <- predict(gbdt, dtest)
      output <- output_xgb_cv
      
      # Information from xgb.DMatrix
      label = getinfo(dtrain, "label")
      # Only makes sense for binary classification where output is {0, 1}
      # err <- as.numeric(sum(as.integer(output_xgb_cv > 0.5) != label))/length(label)
      # print(paste("test-error=", err))
      
      # Feature importance
      importance_matrix <- xgb.importance(feature_names=colnames(x_train), model=gbdt)
      head(importance_matrix)
      # print(importance_matrix)
      xgb.plot.importance(importance_matrix=importance_matrix)
      # Deeper analysis of features
      # How to do it for a regression problem
      # importance_raw <- xgb.importance(feature_names=colnames(x_train), model=gbdt, data=x_train, label=y_train_prepared)
      # importance_clean <- importance_raw[,`:=`(Cover=NULL, Frequency=NULL)]
      
      # Checking Chi^2 between features. Higher means more correlated
      # chi2 <- chisq.test(x_train$OverallQual, y_train_prepared)
      # print(chi2)
      # chi2 <- chisq.test(x_train$GrLivArea)
      # print(chi2)
      
      # Trees from model
      # xgb.plot.tree(model=gbdt)
    }
    
    # Todo: implement Regularized linear models (lasso, ridge)
    
  }
  
  is_make_submission <- F
  if(is_make_submission && is_make_prediction)
  {
    name <- 'submission.csv'
    submission <- fread(paste0(save_path, name, collapse=''), colClasses=c("integer", "numeric"))
    
    if(is_make_prediction && is_xgb_cv)
    {
      # save model to binary local file
      xgb.save(gbdt, paste0(save_path, "xgboost.model", collapse=''))
      
      # Test to see how identical our saved model is to the original by comparison of two predictions
      # load binary model to R
      # bst2 <- xgb.load(paste0(save_path, "model_to_compare_xgboost.model", collapse=''))
      # pred2 <- predict(bst2, dtest)
      
      # And now the test
      # print(paste("sum(abs(pred2 - pred))=", sum(abs(pred2 - output_xgb_cv))))
    }
    
    if(house_prices@is_with_log1p_SalePrice)
    {
      submission$SalePrice <- expm1(output)
    } else
    {
      submission$SalePrice <- output
    }
    
    # submission$Id <- Id_column
    write.csv(submission, paste0(save_path, name, collapse=''), row.names=F)
    
  }

  }


    

    





'merge_train_and_test_dataframe'






    





'merge_train_and_test_dataframe'






    





'drop_variable_before_preparation'






    





'drop_variable_before_preparation'






    





'clean_data'






    





'clean_data'






    





'numerical_feature_logical'






    





'numerical_feature_logical'






    





'extract_numerical_features'






    





'extract_numerical_features'






    





'extract_non_numerical_features'






    





'extract_non_numerical_features'






    





'encode_labels_in_numeric_format'






    





'encode_labels_in_numeric_format'






    





'one_hot_encoder'






    





'one_hot_encoder'






    





'feature_mapping_to_numerical_values'






    





'feature_mapping_to_numerical_values'






    





'feature_names_num'






    





'feature_names_num'






    





'drop_features_num'






    





'drop_features_num'






    





'get_is_one_hot_encoder'






    





'get_is_one_hot_encoder'






    





'set_is_one_hot_encoder'






    





'set_is_one_hot_encoder'






    





'skew_correction'






    





'skew_correction'






    





'skew_correction'






    





'feature_engineering'






    





'feature_engineering'






    





'feature_scaling'






    





'feature_scaling'






    





'plot_histogram'






    





'plot_histogram'






    





'plot_function'






    





'plot_function'






    





'plot_density'






    





'plot_density'






    





'prepare_data'






    





'prepare_data'






    




[1] "Alley"       "FireplaceQu" "PoolQC"      "Fence"       "MiscFeature"
       PoolQC   MiscFeature         Alley         Fence   FireplaceQu 
         2909          2814          2721          2348          1420 
  LotFrontage   GarageYrBlt  GarageFinish    GarageQual    GarageCond 
          486           159           159           159           159 
   GarageType      BsmtCond  BsmtExposure      BsmtQual  BsmtFinType2 
          157            82            82            81            80 
 BsmtFinType1    MasVnrType    MasVnrArea      MSZoning     Utilities 
           79            24            23             4             2 
 BsmtFullBath  BsmtHalfBath    Functional   Exterior1st   Exterior2nd 
            2             2             2             1             1 
   BsmtFinSF1    BsmtFinSF2     BsmtUnfSF   TotalBsmtSF    Electrical 
            1             1             1             1             1 
  KitchenQual    GarageCars    GarageArea      SaleType    MSSubClass 
            1             1             1             1             0 
      LotArea        Street      LotShape   LandContour     LotConfig 
            0             0             0             0             0 
    LandSlope  Neighborhood    Condition1    Condition2      BldgType 
            0             0             0             0             0 
   HouseStyle   OverallQual   OverallCond     YearBuilt  YearRemodAdd 
            0             0             0             0             0 
    RoofStyle      RoofMatl     ExterQual     ExterCond    Foundation 
            0             0             0             0             0 
      Heating     HeatingQC    CentralAir      1stFlrSF      2ndFlrSF 
            0             0             0             0             0 
 LowQualFinSF     GrLivArea      FullBath      HalfBath  BedroomAbvGr 
            0             0             0             0             0 
 KitchenAbvGr  TotRmsAbvGrd    Fireplaces    PavedDrive    WoodDeckSF 
            0             0             0             0             0 
  OpenPorchSF EnclosedPorch     3SsnPorch   ScreenPorch      PoolArea 
            0             0             0             0             0 
      MiscVal        MoSold        YrSold SaleCondition 
            0             0             0             0 
   MSSubClass       MSZoning          LotFrontage        LotArea      
 Min.   : 20.00   Length:2919        Min.   : 21.00   Min.   :  1300  
 1st Qu.: 20.00   Class :character   1st Qu.: 59.00   1st Qu.:  7478  
 Median : 50.00   Mode  :character   Median : 68.00   Median :  9453  
 Mean   : 57.14                      Mean   : 69.31   Mean   : 10168  
 3rd Qu.: 70.00                      3rd Qu.: 80.00   3rd Qu.: 11570  
 Max.   :190.00                      Max.   :313.00   Max.   :215245  
                                     NA's   :486                      
    Street             Alley             LotShape         LandContour       
 Length:2919        Length:2919        Length:2919        Length:2919       
 Class :character   Class :character   Class :character   Class :character  
 Mode  :character   Mode  :character   Mode  :character   Mode  :character  
                                                                            
                                                                            
                                                                            
                                                                            
  Utilities          LotConfig          LandSlope         Neighborhood      
 Length:2919        Length:2919        Length:2919        Length:2919       
 Class :character   Class :character   Class :character   Class :character  
 Mode  :character   Mode  :character   Mode  :character   Mode  :character  
                                                                            
                                                                            
                                                                            
                                                                            
  Condition1         Condition2          BldgType          HouseStyle       
 Length:2919        Length:2919        Length:2919        Length:2919       
 Class :character   Class :character   Class :character   Class :character  
 Mode  :character   Mode  :character   Mode  :character   Mode  :character  
                                                                            
                                                                            
                                                                            
                                                                            
  OverallQual      OverallCond      YearBuilt     YearRemodAdd 
 Min.   : 1.000   Min.   :1.000   Min.   :1872   Min.   :1950  
 1st Qu.: 5.000   1st Qu.:5.000   1st Qu.:1954   1st Qu.:1965  
 Median : 6.000   Median :5.000   Median :1973   Median :1993  
 Mean   : 6.089   Mean   :5.565   Mean   :1971   Mean   :1984  
 3rd Qu.: 7.000   3rd Qu.:6.000   3rd Qu.:2001   3rd Qu.:2004  
 Max.   :10.000   Max.   :9.000   Max.   :2010   Max.   :2010  
                                                               
  RoofStyle           RoofMatl         Exterior1st        Exterior2nd       
 Length:2919        Length:2919        Length:2919        Length:2919       
 Class :character   Class :character   Class :character   Class :character  
 Mode  :character   Mode  :character   Mode  :character   Mode  :character  
                                                                            
                                                                            
                                                                            
                                                                            
  MasVnrType          MasVnrArea      ExterQual          ExterCond        
 Length:2919        Min.   :   0.0   Length:2919        Length:2919       
 Class :character   1st Qu.:   0.0   Class :character   Class :character  
 Mode  :character   Median :   0.0   Mode  :character   Mode  :character  
                    Mean   : 102.2                                        
                    3rd Qu.: 164.0                                        
                    Max.   :1600.0                                        
                    NA's   :23                                            
  Foundation          BsmtQual           BsmtCond         BsmtExposure      
 Length:2919        Length:2919        Length:2919        Length:2919       
 Class :character   Class :character   Class :character   Class :character  
 Mode  :character   Mode  :character   Mode  :character   Mode  :character  
                                                                            
                                                                            
                                                                            
                                                                            
 BsmtFinType1         BsmtFinSF1     BsmtFinType2         BsmtFinSF2     
 Length:2919        Min.   :   0.0   Length:2919        Min.   :   0.00  
 Class :character   1st Qu.:   0.0   Class :character   1st Qu.:   0.00  
 Mode  :character   Median : 368.5   Mode  :character   Median :   0.00  
                    Mean   : 441.4                      Mean   :  49.58  
                    3rd Qu.: 733.0                      3rd Qu.:   0.00  
                    Max.   :5644.0                      Max.   :1526.00  
                    NA's   :1                           NA's   :1        
   BsmtUnfSF       TotalBsmtSF       Heating           HeatingQC        
 Min.   :   0.0   Min.   :   0.0   Length:2919        Length:2919       
 1st Qu.: 220.0   1st Qu.: 793.0   Class :character   Class :character  
 Median : 467.0   Median : 989.5   Mode  :character   Mode  :character  
 Mean   : 560.8   Mean   :1051.8                                        
 3rd Qu.: 805.5   3rd Qu.:1302.0                                        
 Max.   :2336.0   Max.   :6110.0                                        
 NA's   :1        NA's   :1                                             
  CentralAir         Electrical           1stFlrSF       2ndFlrSF     
 Length:2919        Length:2919        Min.   : 334   Min.   :   0.0  
 Class :character   Class :character   1st Qu.: 876   1st Qu.:   0.0  
 Mode  :character   Mode  :character   Median :1082   Median :   0.0  
                                       Mean   :1160   Mean   : 336.5  
                                       3rd Qu.:1388   3rd Qu.: 704.0  
                                       Max.   :5095   Max.   :2065.0  
                                                                      
  LowQualFinSF        GrLivArea     BsmtFullBath     BsmtHalfBath    
 Min.   :   0.000   Min.   : 334   Min.   :0.0000   Min.   :0.00000  
 1st Qu.:   0.000   1st Qu.:1126   1st Qu.:0.0000   1st Qu.:0.00000  
 Median :   0.000   Median :1444   Median :0.0000   Median :0.00000  
 Mean   :   4.694   Mean   :1501   Mean   :0.4299   Mean   :0.06136  
 3rd Qu.:   0.000   3rd Qu.:1744   3rd Qu.:1.0000   3rd Qu.:0.00000  
 Max.   :1064.000   Max.   :5642   Max.   :3.0000   Max.   :2.00000  
                                   NA's   :2        NA's   :2        
    FullBath        HalfBath       BedroomAbvGr   KitchenAbvGr  
 Min.   :0.000   Min.   :0.0000   Min.   :0.00   Min.   :0.000  
 1st Qu.:1.000   1st Qu.:0.0000   1st Qu.:2.00   1st Qu.:1.000  
 Median :2.000   Median :0.0000   Median :3.00   Median :1.000  
 Mean   :1.568   Mean   :0.3803   Mean   :2.86   Mean   :1.045  
 3rd Qu.:2.000   3rd Qu.:1.0000   3rd Qu.:3.00   3rd Qu.:1.000  
 Max.   :4.000   Max.   :2.0000   Max.   :8.00   Max.   :3.000  
                                                                
 KitchenQual         TotRmsAbvGrd     Functional          Fireplaces    
 Length:2919        Min.   : 2.000   Length:2919        Min.   :0.0000  
 Class :character   1st Qu.: 5.000   Class :character   1st Qu.:0.0000  
 Mode  :character   Median : 6.000   Mode  :character   Median :1.0000  
                    Mean   : 6.452                      Mean   :0.5971  
                    3rd Qu.: 7.000                      3rd Qu.:1.0000  
                    Max.   :15.000                      Max.   :4.0000  
                                                                        
 FireplaceQu         GarageType         GarageYrBlt   GarageFinish      
 Length:2919        Length:2919        Min.   :1895   Length:2919       
 Class :character   Class :character   1st Qu.:1960   Class :character  
 Mode  :character   Mode  :character   Median :1979   Mode  :character  
                                       Mean   :1978                     
                                       3rd Qu.:2002                     
                                       Max.   :2207                     
                                       NA's   :159                      
   GarageCars      GarageArea      GarageQual         GarageCond       
 Min.   :0.000   Min.   :   0.0   Length:2919        Length:2919       
 1st Qu.:1.000   1st Qu.: 320.0   Class :character   Class :character  
 Median :2.000   Median : 480.0   Mode  :character   Mode  :character  
 Mean   :1.767   Mean   : 472.9                                        
 3rd Qu.:2.000   3rd Qu.: 576.0                                        
 Max.   :5.000   Max.   :1488.0                                        
 NA's   :1       NA's   :1                                             
  PavedDrive          WoodDeckSF       OpenPorchSF     EnclosedPorch   
 Length:2919        Min.   :   0.00   Min.   :  0.00   Min.   :   0.0  
 Class :character   1st Qu.:   0.00   1st Qu.:  0.00   1st Qu.:   0.0  
 Mode  :character   Median :   0.00   Median : 26.00   Median :   0.0  
                    Mean   :  93.71   Mean   : 47.49   Mean   :  23.1  
                    3rd Qu.: 168.00   3rd Qu.: 70.00   3rd Qu.:   0.0  
                    Max.   :1424.00   Max.   :742.00   Max.   :1012.0  
                                                                       
   3SsnPorch        ScreenPorch        PoolArea          PoolQC         
 Min.   :  0.000   Min.   :  0.00   Min.   :  0.000   Length:2919       
 1st Qu.:  0.000   1st Qu.:  0.00   1st Qu.:  0.000   Class :character  
 Median :  0.000   Median :  0.00   Median :  0.000   Mode  :character  
 Mean   :  2.602   Mean   : 16.06   Mean   :  2.252                     
 3rd Qu.:  0.000   3rd Qu.:  0.00   3rd Qu.:  0.000                     
 Max.   :508.000   Max.   :576.00   Max.   :800.000                     
                                                                        
    Fence           MiscFeature           MiscVal             MoSold      
 Length:2919        Length:2919        Min.   :    0.00   Min.   : 1.000  
 Class :character   Class :character   1st Qu.:    0.00   1st Qu.: 4.000  
 Mode  :character   Mode  :character   Median :    0.00   Median : 6.000  
                                       Mean   :   50.83   Mean   : 6.213  
                                       3rd Qu.:    0.00   3rd Qu.: 8.000  
                                       Max.   :17000.00   Max.   :12.000  
                                                                          
     YrSold       SaleType         SaleCondition     
 Min.   :2006   Length:2919        Length:2919       
 1st Qu.:2007   Class :character   Class :character  
 Median :2008   Mode  :character   Mode  :character  
 Mean   :2008                                        
 3rd Qu.:2009                                        
 Max.   :2010                                        
                                                     
[1] "After preperation ================="
      MSSubClass      LotFrontage          LotArea      OverallQual 
               0                0                0                0 
     OverallCond        YearBuilt     YearRemodAdd       MasVnrArea 
               0                0                0                0 
      BsmtFinSF1       BsmtFinSF2        BsmtUnfSF      TotalBsmtSF 
               0                0                0                0 
        1stFlrSF         2ndFlrSF     LowQualFinSF        GrLivArea 
               0                0                0                0 
    BsmtFullBath     BsmtHalfBath         FullBath         HalfBath 
               0                0                0                0 
    BedroomAbvGr     KitchenAbvGr     TotRmsAbvGrd       Fireplaces 
               0                0                0                0 
     GarageYrBlt       GarageCars       GarageArea       WoodDeckSF 
               0                0                0                0 
     OpenPorchSF    EnclosedPorch        3SsnPorch      ScreenPorch 
               0                0                0                0 
        PoolArea          MiscVal           MoSold           YrSold 
               0                0                0                0 
     MSZoningNum        StreetNum      LotShapeNum   LandContourNum 
               0                0                0                0 
    UtilitiesNum     LotConfigNum     LandSlopeNum  NeighborhoodNum 
            2202                0                0                0 
   Condition1Num    Condition2Num      BldgTypeNum    HouseStyleNum 
               0                0                0                0 
    RoofStyleNum      RoofMatlNum   Exterior1stNum   Exterior2ndNum 
               0                0                0                0 
   MasVnrTypeNum     ExterQualNum     ExterCondNum    FoundationNum 
               0                0                0                0 
     BsmtQualNum      BsmtCondNum  BsmtExposureNum  BsmtFinType1Num 
               0                0                0                0 
 BsmtFinType2Num       HeatingNum     HeatingQCNum    CentralAirNum 
               0                0                0                0 
   ElectricalNum   KitchenQualNum    FunctionalNum    GarageTypeNum 
               0                0                0                0 
 GarageFinishNum    GarageQualNum    GarageCondNum    PavedDriveNum 
               0                0                0                0 
     SaleTypeNum SaleConditionNum 
               0                0 
   MSSubClass       LotFrontage         LotArea         OverallQual     
 Min.   :-1.1042   Min.   :-3.1881   Min.   :-3.8839   Min.   :-3.0266  
 1st Qu.:-1.1042   1st Qu.:-0.3097   1st Qu.:-0.3365   1st Qu.:-0.8756  
 Median : 0.1783   Median : 0.1529   Median : 0.1430   Median :-0.1586  
 Mean   : 0.0000   Mean   : 0.0000   Mean   : 0.0000   Mean   : 0.0000  
 3rd Qu.: 0.6565   3rd Qu.: 0.5512   3rd Qu.: 0.5283   3rd Qu.: 0.5584  
 Max.   : 2.0869   Max.   : 4.4384   Max.   : 6.5368   Max.   : 2.7094  
                                                                        
  OverallCond        YearBuilt         YearRemodAdd       MasVnrArea     
 Min.   :-4.3463   Min.   :-3.02327   Min.   :-1.6737   Min.   :-0.8139  
 1st Qu.:-0.5575   1st Qu.:-0.62734   1st Qu.:-0.9139   1st Qu.:-0.8139  
 Median :-0.5575   Median : 0.05258   Median : 0.4633   Median :-0.8139  
 Mean   : 0.0000   Mean   : 0.00000   Mean   : 0.0000   Mean   : 0.0000  
 3rd Qu.: 0.3897   3rd Qu.: 0.99152   3rd Qu.: 0.9381   3rd Qu.: 1.1347  
 Max.   : 3.2313   Max.   : 1.21816   Max.   : 1.1756   Max.   : 1.9714  
                                                                         
   BsmtFinSF1        BsmtFinSF2        BsmtUnfSF        TotalBsmtSF      
 Min.   :-1.4616   Min.   :-0.3659   Min.   :-3.6808   Min.   :-6.00199  
 1st Qu.:-1.4616   1st Qu.:-0.3659   1st Qu.:-0.1912   1st Qu.:-0.60468  
 Median : 0.5534   Median :-0.3659   Median : 0.2364   Median :-0.02616  
 Mean   : 0.0000   Mean   : 0.0000   Mean   : 0.0000   Mean   : 0.00000  
 3rd Qu.: 0.7812   3rd Qu.:-0.3659   3rd Qu.: 0.5575   3rd Qu.: 0.72031  
 Max.   : 1.4739   Max.   : 3.4941   Max.   : 1.2100   Max.   : 4.75055  
                                                                         
    1stFlrSF           2ndFlrSF        LowQualFinSF       GrLivArea       
 Min.   :-3.06491   Min.   :-0.8791   Min.   :-0.1063   Min.   :-3.95935  
 1st Qu.:-0.71194   1st Qu.:-0.8791   1st Qu.:-0.1063   1st Qu.:-0.71026  
 Median :-0.05328   Median :-0.8791   Median :-0.1063   Median : 0.03983  
 Mean   : 0.00000   Mean   : 0.0000   Mean   : 0.0000   Mean   : 0.00000  
 3rd Qu.: 0.71081   3rd Qu.: 1.1193   3rd Qu.:-0.1063   3rd Qu.: 0.60649  
 Max.   : 4.69883   Max.   : 1.4476   Max.   :11.4551   Max.   : 4.31190  
                                                                          
  BsmtFullBath      BsmtHalfBath        FullBath          HalfBath      
 Min.   :-0.8372   Min.   :-0.2508   Min.   :-2.8828   Min.   :-0.7632  
 1st Qu.:-0.8372   1st Qu.:-0.2508   1st Qu.:-1.0455   1st Qu.:-0.7632  
 Median :-0.8372   Median :-0.2508   Median : 0.7918   Median :-0.7632  
 Mean   : 0.0000   Mean   : 0.0000   Mean   : 0.0000   Mean   : 0.0000  
 3rd Qu.: 1.0892   3rd Qu.:-0.2508   3rd Qu.: 0.7918   3rd Qu.: 1.2471  
 Max.   : 3.0156   Max.   : 6.3712   Max.   : 4.4665   Max.   : 3.2574  
                                                                        
  BedroomAbvGr      KitchenAbvGr      TotRmsAbvGrd       Fireplaces     
 Min.   :-3.6777   Min.   :-0.1709   Min.   :-3.0010   Min.   :-0.9367  
 1st Qu.:-1.0883   1st Qu.:-0.1709   1st Qu.:-0.9841   1st Qu.:-0.9367  
 Median : 0.2064   Median :-0.1709   Median :-0.2173   Median : 0.6295  
 Mean   : 0.0000   Mean   : 0.0000   Mean   : 0.0000   Mean   : 0.0000  
 3rd Qu.: 0.2064   3rd Qu.:-0.1709   3rd Qu.: 0.4469   3rd Qu.: 0.6295  
 Max.   : 4.0905   Max.   : 9.9272   Max.   : 3.8948   Max.   : 5.3280  
                                                                        
  GarageYrBlt         GarageCars       GarageArea        WoodDeckSF     
 Min.   :-3.13993   Min.   :-1.285   Min.   :-2.0559   Min.   :-0.9641  
 1st Qu.:-0.73040   1st Qu.:-1.285   1st Qu.:-0.7701   1st Qu.:-0.9641  
 Median : 0.07278   Median : 0.205   Median :-0.0828   Median :-0.9641  
 Mean   : 0.00000   Mean   : 0.000   Mean   : 0.0000   Mean   : 0.0000  
 3rd Qu.: 0.95245   3rd Qu.: 0.205   3rd Qu.: 0.4593   3rd Qu.: 1.0173  
 Max.   : 1.22017   Max.   : 4.675   Max.   : 5.0761   Max.   : 1.6507  
                                                                        
  OpenPorchSF      EnclosedPorch       3SsnPorch        ScreenPorch     
 Min.   :-1.1116   Min.   :-0.4245   Min.   :-0.1107   Min.   :-0.3203  
 1st Qu.:-1.1116   1st Qu.:-0.4245   1st Qu.:-0.1107   1st Qu.:-0.3203  
 Median : 0.4606   Median :-0.4245   Median :-0.1107   Median :-0.3203  
 Mean   : 0.0000   Mean   : 0.0000   Mean   : 0.0000   Mean   : 0.0000  
 3rd Qu.: 0.8787   3rd Qu.:-0.4245   3rd Qu.:-0.1107   3rd Qu.:-0.3203  
 Max.   : 1.8520   Max.   : 3.5027   Max.   :10.7311   Max.   : 3.9089  
                                                                        
    PoolArea           MiscVal            MoSold             YrSold       
 Min.   :-0.07063   Min.   :-0.1792   Min.   :-1.91609   Min.   :-1.3431  
 1st Qu.:-0.07063   1st Qu.:-0.1792   1st Qu.:-0.81532   1st Qu.:-0.5883  
 Median :-0.07063   Median :-0.1792   Median :-0.08148   Median : 0.1666  
 Mean   : 0.00000   Mean   : 0.0000   Mean   : 0.00000   Mean   : 0.0000  
 3rd Qu.:-0.07063   3rd Qu.:-0.1792   3rd Qu.: 0.65236   3rd Qu.: 0.9215  
 Max.   :15.46611   Max.   : 8.1388   Max.   : 2.12004   Max.   : 1.6763  
                                                                          
  MSZoningNum         StreetNum          LotShapeNum      LandContourNum   
 Min.   :-4.63871   Min.   :-17.70393   Min.   :-1.5628   Min.   :-3.9448  
 1st Qu.:-0.05756   1st Qu.:  0.05646   1st Qu.:-1.5628   1st Qu.: 0.3147  
 Median :-0.05756   Median :  0.05646   Median : 0.6574   Median : 0.3147  
 Mean   : 0.00000   Mean   :  0.00000   Mean   : 0.0000   Mean   : 0.0000  
 3rd Qu.:-0.05756   3rd Qu.:  0.05646   3rd Qu.: 0.6574   3rd Qu.: 0.3147  
 Max.   : 1.46949   Max.   :  0.05646   Max.   : 0.6574   Max.   : 0.3147  
                                                                           
  UtilitiesNum   LotConfigNum      LandSlopeNum     NeighborhoodNum   
 Min.   : NA    Min.   :-2.0191   Min.   :-0.2151   Min.   :-2.12902  
 1st Qu.: NA    1st Qu.: 0.5424   1st Qu.:-0.2151   1st Qu.:-0.92796  
 Median : NA    Median : 0.5424   Median :-0.2151   Median :-0.07005  
 Mean   :NaN    Mean   : 0.0000   Mean   : 0.0000   Mean   : 0.00000  
 3rd Qu.: NA    3rd Qu.: 0.5424   3rd Qu.:-0.2151   3rd Qu.: 0.78786  
 Max.   : NA    Max.   : 0.5424   Max.   : 7.5528   Max.   : 1.98892  
 NA's   :2202                                                         
 Condition1Num      Condition2Num        BldgTypeNum     HouseStyleNum    
 Min.   :-3.53906   Min.   :-15.13893   Min.   :-0.420   Min.   :-1.5712  
 1st Qu.: 0.05007   1st Qu.:  0.01385   1st Qu.:-0.420   1st Qu.:-0.5204  
 Median : 0.05007   Median :  0.01385   Median :-0.420   Median :-0.5204  
 Mean   : 0.00000   Mean   :  0.00000   Mean   : 0.000   Mean   : 0.0000  
 3rd Qu.: 0.05007   3rd Qu.:  0.01385   3rd Qu.:-0.420   3rd Qu.: 1.0558  
 Max.   : 4.79464   Max.   : 17.74147   Max.   : 2.711   Max.   : 2.1067  
                                                                          
  RoofStyleNum      RoofMatlNum       Exterior1stNum    Exterior2ndNum   
 Min.   :-2.4342   Min.   :-4.32551   Min.   :-3.0509   Min.   :-2.9080  
 1st Qu.:-0.5032   1st Qu.:-0.09846   1st Qu.:-0.5306   1st Qu.:-0.6646  
 Median :-0.5032   Median :-0.09846   Median : 0.7295   Median : 0.7375  
 Mean   : 0.0000   Mean   : 0.00000   Mean   : 0.0000   Mean   : 0.0000  
 3rd Qu.:-0.5032   3rd Qu.:-0.09846   3rd Qu.: 0.7295   3rd Qu.: 0.7375  
 Max.   : 3.5318   Max.   :11.35480   Max.   : 1.3596   Max.   : 1.2983  
                                                                         
 MasVnrTypeNum      ExterQualNum      ExterCondNum     FoundationNum    
 Min.   :-2.8858   Min.   :-3.4109   Min.   :-5.0216   Min.   :-1.9376  
 1st Qu.:-1.2763   1st Qu.:-0.6752   1st Qu.: 0.3638   1st Qu.:-0.5262  
 Median : 0.3333   Median : 0.6926   Median : 0.3638   Median :-0.5262  
 Mean   : 0.0000   Mean   : 0.0000   Mean   : 0.0000   Mean   : 0.0000  
 3rd Qu.: 0.3333   3rd Qu.: 0.6926   3rd Qu.: 0.3638   3rd Qu.: 0.8852  
 Max.   : 1.9429   Max.   : 0.6926   Max.   : 0.3638   Max.   : 5.1194  
                                                                        
  BsmtQualNum       BsmtCondNum      BsmtExposureNum   BsmtFinType1Num  
 Min.   :-2.3864   Min.   :-4.1343   Min.   :-2.0130   Min.   :-1.5493  
 1st Qu.:-0.2261   1st Qu.: 0.2886   1st Qu.:-0.2474   1st Qu.:-0.4404  
 Median :-0.2261   Median : 0.2886   Median : 0.6354   Median :-0.4404  
 Mean   : 0.0000   Mean   : 0.0000   Mean   : 0.0000   Mean   : 0.0000  
 3rd Qu.: 0.8540   3rd Qu.: 0.2886   3rd Qu.: 0.6354   3rd Qu.: 1.2230  
 Max.   : 0.8540   Max.   : 0.2886   Max.   : 0.6354   Max.   : 1.2230  
                                                                        
 BsmtFinType2Num    HeatingNum       HeatingQCNum     CentralAirNum    
 Min.   :-4.714   Min.   :-0.1047   Min.   :-0.8372   Min.   :-4.4457  
 1st Qu.: 0.320   1st Qu.:-0.1047   1st Qu.:-0.8372   1st Qu.: 0.2248  
 Median : 0.320   Median :-0.1047   Median :-0.8372   Median : 0.2248  
 Mean   : 0.000   Mean   : 0.0000   Mean   : 0.0000   Mean   : 0.0000  
 3rd Qu.: 0.320   3rd Qu.:-0.1047   3rd Qu.: 1.4824   3rd Qu.: 0.2248  
 Max.   : 0.320   Max.   :19.0762   Max.   : 1.4824   Max.   : 0.2248  
                                                                       
 ElectricalNum     KitchenQualNum    FunctionalNum     GarageTypeNum    
 Min.   :-3.6187   Min.   :-2.6621   Min.   :-6.5421   Min.   :-1.2910  
 1st Qu.: 0.2898   1st Qu.:-0.3533   1st Qu.: 0.2402   1st Qu.:-0.7423  
 Median : 0.2898   Median :-0.3533   Median : 0.2402   Median :-0.7423  
 Mean   : 0.0000   Mean   : 0.0000   Mean   : 0.0000   Mean   : 0.0000  
 3rd Qu.: 0.2898   3rd Qu.: 0.8010   3rd Qu.: 0.2402   3rd Qu.: 1.4525  
 Max.   : 0.2898   Max.   : 0.8010   Max.   : 0.2402   Max.   : 1.4525  
                                                                        
 GarageFinishNum   GarageQualNum     GarageCondNum     PavedDriveNum    
 Min.   :-1.4551   Min.   :-5.5261   Min.   :-7.4932   Min.   :-4.0008  
 1st Qu.:-1.4551   1st Qu.: 0.2533   1st Qu.: 0.1884   1st Qu.: 0.2758  
 Median :-0.2349   Median : 0.2533   Median : 0.1884   Median : 0.2758  
 Mean   : 0.0000   Mean   : 0.0000   Mean   : 0.0000   Mean   : 0.0000  
 3rd Qu.: 0.9852   3rd Qu.: 0.2533   3rd Qu.: 0.1884   3rd Qu.: 0.2758  
 Max.   : 0.9852   Max.   : 0.2533   Max.   : 0.1884   Max.   : 0.2758  
                                                                        
  SaleTypeNum      SaleConditionNum 
 Min.   :-4.6924   Min.   :-3.6552  
 1st Qu.: 0.3366   1st Qu.: 0.1666  
 Median : 0.3366   Median : 0.1666  
 Mean   : 0.0000   Mean   : 0.0000  
 3rd Qu.: 0.3366   3rd Qu.: 0.1666  
 Max.   : 0.3366   Max.   : 1.1221  
                                    
Train has 1460 rows and 81 columns.NULL
Test has 1459 rows and 80  columns.NULL
The number of row duplicates: 0





    













    













    













    













    













    













    













    




[1]	train-rmse:11.300943+0.006079	test-rmse:11.300918+0.025091 
Multiple eval metrics are present. Will use test_rmse for early stopping.
Will train until test_rmse hasn't improved in 100 rounds.

[2]	train-rmse:11.075554+0.006044	test-rmse:11.075528+0.025147 
[3]	train-rmse:10.854649+0.005855	test-rmse:10.854622+0.025356 
[4]	train-rmse:10.638141+0.005787	test-rmse:10.638114+0.025454 
[5]	train-rmse:10.426013+0.005756	test-rmse:10.425985+0.025502 
[6]	train-rmse:10.218048+0.005602	test-rmse:10.218019+0.025682 
[7]	train-rmse:10.014337+0.005571	test-rmse:10.014308+0.025752 
[8]	train-rmse:9.814757+0.005422	test-rmse:9.814727+0.025904 
[9]	train-rmse:9.619023+0.005297	test-rmse:9.618992+0.026054 
[10]	train-rmse:9.427322+0.005092	test-rmse:9.427291+0.026278 
[11]	train-rmse:9.239488+0.005065	test-rmse:9.239456+0.026326 
[12]	train-rmse:9.055399+0.005104	test-rmse:9.055367+0.026309 
[13]	train-rmse:8.874821+0.005181	test-rmse:8.874788+0.026253 
[14]	train-rmse:8.697896+0.005257	test-rmse:8.697862+0.026207 
[15]	train-rmse:8.524774+0.005086	test-rmse:8.524739+0.026405 
[16]	train-rmse:8.354870+0.004839	test-rmse:8.354834+0.026667 
[17]	train-rmse:8.188256+0.004704	test-rmse:8.188219+0.026830 
[18]	train-rmse:8.024926+0.004732	test-rmse:8.024889+0.026820 
[19]	train-rmse:7.865139+0.004815	test-rmse:7.865101+0.026771 
[20]	train-rmse:7.708553+0.004774	test-rmse:7.708514+0.026830 
[21]	train-rmse:7.554947+0.004607	test-rmse:7.554907+0.027043 
[22]	train-rmse:7.404533+0.004456	test-rmse:7.404492+0.027205 
[23]	train-rmse:7.257022+0.004131	test-rmse:7.256980+0.027551 
[24]	train-rmse:7.112614+0.004089	test-rmse:7.112571+0.027625 
[25]	train-rmse:6.970994+0.003907	test-rmse:6.970950+0.027828 
[26]	train-rmse:6.832322+0.003786	test-rmse:6.832277+0.027980 
[27]	train-rmse:6.696400+0.003679	test-rmse:6.696354+0.028112 
[28]	train-rmse:6.562988+0.003550	test-rmse:6.562941+0.028261 
[29]	train-rmse:6.432531+0.003511	test-rmse:6.432483+0.028332 
[30]	train-rmse:6.304679+0.003561	test-rmse:6.304630+0.028312 
[31]	train-rmse:6.179193+0.003557	test-rmse:6.179142+0.028349 
[32]	train-rmse:6.056293+0.003565	test-rmse:6.056242+0.028378 
[33]	train-rmse:5.935892+0.003600	test-rmse:5.935840+0.028377 
[34]	train-rmse:5.817896+0.003437	test-rmse:5.817842+0.028572 
[35]	train-rmse:5.702334+0.003396	test-rmse:5.702279+0.028652 
[36]	train-rmse:5.589129+0.003253	test-rmse:5.589073+0.028826 
[37]	train-rmse:5.478026+0.003278	test-rmse:5.477969+0.028836 
[38]	train-rmse:5.369189+0.003088	test-rmse:5.369130+0.029061 
[39]	train-rmse:5.262688+0.002918	test-rmse:5.262627+0.029265 
[40]	train-rmse:5.158170+0.003020	test-rmse:5.158108+0.029206 
[41]	train-rmse:5.055983+0.002864	test-rmse:5.055921+0.029402 
[42]	train-rmse:4.955539+0.002891	test-rmse:4.955474+0.029417 
[43]	train-rmse:4.857266+0.002789	test-rmse:4.857200+0.029563 
[44]	train-rmse:4.761071+0.002784	test-rmse:4.761003+0.029613 
[45]	train-rmse:4.666656+0.002709	test-rmse:4.666586+0.029752 
[46]	train-rmse:4.574139+0.002636	test-rmse:4.574068+0.029872 
[47]	train-rmse:4.483544+0.002839	test-rmse:4.483472+0.029725 
[48]	train-rmse:4.394718+0.002867	test-rmse:4.394643+0.029740 
[49]	train-rmse:4.307704+0.002886	test-rmse:4.307628+0.029757 
[50]	train-rmse:4.222409+0.002684	test-rmse:4.222331+0.030023 
[51]	train-rmse:4.138836+0.002648	test-rmse:4.138756+0.030108 
[52]	train-rmse:4.056786+0.002637	test-rmse:4.056704+0.030142 
[53]	train-rmse:3.976594+0.002601	test-rmse:3.976511+0.030248 
[54]	train-rmse:3.898089+0.002570	test-rmse:3.898003+0.030305 
[55]	train-rmse:3.821072+0.002483	test-rmse:3.820984+0.030441 
[56]	train-rmse:3.745658+0.002217	test-rmse:3.745568+0.030734 
[57]	train-rmse:3.671576+0.002130	test-rmse:3.671484+0.030885 
[58]	train-rmse:3.599085+0.002001	test-rmse:3.598990+0.031066 
[59]	train-rmse:3.528146+0.001843	test-rmse:3.528049+0.031265 
[60]	train-rmse:3.458476+0.001934	test-rmse:3.458377+0.031237 
[61]	train-rmse:3.390209+0.002050	test-rmse:3.390108+0.031175 
[62]	train-rmse:3.323400+0.002023	test-rmse:3.323296+0.031281 
[63]	train-rmse:3.257993+0.001894	test-rmse:3.257887+0.031430 
[64]	train-rmse:3.193950+0.001819	test-rmse:3.193841+0.031561 
[65]	train-rmse:3.131126+0.001763	test-rmse:3.131014+0.031712 
[66]	train-rmse:3.069510+0.001763	test-rmse:3.069395+0.031802 
[67]	train-rmse:3.009252+0.001722	test-rmse:3.009135+0.031928 
[68]	train-rmse:2.950232+0.001628	test-rmse:2.950112+0.032091 
[69]	train-rmse:2.892468+0.001647	test-rmse:2.892345+0.032168 
[70]	train-rmse:2.836002+0.001613	test-rmse:2.835876+0.032286 
[71]	train-rmse:2.780507+0.001682	test-rmse:2.780377+0.032255 
[72]	train-rmse:2.726065+0.001802	test-rmse:2.725932+0.032239 
[73]	train-rmse:2.672865+0.001849	test-rmse:2.672729+0.032293 
[74]	train-rmse:2.620700+0.001810	test-rmse:2.620561+0.032444 
[75]	train-rmse:2.569625+0.001837	test-rmse:2.569483+0.032532 
[76]	train-rmse:2.519520+0.001781	test-rmse:2.519374+0.032600 
[77]	train-rmse:2.470549+0.001784	test-rmse:2.470399+0.032622 
[78]	train-rmse:2.422512+0.001574	test-rmse:2.422358+0.032829 
[79]	train-rmse:2.375516+0.001560	test-rmse:2.375397+0.032911 
[80]	train-rmse:2.329287+0.001616	test-rmse:2.329164+0.032873 
[81]	train-rmse:2.284167+0.001718	test-rmse:2.284123+0.032776 
[82]	train-rmse:2.239998+0.001727	test-rmse:2.239975+0.032796 
[83]	train-rmse:2.196755+0.001781	test-rmse:2.196728+0.032805 
[84]	train-rmse:2.154454+0.001850	test-rmse:2.154408+0.032840 
[85]	train-rmse:2.112959+0.001785	test-rmse:2.112909+0.032986 
[86]	train-rmse:2.072233+0.001810	test-rmse:2.072202+0.033036 
[87]	train-rmse:2.032498+0.001826	test-rmse:2.032461+0.033117 
[88]	train-rmse:1.993451+0.001890	test-rmse:1.993445+0.033097 
[89]	train-rmse:1.955211+0.001958	test-rmse:1.955199+0.033137 
[90]	train-rmse:1.917831+0.002009	test-rmse:1.917848+0.033103 
[91]	train-rmse:1.881279+0.002057	test-rmse:1.881318+0.033090 
[92]	train-rmse:1.845352+0.002094	test-rmse:1.845418+0.033137 
[93]	train-rmse:1.810311+0.001994	test-rmse:1.810426+0.033357 
[94]	train-rmse:1.775978+0.001864	test-rmse:1.776120+0.033504 
[95]	train-rmse:1.742407+0.001917	test-rmse:1.742533+0.033459 
[96]	train-rmse:1.709396+0.001949	test-rmse:1.709520+0.033444 
[97]	train-rmse:1.677032+0.002127	test-rmse:1.677181+0.033339 
[98]	train-rmse:1.645333+0.002176	test-rmse:1.645497+0.033305 
[99]	train-rmse:1.614599+0.002065	test-rmse:1.614831+0.033433 
[100]	train-rmse:1.584361+0.002083	test-rmse:1.584589+0.033476 
[101]	train-rmse:1.554796+0.002164	test-rmse:1.555056+0.033460 
[102]	train-rmse:1.525838+0.002108	test-rmse:1.526204+0.033616 
[103]	train-rmse:1.497419+0.002170	test-rmse:1.497776+0.033676 
[104]	train-rmse:1.469729+0.002278	test-rmse:1.470120+0.033619 
[105]	train-rmse:1.442575+0.002336	test-rmse:1.442988+0.033675 
[106]	train-rmse:1.416204+0.002402	test-rmse:1.416698+0.033598 
[107]	train-rmse:1.390127+0.002502	test-rmse:1.390903+0.033595 
[108]	train-rmse:1.364614+0.002548	test-rmse:1.365453+0.033678 
[109]	train-rmse:1.339689+0.002476	test-rmse:1.340718+0.033833 
[110]	train-rmse:1.315321+0.002518	test-rmse:1.316412+0.033901 
[111]	train-rmse:1.291561+0.002560	test-rmse:1.292729+0.033845 
[112]	train-rmse:1.268290+0.002574	test-rmse:1.269691+0.033856 
[113]	train-rmse:1.245536+0.002617	test-rmse:1.247141+0.033830 
[114]	train-rmse:1.223322+0.002587	test-rmse:1.225190+0.034091 
[115]	train-rmse:1.201359+0.002577	test-rmse:1.203339+0.034144 
[116]	train-rmse:1.180115+0.002474	test-rmse:1.182222+0.034431 
[117]	train-rmse:1.159258+0.002273	test-rmse:1.161628+0.034664 
[118]	train-rmse:1.138810+0.002187	test-rmse:1.141322+0.034881 
[119]	train-rmse:1.118938+0.002219	test-rmse:1.121730+0.035127 
[120]	train-rmse:1.099339+0.002298	test-rmse:1.102478+0.035222 
[121]	train-rmse:1.080183+0.002409	test-rmse:1.083619+0.035236 
[122]	train-rmse:1.061643+0.002432	test-rmse:1.065322+0.035273 
[123]	train-rmse:1.043398+0.002444	test-rmse:1.047546+0.035472 
[124]	train-rmse:1.025695+0.002443	test-rmse:1.030394+0.035326 
[125]	train-rmse:1.008311+0.002398	test-rmse:1.013573+0.035483 
[126]	train-rmse:0.991191+0.002569	test-rmse:0.996938+0.035401 
[127]	train-rmse:0.974476+0.002775	test-rmse:0.980698+0.035503 
[128]	train-rmse:0.958181+0.002789	test-rmse:0.965159+0.035505 
[129]	train-rmse:0.942229+0.002850	test-rmse:0.949760+0.035631 
[130]	train-rmse:0.926523+0.002989	test-rmse:0.934829+0.035500 
[131]	train-rmse:0.911159+0.002997	test-rmse:0.920443+0.035540 
[132]	train-rmse:0.896266+0.002943	test-rmse:0.906551+0.035541 
[133]	train-rmse:0.881586+0.002989	test-rmse:0.892678+0.035402 
[134]	train-rmse:0.867132+0.003072	test-rmse:0.879240+0.035544 
[135]	train-rmse:0.853123+0.003109	test-rmse:0.866414+0.035536 
[136]	train-rmse:0.839114+0.003113	test-rmse:0.853352+0.035744 
[137]	train-rmse:0.825516+0.003037	test-rmse:0.840840+0.036194 
[138]	train-rmse:0.812274+0.003165	test-rmse:0.828327+0.036260 
[139]	train-rmse:0.799440+0.003404	test-rmse:0.816682+0.036354 
[140]	train-rmse:0.786781+0.003410	test-rmse:0.805477+0.036482 
[141]	train-rmse:0.774433+0.003364	test-rmse:0.794541+0.036843 
[142]	train-rmse:0.762430+0.003431	test-rmse:0.784053+0.036839 
[143]	train-rmse:0.750515+0.003362	test-rmse:0.773464+0.036902 
[144]	train-rmse:0.738892+0.003375	test-rmse:0.763524+0.037364 
[145]	train-rmse:0.727469+0.003250	test-rmse:0.753408+0.037502 
[146]	train-rmse:0.716312+0.003245	test-rmse:0.743795+0.038067 
[147]	train-rmse:0.705260+0.003135	test-rmse:0.734247+0.038383 
[148]	train-rmse:0.694454+0.003225	test-rmse:0.724953+0.038133 
[149]	train-rmse:0.683760+0.003276	test-rmse:0.715646+0.038403 
[150]	train-rmse:0.673188+0.003287	test-rmse:0.707126+0.038147 
[151]	train-rmse:0.663192+0.003227	test-rmse:0.698870+0.038457 
[152]	train-rmse:0.653335+0.003471	test-rmse:0.690778+0.038227 
[153]	train-rmse:0.643645+0.003428	test-rmse:0.682791+0.038458 
[154]	train-rmse:0.633919+0.003356	test-rmse:0.674787+0.038560 
[155]	train-rmse:0.624715+0.003400	test-rmse:0.667403+0.038434 
[156]	train-rmse:0.615555+0.003427	test-rmse:0.660119+0.038247 
[157]	train-rmse:0.606579+0.003384	test-rmse:0.652845+0.038345 
[158]	train-rmse:0.597656+0.003530	test-rmse:0.645790+0.038436 
[159]	train-rmse:0.589011+0.003500	test-rmse:0.638878+0.038296 
[160]	train-rmse:0.580491+0.003597	test-rmse:0.632208+0.038322 
[161]	train-rmse:0.572111+0.003432	test-rmse:0.625654+0.038307 
[162]	train-rmse:0.563905+0.003415	test-rmse:0.619404+0.038338 
[163]	train-rmse:0.555835+0.003393	test-rmse:0.613508+0.038205 
[164]	train-rmse:0.547843+0.003351	test-rmse:0.607516+0.038118 
[165]	train-rmse:0.540075+0.003515	test-rmse:0.601750+0.037993 
[166]	train-rmse:0.532634+0.003488	test-rmse:0.596176+0.038100 
[167]	train-rmse:0.525219+0.003521	test-rmse:0.590689+0.038171 
[168]	train-rmse:0.517883+0.003582	test-rmse:0.585329+0.038385 
[169]	train-rmse:0.510728+0.003702	test-rmse:0.580304+0.038444 
[170]	train-rmse:0.503628+0.003649	test-rmse:0.575135+0.038365 
[171]	train-rmse:0.496817+0.003677	test-rmse:0.570303+0.038329 
[172]	train-rmse:0.489917+0.003788	test-rmse:0.565387+0.038314 
[173]	train-rmse:0.483324+0.003782	test-rmse:0.560602+0.038244 
[174]	train-rmse:0.476749+0.003850	test-rmse:0.556118+0.038302 
[175]	train-rmse:0.470351+0.003987	test-rmse:0.551861+0.038336 
[176]	train-rmse:0.463955+0.003943	test-rmse:0.547293+0.038554 
[177]	train-rmse:0.457713+0.004070	test-rmse:0.543565+0.038525 
[178]	train-rmse:0.451502+0.004107	test-rmse:0.539469+0.038392 
[179]	train-rmse:0.445484+0.004184	test-rmse:0.535680+0.038420 
[180]	train-rmse:0.439689+0.004274	test-rmse:0.532035+0.038373 
[181]	train-rmse:0.434054+0.004478	test-rmse:0.528344+0.038147 
[182]	train-rmse:0.428475+0.004443	test-rmse:0.524913+0.038367 
[183]	train-rmse:0.422918+0.004425	test-rmse:0.521373+0.038430 
[184]	train-rmse:0.417593+0.004358	test-rmse:0.518200+0.038397 
[185]	train-rmse:0.412173+0.004431	test-rmse:0.514952+0.038362 
[186]	train-rmse:0.406751+0.004425	test-rmse:0.511826+0.038151 
[187]	train-rmse:0.401374+0.004485	test-rmse:0.508979+0.037999 
[188]	train-rmse:0.396167+0.004453	test-rmse:0.506059+0.038113 
[189]	train-rmse:0.391127+0.004386	test-rmse:0.503384+0.038114 
[190]	train-rmse:0.386258+0.004403	test-rmse:0.500524+0.037986 
[191]	train-rmse:0.381375+0.004478	test-rmse:0.497873+0.038111 
[192]	train-rmse:0.376711+0.004511	test-rmse:0.495423+0.038126 
[193]	train-rmse:0.372121+0.004519	test-rmse:0.492928+0.038081 
[194]	train-rmse:0.367571+0.004391	test-rmse:0.490698+0.038146 
[195]	train-rmse:0.362957+0.004309	test-rmse:0.488230+0.037932 
[196]	train-rmse:0.358534+0.004319	test-rmse:0.485887+0.037968 
[197]	train-rmse:0.354222+0.004311	test-rmse:0.483538+0.037928 
[198]	train-rmse:0.349833+0.004360	test-rmse:0.481623+0.038042 
[199]	train-rmse:0.345672+0.004403	test-rmse:0.479596+0.037790 
[200]	train-rmse:0.341564+0.004487	test-rmse:0.477679+0.037847 
[201]	train-rmse:0.337371+0.004498	test-rmse:0.475539+0.037867 
[202]	train-rmse:0.333406+0.004685	test-rmse:0.473666+0.037643 
[203]	train-rmse:0.329423+0.004687	test-rmse:0.471775+0.037583 
[204]	train-rmse:0.325530+0.004615	test-rmse:0.469785+0.037593 
[205]	train-rmse:0.321812+0.004755	test-rmse:0.468246+0.037583 
[206]	train-rmse:0.318126+0.004786	test-rmse:0.466685+0.037634 
[207]	train-rmse:0.314369+0.004808	test-rmse:0.465217+0.037379 
[208]	train-rmse:0.310641+0.004793	test-rmse:0.463586+0.037520 
[209]	train-rmse:0.306995+0.004786	test-rmse:0.462101+0.037672 
[210]	train-rmse:0.303365+0.004728	test-rmse:0.460654+0.037577 
[211]	train-rmse:0.300060+0.004728	test-rmse:0.459189+0.037503 
[212]	train-rmse:0.296724+0.004750	test-rmse:0.457668+0.037576 
[213]	train-rmse:0.293352+0.004698	test-rmse:0.456402+0.037345 
[214]	train-rmse:0.289987+0.004677	test-rmse:0.455025+0.037356 
[215]	train-rmse:0.286697+0.004627	test-rmse:0.453837+0.037403 
[216]	train-rmse:0.283524+0.004606	test-rmse:0.452553+0.037240 
[217]	train-rmse:0.280350+0.004623	test-rmse:0.451505+0.037100 
[218]	train-rmse:0.277228+0.004573	test-rmse:0.450296+0.036967 
[219]	train-rmse:0.274083+0.004520	test-rmse:0.449201+0.037045 
[220]	train-rmse:0.270978+0.004408	test-rmse:0.448193+0.036907 
[221]	train-rmse:0.267920+0.004337	test-rmse:0.447074+0.036924 
[222]	train-rmse:0.264939+0.004406	test-rmse:0.446159+0.036732 
[223]	train-rmse:0.262070+0.004303	test-rmse:0.445296+0.036586 
[224]	train-rmse:0.259268+0.004169	test-rmse:0.444232+0.036516 
[225]	train-rmse:0.256715+0.004209	test-rmse:0.443244+0.036447 
[226]	train-rmse:0.253896+0.004143	test-rmse:0.442426+0.036397 
[227]	train-rmse:0.251211+0.004138	test-rmse:0.441477+0.036408 
[228]	train-rmse:0.248624+0.004143	test-rmse:0.440715+0.036236 
[229]	train-rmse:0.245986+0.004094	test-rmse:0.440024+0.036146 
[230]	train-rmse:0.243423+0.004191	test-rmse:0.439237+0.036206 
[231]	train-rmse:0.240915+0.004093	test-rmse:0.438588+0.036119 
[232]	train-rmse:0.238423+0.004065	test-rmse:0.437877+0.036064 
[233]	train-rmse:0.236132+0.003988	test-rmse:0.437231+0.035906 
[234]	train-rmse:0.233873+0.004021	test-rmse:0.436642+0.035794 
[235]	train-rmse:0.231408+0.004077	test-rmse:0.436067+0.035776 
[236]	train-rmse:0.229067+0.004002	test-rmse:0.435530+0.035758 
[237]	train-rmse:0.226795+0.004041	test-rmse:0.434889+0.035639 
[238]	train-rmse:0.224569+0.004019	test-rmse:0.434282+0.035582 
[239]	train-rmse:0.222309+0.004140	test-rmse:0.433704+0.035499 
[240]	train-rmse:0.220123+0.004097	test-rmse:0.433113+0.035455 
[241]	train-rmse:0.218002+0.004112	test-rmse:0.432489+0.035371 
[242]	train-rmse:0.215829+0.004135	test-rmse:0.431974+0.035269 
[243]	train-rmse:0.213686+0.004136	test-rmse:0.431528+0.035252 
[244]	train-rmse:0.211673+0.004131	test-rmse:0.430954+0.035120 
[245]	train-rmse:0.209539+0.004168	test-rmse:0.430559+0.035027 
[246]	train-rmse:0.207474+0.004085	test-rmse:0.430204+0.034971 
[247]	train-rmse:0.205429+0.004051	test-rmse:0.429792+0.034903 
[248]	train-rmse:0.203444+0.003949	test-rmse:0.429431+0.034847 
[249]	train-rmse:0.201442+0.003983	test-rmse:0.429058+0.034704 
[250]	train-rmse:0.199597+0.003966	test-rmse:0.428636+0.034540 
[251]	train-rmse:0.197811+0.004075	test-rmse:0.428328+0.034477 
[252]	train-rmse:0.195928+0.003967	test-rmse:0.427958+0.034444 
[253]	train-rmse:0.194084+0.004031	test-rmse:0.427668+0.034441 
[254]	train-rmse:0.192235+0.004041	test-rmse:0.427195+0.034410 
[255]	train-rmse:0.190453+0.004037	test-rmse:0.426957+0.034362 
[256]	train-rmse:0.188603+0.004088	test-rmse:0.426624+0.034304 
[257]	train-rmse:0.186833+0.004048	test-rmse:0.426316+0.034324 
[258]	train-rmse:0.185107+0.003937	test-rmse:0.426040+0.034183 
[259]	train-rmse:0.183472+0.004063	test-rmse:0.425826+0.034187 
[260]	train-rmse:0.181862+0.004044	test-rmse:0.425533+0.034126 
[261]	train-rmse:0.180251+0.003991	test-rmse:0.425208+0.034114 
[262]	train-rmse:0.178759+0.003990	test-rmse:0.425012+0.033990 
[263]	train-rmse:0.177122+0.003937	test-rmse:0.424758+0.033954 
[264]	train-rmse:0.175457+0.003876	test-rmse:0.424532+0.033972 
[265]	train-rmse:0.173840+0.003861	test-rmse:0.424302+0.033938 
[266]	train-rmse:0.172355+0.003806	test-rmse:0.424122+0.033915 
[267]	train-rmse:0.170879+0.003835	test-rmse:0.423913+0.033875 
[268]	train-rmse:0.169336+0.003793	test-rmse:0.423706+0.033811 
[269]	train-rmse:0.167798+0.003777	test-rmse:0.423515+0.033855 
[270]	train-rmse:0.166205+0.003741	test-rmse:0.423207+0.033839 
[271]	train-rmse:0.164710+0.003670	test-rmse:0.423005+0.033783 
[272]	train-rmse:0.163249+0.003626	test-rmse:0.422926+0.033783 
[273]	train-rmse:0.161777+0.003573	test-rmse:0.422808+0.033767 
[274]	train-rmse:0.160458+0.003517	test-rmse:0.422537+0.033699 
[275]	train-rmse:0.159083+0.003500	test-rmse:0.422291+0.033634 
[276]	train-rmse:0.157627+0.003437	test-rmse:0.422153+0.033738 
[277]	train-rmse:0.156291+0.003395	test-rmse:0.421999+0.033671 
[278]	train-rmse:0.154864+0.003456	test-rmse:0.421941+0.033580 
[279]	train-rmse:0.153513+0.003445	test-rmse:0.421874+0.033612 
[280]	train-rmse:0.152210+0.003435	test-rmse:0.421791+0.033564 
[281]	train-rmse:0.150939+0.003369	test-rmse:0.421698+0.033607 
[282]	train-rmse:0.149738+0.003386	test-rmse:0.421573+0.033522 
[283]	train-rmse:0.148470+0.003399	test-rmse:0.421487+0.033486 
[284]	train-rmse:0.147130+0.003438	test-rmse:0.421280+0.033404 
[285]	train-rmse:0.145926+0.003393	test-rmse:0.421205+0.033368 
[286]	train-rmse:0.144681+0.003435	test-rmse:0.421111+0.033331 
[287]	train-rmse:0.143504+0.003361	test-rmse:0.420985+0.033289 
[288]	train-rmse:0.142318+0.003222	test-rmse:0.420905+0.033277 
[289]	train-rmse:0.141144+0.003171	test-rmse:0.420825+0.033156 
[290]	train-rmse:0.140030+0.003062	test-rmse:0.420716+0.033027 
[291]	train-rmse:0.138937+0.003082	test-rmse:0.420629+0.032929 
[292]	train-rmse:0.137799+0.003032	test-rmse:0.420481+0.032873 
[293]	train-rmse:0.136785+0.003115	test-rmse:0.420396+0.032748 
[294]	train-rmse:0.135627+0.003040	test-rmse:0.420337+0.032753 
[295]	train-rmse:0.134658+0.003076	test-rmse:0.420228+0.032708 
[296]	train-rmse:0.133607+0.003077	test-rmse:0.420159+0.032639 
[297]	train-rmse:0.132537+0.003023	test-rmse:0.420064+0.032610 
[298]	train-rmse:0.131608+0.002970	test-rmse:0.419943+0.032609 
[299]	train-rmse:0.130603+0.002942	test-rmse:0.419967+0.032466 
[300]	train-rmse:0.129594+0.003023	test-rmse:0.419909+0.032428 
[301]	train-rmse:0.128559+0.003057	test-rmse:0.419846+0.032442 
[302]	train-rmse:0.127552+0.002966	test-rmse:0.419676+0.032432 
[303]	train-rmse:0.126574+0.002970	test-rmse:0.419585+0.032376 
[304]	train-rmse:0.125594+0.002963	test-rmse:0.419591+0.032305 
[305]	train-rmse:0.124671+0.002936	test-rmse:0.419551+0.032291 
[306]	train-rmse:0.123672+0.002887	test-rmse:0.419551+0.032231 
[307]	train-rmse:0.122687+0.002877	test-rmse:0.419509+0.032168 
[308]	train-rmse:0.121661+0.002842	test-rmse:0.419465+0.032100 
[309]	train-rmse:0.120710+0.002865	test-rmse:0.419502+0.032087 
[310]	train-rmse:0.119779+0.002877	test-rmse:0.419441+0.032153 
[311]	train-rmse:0.118776+0.002877	test-rmse:0.419477+0.032083 
[312]	train-rmse:0.117845+0.002825	test-rmse:0.419413+0.032047 
[313]	train-rmse:0.117023+0.002746	test-rmse:0.419382+0.031975 
[314]	train-rmse:0.116151+0.002683	test-rmse:0.419375+0.031957 
[315]	train-rmse:0.115268+0.002655	test-rmse:0.419331+0.031947 
[316]	train-rmse:0.114348+0.002605	test-rmse:0.419239+0.031937 
[317]	train-rmse:0.113483+0.002633	test-rmse:0.419273+0.031932 
[318]	train-rmse:0.112628+0.002654	test-rmse:0.419174+0.031890 
[319]	train-rmse:0.111723+0.002652	test-rmse:0.419150+0.031845 
[320]	train-rmse:0.110900+0.002579	test-rmse:0.419205+0.031826 
[321]	train-rmse:0.110044+0.002541	test-rmse:0.419201+0.031810 
[322]	train-rmse:0.109218+0.002607	test-rmse:0.419309+0.031829 
[323]	train-rmse:0.108434+0.002606	test-rmse:0.419326+0.031849 
[324]	train-rmse:0.107617+0.002546	test-rmse:0.419343+0.031831 
[325]	train-rmse:0.106789+0.002530	test-rmse:0.419391+0.031777 
[326]	train-rmse:0.106003+0.002532	test-rmse:0.419386+0.031733 
[327]	train-rmse:0.105208+0.002406	test-rmse:0.419425+0.031655 
[328]	train-rmse:0.104411+0.002405	test-rmse:0.419372+0.031667 
[329]	train-rmse:0.103647+0.002441	test-rmse:0.419345+0.031637 
[330]	train-rmse:0.102907+0.002416	test-rmse:0.419402+0.031616 
[331]	train-rmse:0.102141+0.002356	test-rmse:0.419384+0.031610 
[332]	train-rmse:0.101442+0.002306	test-rmse:0.419409+0.031605 
[333]	train-rmse:0.100716+0.002344	test-rmse:0.419411+0.031582 
[334]	train-rmse:0.099918+0.002317	test-rmse:0.419408+0.031575 
[335]	train-rmse:0.099197+0.002260	test-rmse:0.419378+0.031582 
[336]	train-rmse:0.098494+0.002210	test-rmse:0.419336+0.031580 
[337]	train-rmse:0.097831+0.002176	test-rmse:0.419394+0.031580 
[338]	train-rmse:0.097129+0.002176	test-rmse:0.419381+0.031556 
[339]	train-rmse:0.096422+0.002126	test-rmse:0.419411+0.031462 
[340]	train-rmse:0.095728+0.002017	test-rmse:0.419489+0.031397 
[341]	train-rmse:0.095035+0.002019	test-rmse:0.419509+0.031415 
[342]	train-rmse:0.094357+0.001994	test-rmse:0.419547+0.031342 
[343]	train-rmse:0.093688+0.001946	test-rmse:0.419546+0.031364 
[344]	train-rmse:0.093039+0.001915	test-rmse:0.419557+0.031370 
[345]	train-rmse:0.092390+0.001860	test-rmse:0.419534+0.031338 
[346]	train-rmse:0.091727+0.001879	test-rmse:0.419529+0.031334 
[347]	train-rmse:0.091048+0.001922	test-rmse:0.419551+0.031336 
[348]	train-rmse:0.090351+0.001909	test-rmse:0.419579+0.031300 
[349]	train-rmse:0.089724+0.001902	test-rmse:0.419586+0.031368 
[350]	train-rmse:0.089080+0.001846	test-rmse:0.419625+0.031363 
[351]	train-rmse:0.088510+0.001904	test-rmse:0.419645+0.031360 
[352]	train-rmse:0.087915+0.001936	test-rmse:0.419654+0.031316 
[353]	train-rmse:0.087226+0.001916	test-rmse:0.419744+0.031328 
[354]	train-rmse:0.086606+0.001872	test-rmse:0.419721+0.031295 
[355]	train-rmse:0.085999+0.001827	test-rmse:0.419757+0.031287 
[356]	train-rmse:0.085380+0.001845	test-rmse:0.419711+0.031337 
[357]	train-rmse:0.084811+0.001739	test-rmse:0.419698+0.031322 
[358]	train-rmse:0.084212+0.001717	test-rmse:0.419695+0.031317 
[359]	train-rmse:0.083619+0.001744	test-rmse:0.419710+0.031346 
[360]	train-rmse:0.083021+0.001755	test-rmse:0.419729+0.031328 
[361]	train-rmse:0.082436+0.001714	test-rmse:0.419699+0.031316 
[362]	train-rmse:0.081838+0.001706	test-rmse:0.419732+0.031295 
[363]	train-rmse:0.081302+0.001766	test-rmse:0.419713+0.031304 
[364]	train-rmse:0.080731+0.001749	test-rmse:0.419745+0.031287 
[365]	train-rmse:0.080138+0.001719	test-rmse:0.419772+0.031258 
[366]	train-rmse:0.079571+0.001663	test-rmse:0.419797+0.031274 
[367]	train-rmse:0.079000+0.001647	test-rmse:0.419811+0.031296 
[368]	train-rmse:0.078493+0.001644	test-rmse:0.419820+0.031277 
[369]	train-rmse:0.077974+0.001590	test-rmse:0.419841+0.031284 
[370]	train-rmse:0.077394+0.001584	test-rmse:0.419862+0.031280 
[371]	train-rmse:0.076832+0.001586	test-rmse:0.419898+0.031289 
[372]	train-rmse:0.076328+0.001539	test-rmse:0.419937+0.031321 
[373]	train-rmse:0.075789+0.001506	test-rmse:0.419971+0.031279 
[374]	train-rmse:0.075249+0.001560	test-rmse:0.419991+0.031259 
[375]	train-rmse:0.074756+0.001510	test-rmse:0.420020+0.031272 
[376]	train-rmse:0.074256+0.001525	test-rmse:0.420072+0.031266 
[377]	train-rmse:0.073787+0.001464	test-rmse:0.420088+0.031268 
[378]	train-rmse:0.073283+0.001389	test-rmse:0.420109+0.031312 
[379]	train-rmse:0.072861+0.001401	test-rmse:0.420159+0.031302 
[380]	train-rmse:0.072382+0.001367	test-rmse:0.420146+0.031255 
[381]	train-rmse:0.071870+0.001381	test-rmse:0.420132+0.031260 
[382]	train-rmse:0.071376+0.001345	test-rmse:0.420159+0.031265 
[383]	train-rmse:0.070884+0.001308	test-rmse:0.420179+0.031217 
[384]	train-rmse:0.070421+0.001340	test-rmse:0.420168+0.031183 
[385]	train-rmse:0.069927+0.001374	test-rmse:0.420145+0.031190 
[386]	train-rmse:0.069476+0.001336	test-rmse:0.420129+0.031197 
[387]	train-rmse:0.069012+0.001376	test-rmse:0.420142+0.031204 
[388]	train-rmse:0.068580+0.001361	test-rmse:0.420148+0.031154 
[389]	train-rmse:0.068128+0.001383	test-rmse:0.420181+0.031156 
[390]	train-rmse:0.067647+0.001362	test-rmse:0.420220+0.031153 
[391]	train-rmse:0.067237+0.001337	test-rmse:0.420240+0.031156 
[392]	train-rmse:0.066825+0.001333	test-rmse:0.420261+0.031126 
[393]	train-rmse:0.066400+0.001302	test-rmse:0.420307+0.031125 
[394]	train-rmse:0.065972+0.001293	test-rmse:0.420339+0.031112 
[395]	train-rmse:0.065548+0.001297	test-rmse:0.420343+0.031146 
[396]	train-rmse:0.065073+0.001303	test-rmse:0.420385+0.031125 
[397]	train-rmse:0.064603+0.001300	test-rmse:0.420383+0.031061 
[398]	train-rmse:0.064162+0.001303	test-rmse:0.420397+0.031055 
[399]	train-rmse:0.063742+0.001323	test-rmse:0.420456+0.031062 
[400]	train-rmse:0.063290+0.001303	test-rmse:0.420436+0.031059 
[401]	train-rmse:0.062888+0.001269	test-rmse:0.420454+0.031069 
[402]	train-rmse:0.062473+0.001244	test-rmse:0.420488+0.031092 
[403]	train-rmse:0.062050+0.001240	test-rmse:0.420515+0.031104 
[404]	train-rmse:0.061643+0.001238	test-rmse:0.420553+0.031085 
[405]	train-rmse:0.061265+0.001302	test-rmse:0.420570+0.031052 
[406]	train-rmse:0.060833+0.001241	test-rmse:0.420612+0.031025 
[407]	train-rmse:0.060440+0.001237	test-rmse:0.420637+0.031003 
[408]	train-rmse:0.060040+0.001186	test-rmse:0.420665+0.030987 
[409]	train-rmse:0.059664+0.001163	test-rmse:0.420692+0.030922 
[410]	train-rmse:0.059281+0.001166	test-rmse:0.420709+0.030944 
[411]	train-rmse:0.058952+0.001207	test-rmse:0.420724+0.030942 
[412]	train-rmse:0.058573+0.001227	test-rmse:0.420776+0.030948 
[413]	train-rmse:0.058196+0.001211	test-rmse:0.420763+0.030899 
[414]	train-rmse:0.057847+0.001173	test-rmse:0.420794+0.030935 
[415]	train-rmse:0.057456+0.001156	test-rmse:0.420827+0.030936 
[416]	train-rmse:0.057072+0.001144	test-rmse:0.420822+0.030900 
[417]	train-rmse:0.056686+0.001129	test-rmse:0.420838+0.030915 
[418]	train-rmse:0.056304+0.001158	test-rmse:0.420851+0.030906 
[419]	train-rmse:0.056002+0.001175	test-rmse:0.420878+0.030902 
Stopping. Best iteration:
[319]	train-rmse:0.111723+0.002652	test-rmse:0.419150+0.031845