def __init__(self, methodname='linear regression', trainingpart=0.9, ):
     """
     类初始化函数。
         :param self: 类变量本身
         :param method='linear regression': model类别，可以为'linear regression', 'svc', 'neural netword'
         :param trainingPart=0.9: 训练集占整体的比例，默认为0.9
     """
     if trainingpart <= 0 or trainingpart >=1:
         raise Exception("Training Part is belong to (0, 1)")
     # 设置model
     if methodname == 'linear regression':
         self.model = LinearRegression()
     elif methodname == 'svc':
         self.model = SVC()
         print 'Warning: your\'s y data\'s type need to be int!'
     elif methodname == 'neural netword':
         self.model = MLPClassifier()
         print 'Warning: your\'s y data\'s type need to be int!'
     else:
         methodname = 'linear regression'
         self.model = LinearRegression()
     # 设置其他属性
     self.trainingpart = trainingpart
     self.methodname = methodname
     self.X = None
     self.y = None
     self.train_X = None
     self.test_X = None
     self.train_y = None
     self.test_y = None

def test_partial_fit_classes_error():
    # Tests that passing different classes to partial_fit raises an error
    X = [[3, 2]]
    y = [0]
    clf = MLPClassifier(solver='sgd')
    clf.partial_fit(X, y, classes=[0, 1])
    assert_raises(ValueError, clf.partial_fit, X, y, classes=[1, 2])

def init_Q():
    # make some dummy training set
    board = init_board()
    board_vec = board2vec(board)
    X = np.array([board_vec])
    y = [(BOARD_SIZE-1)**2]
    board_vec = np.invert(board_vec)
    X = np.append(X,np.array([board_vec]),axis=0)
    y.append(0)
    
    edges = get_potential_moves(board) # all the edges, since the board is empty
    for edge in edges:
        i = edge2ind(edge)
        board_vec[i] = False
        X = np.append(X,np.array([board_vec]),axis=0)
        y.append(check_surrounding_squares(board,edge,0))
        board_vec[i] = True       
    
    
        
    Q = MLPClassifier(warm_start=True, 
                      hidden_layer_sizes=(BOARD_SIZE,10*BOARD_SIZE,BOARD_SIZE), 
                      tol = 1e-10,
                      )
    # Q = DecisionTreeRegressor()
                     
    #    shf = range(len(y))
    #    for j in xrange(100):
    #        random.shuffle(shf)
    #        Xshf = [X[i] for i in shf]
    #        yshf = [y[i] for i in shf]
    triedy = range((BOARD_SIZE-1)**2+1)
    Q.partial_fit(np.repeat(X,100,axis=0),np.repeat(y,100,axis=0),classes=triedy)
    print(Q.predict(X))
    return(Q)

def test_gradient():
    # Test gradient.

    # This makes sure that the activation functions and their derivatives
    # are correct. The numerical and analytical computation of the gradient
    # should be close.
    for n_labels in [2, 3]:
        n_samples = 5
        n_features = 10
        X = np.random.random((n_samples, n_features))
        y = 1 + np.mod(np.arange(n_samples) + 1, n_labels)
        Y = LabelBinarizer().fit_transform(y)

        for activation in ACTIVATION_TYPES:
            mlp = MLPClassifier(activation=activation, hidden_layer_sizes=10,
                                solver='lbfgs', alpha=1e-5,
                                learning_rate_init=0.2, max_iter=1,
                                random_state=1)
            mlp.fit(X, y)

            theta = np.hstack([l.ravel() for l in mlp.coefs_ +
                               mlp.intercepts_])

            layer_units = ([X.shape[1]] + [mlp.hidden_layer_sizes] +
                           [mlp.n_outputs_])

            activations = []
            deltas = []
            coef_grads = []
            intercept_grads = []

            activations.append(X)
            for i in range(mlp.n_layers_ - 1):
                activations.append(np.empty((X.shape[0],
                                             layer_units[i + 1])))
                deltas.append(np.empty((X.shape[0],
                                        layer_units[i + 1])))

                fan_in = layer_units[i]
                fan_out = layer_units[i + 1]
                coef_grads.append(np.empty((fan_in, fan_out)))
                intercept_grads.append(np.empty(fan_out))

            # analytically compute the gradients
            def loss_grad_fun(t):
                return mlp._loss_grad_lbfgs(t, X, Y, activations, deltas,
                                            coef_grads, intercept_grads)

            [value, grad] = loss_grad_fun(theta)
            numgrad = np.zeros(np.size(theta))
            n = np.size(theta, 0)
            E = np.eye(n)
            epsilon = 1e-5
            # numerically compute the gradients
            for i in range(n):
                dtheta = E[:, i] * epsilon
                numgrad[i] = ((loss_grad_fun(theta + dtheta)[0] -
                              loss_grad_fun(theta - dtheta)[0]) /
                              (epsilon * 2.0))
            assert_almost_equal(numgrad, grad)

def main():
    enc = OneHotEncoder(n_values=[7,7,7,7,7,7])
    conn = sqlite3.connect('server.db')
    cursor = conn.cursor()
    all_ = pandas.read_sql_query('SELECT layers.burger, labels.output, layers.layer0, layers.layer1, layers.layer2, layers.layer3, layers.layer4, layers.layer5 FROM layers,labels WHERE layers.burger = labels.burger', conn, index_col='burger')
    
    X = all_.drop(['output'], axis=1)
    y = all_['output']

    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5)

               
    clf = MLPClassifier(solver='adam',  activation='relu',
                        verbose=False,
                        max_iter=10000,
                        tol=1e-9,
                        random_state=1)
    
    X_train_categoricals = X_train[column_names]
    tX_train_categoricals = enc.fit_transform(X_train_categoricals)
    clf.fit(tX_train_categoricals, y_train.as_matrix().astype(int))

    
    X_test_categoricals = X_test[column_names]
    tX_test_categoricals = enc.fit_transform(X_test_categoricals)
    prediction = clf.predict(tX_test_categoricals)
    
    print(classification_report(y_test, prediction))
    
    print_eval(y_test, prediction)

def train_on_source(X,Y):

    print "Start Learning Net on source"

    clf = MLPClassifier( algorithm = 'l-bfgs',
            alpha = 1e-5,
            hidden_layer_sizes = (500,2),
            random_state = 1,
            warm_start = 1,
            max_iter = 400)

    clf.fit(X,Y)
    #new_loss = 0
    #old_loss = 10000
    #for step in range(200):
    #    clf.fit(X,Y)
    #    new_loss = clf.loss_
    #    # stop training, if improvement is small
    #    improvement = abs(new_loss - old_loss)
    #    print "Step:", step, "Loss:", new_loss, "Improvement:", improvement
    #    if improvement < 1.e-5:
    #        print "Training converged!"
    #        break
    #    old_loss = new_loss
    print "Pretrained CLF on Source with num_iter:", clf.n_iter_
    return clf

class NeuralLearner(Learner.Learner):
	def __init__(self, FeatureMask):
		super(NeuralLearner, self).__init__(FeatureMask)
	        self.expected = FeatureMask.LabelsForAllPoints
		#self.model = MLPClassifier(algorithm='sgd', hidden_layer_sizes=(64,32))
                self.model = MLPClassifier(algorithm = 'sgd', 
                                           learning_rate = 'constant',
                                           momentum = .9,
                                           nesterovs_momentum = True, 
                                           learning_rate_init = 0.2)
        def FitAndPredict(self, mask):
                return self.Predict(self.Fit(mask))
        
        def SetupInputActivations(self, FeatureMask):
		arr = np.hstack([FeatureMask.ForceStd.reshape(-1,1), 
                                 FeatureMask.ForceMinMax.reshape(-1,1),
                                 FeatureMask.CannyFilter.reshape(-1,1)])
	        expected = FeatureMask.LabelsForAllPoints
		return arr, expected

        def Fit(self, mask):
                arr, expected = self.SetupInputActivations(mask)
                self.model.fit(arr, expected)

        def Predict(self, mask):
                arr, expected = self.SetupInputActivations(mask)
                return self.model.predict(arr).reshape(-1,1)

  def create(self):
    csvPath = self.sourceCsvFile

    dataset = np.loadtxt( csvPath, dtype='int', delimiter=",", skiprows=1,converters={ \
        4: convertCell, \
        5: convertCell, \
        6: convertCell, \
        7: convertCell, \
        8: convertCell, \
        9: convertCell, \
        10: convertCell, \
        11: convertCell, \
        12: convertCell, \
        13: convertCell, \
        14: convertCell, \
        15: convertCell \
        } )

    non_cat_data = dataset[:, [0,1,2] ]
    cat_data = dataset[:, [4,5,6,7,8,9,10,11,12,13,14,15] ]

    output_data = dataset[:, 3]

    enc =  preprocessing.OneHotEncoder()
    enc.fit(cat_data)
    cat_out = enc.transform(cat_data).toarray() 
    merge_data = np.concatenate((non_cat_data,cat_data),axis=1)
    d(merge_data[0])

    clf = MLPClassifier(algorithm='l-bfgs', alpha=1e-5, hidden_layer_sizes=(5, 2), random_state=1)
    #clf = tree.DecisionTreeClassifier()
    clf = clf.fit(merge_data, output_data)

    s = pickle.dumps(clf)
    dtFileName = "%s\\save.pkl"%self.outDir
    dtFile = open( dtFileName, 'w' )
    print dtFileName
    dtFile.write( s );
    dtFile.close()

    choicesFile = open( "%s\\choices.pkl"%self.outDir, 'w' )
    s = pickle.dumps(choiceArr)
    choicesFile.write( s );
    choicesFile.close()

    sample_inputs = []
    for i in range( 100 ):
      sample_inputs.append( merge_data[i*500] )
    file = open( "%s\\sampleInputs.pkl"%self.outDir, 'w' )
    file.write( pickle.dumps(sample_inputs) )
    file.close()

    file = open( "%s\\def.txt"%self.outDir, 'w' )
    file.write( "input file: %s\n"%self.sourceCsvFile )
    file.close()

    print dataset[722]
    print merge_data[722]
    print output_data[722]
    print clf.predict( sample_inputs ) 

class BCISignal():
    def __init__(self, fs, bands, ch_names, states_labels, indexes):
        self.states_labels = states_labels
        self.bands = bands
        self.prefilter = FilterSequence([ButterFilter((0.5, 45), fs, len(ch_names))])
        self.csp_pools = [SpatialDecompositionPool(ch_names, fs, bands, 'csp', indexes) for _label in states_labels]
        self.csp_transformer = None
        self.var_detector = InstantaneousVarianceFilter(len(bands)*len(indexes)*len(states_labels), n_taps=fs//2)
        self.classifier = MLPClassifier(hidden_layer_sizes=(), early_stopping=True, verbose=True)
        #self.classifier = RandomForestClassifier(max_depth=3, min_samples_leaf=100)

    def fit(self, X, y=None):
        X = self.prefilter.apply(X)
        for csp_pool, label in zip(self.csp_pools, self.states_labels):
            csp_pool.fit(X, y == label)
        self.csp_transformer = FilterStack([pool.get_filter_stack() for pool in self.csp_pools])
        X = self.csp_transformer.apply(X)
        X = self.var_detector.apply(X)
        self.classifier.fit(X, y)
        print('Fit accuracy {}'.format(sum(self.classifier.predict(X) == y)/len(y)))

    def apply(self, chunk: np.ndarray):
        chunk = self.prefilter.apply(chunk)
        chunk = self.csp_transformer.apply(chunk)
        chunk = self.var_detector.apply(chunk)
        predicted_labels = self.classifier.predict(chunk)
        return predicted_labels

def mlp_cv_architecture(X,Y):
    kfold = KFold(X.shape[0], n_folds = 10)

    architectures = ( (500,2), (400,2), (400,100,2), (400,200,2), (400,100,50,2), (400,200,50,2) )

    res_dict = {}

    for architecture in architectures:
        mlp = MLPClassifier( algorithm = 'sgd',
                learning_rate = 'adaptive',
                hidden_layer_sizes = architecture,
                random_state = 1)

        train_times    = []
        train_accuracy = []
        test_accuracy  = []

        for train, test in kfold:
            t_tr = time.time()
            mlp.fit( X[train], Y[train] )
            train_times.append( time.time() - t_tr )
            acc_train = np.sum( np.equal( mlp.predict( X[train]), Y[train] ) ) / float(X[train].shape[0])
            acc_test  = np.sum( np.equal( mlp.predict( X[test]), Y[test] ) ) / float(X[test].shape[0])
            train_accuracy.append( acc_train )
            test_accuracy.append(  acc_test )

        res_dict[str(architecture)] = (np.mean(train_accuracy), np.std(train_accuracy),
                          np.mean(test_accuracy), np.std(test_accuracy),
                          np.mean(train_times), np.std(train_times))

    with open('./../results/res_nncv_architecture.pkl', 'w') as f:
        pickle.dump(res_dict,f)

def train():
    utl.print_title('Getting data...')
    X, Tc, X_test, Tc_test = dpp.getdata_arnold()
    #X, Tc, X_test, Tc_test = dpp.getdata_mnist()

    utl.print_title('Preparing data...')
    X, X_test = dpp.scale_data(X, X_test)
    T = dpp.one_hot_encode(Tc)
    T_test = dpp.one_hot_encode(Tc_test)

    utl.print_title('Sanity checks...')
    print('Shape X:', X.shape)
    print('Shape Tc:', Tc.shape)
    print('Shape T:', T.shape)
    print('Shape X_test:', X_test.shape)
    print('Shape Tc_test:', Tc_test.shape)
    print('Shape T_test:', T_test.shape)

    utl.print_title('Training the network...')
    classifier = MLPClassifier(solver='adam', learning_rate_init=1e-3, hidden_layer_sizes=(100), verbose=True, max_iter=200)
    classifier.fit(X, T)

    train_score, Pc = get_results(classifier, X, T)
    test_score, Pc_test = get_results(classifier, X_test, T_test)

    utl.print_title('Results:')
    print('Classification counts train (target):     ',  np.bincount(Tc.reshape(-1)))
    print('Classification counts train (prediction): ',  np.bincount(Pc))

    print('\nClassification counts test (target):     ',  np.bincount(Tc_test.reshape(-1)))
    print('Classification counts test (prediction): ',  np.bincount(Pc_test))

    print('\nTrain score: ', train_score)
    print('Test score:  ', test_score)

 def mlp_train(self,x_train,y_train):
     scaler = StandardScaler()
     scaler.fit(x_train)
     x_train = scaler.transform(x_train)
     clf = MLPClassifier(max_iter=500,alpha=1e-5,hidden_layer_sizes=(40,100,80),warm_start=True,random_state=0)
     clf.fit(x_train,y_train)
     
     return clf

def fitMLPs(trainIndexes, datasets):
	classifiers = []
	for (x,y) in datasets:
		cl =  MLPClassifier(algorithm='l-bfgs', alpha=1e-4, hidden_layer_sizes=(76, 30), random_state=1, momentum=0.8)
		data, target = listToData(trainIndexes, x, y)
		cl.fit(data, target)
		classifiers.append(cl)
	return classifiers 

def train(classes, y_samples, feature_dict, classes_dict):
    # Using dev version of slearn, 1.9
    from sklearn.neural_network import MLPClassifier

    clf = MLPClassifier(algorithm='l-bfgs', alpha=1e-5, hidden_layer_sizes=(50, 25), random_state=1, verbose=True)
    clf.fit(y_samples, classes)

    return clf

def main():
    iris = datasets.load_iris()
    X_train, X_test, y_train, y_test = train_test_split(iris.data, iris.target)

    classifier = MLPClassifier(max_iter=1000)
    classifier.fit(X_train, y_train)
    s = classifier.score(X_test, y_test)
    print(s)

def test_tolerance():
    # Test tolerance.
    # It should force the solver to exit the loop when it converges.
    X = [[3, 2], [1, 6]]
    y = [1, 0]
    clf = MLPClassifier(tol=0.5, max_iter=3000, solver='sgd')
    clf.fit(X, y)
    assert_greater(clf.max_iter, clf.n_iter_)

def test_adaptive_learning_rate():
    X = [[3, 2], [1, 6]]
    y = [1, 0]
    clf = MLPClassifier(tol=0.5, max_iter=3000, solver='sgd',
                        learning_rate='adaptive')
    clf.fit(X, y)
    assert_greater(clf.max_iter, clf.n_iter_)
    assert_greater(1e-6, clf._optimizer.learning_rate)

 def mlpTest(self):
     mlp = MLPClassifier(hidden_layer_sizes=(100, 100), max_iter=1000, alpha=1e-4,
         solver  ='sgd', verbose=10, tol=1e-4, random_state=1)
     mlp.fit(self.X_train,self.Y_train)
     predicted = mlp.predict(self.X_test)
     print("Classification report for classifier %s:\n%s\n"
         % (mlp, metrics.classification_report(self.Y_test, predicted)))
     print("Confusion matrix:\n%s"
         % metrics.confusion_matrix(self.Y_test, predicted))

def do_mlp(x_train, x_test, y_train, y_test):

    clf = MLPClassifier(solver='lbfgs',
                        alpha=1e-5,
                        hidden_layer_sizes = (10, 4),
                        random_state = 1)
    clf.fit(x_train, y_train)
    y_pred = clf.predict(x_test)
    print(classification_report(y_test, y_pred))

def main():
    np.random.seed(RANDOM_STATE)
    pd.set_option('display.width', 0)
    pd.set_option('display.max_rows', None)
    pd.set_option('display.max_columns', None)
    
    data = pd.read_csv('data/train.csv')
    
    #test_data = pd.read_csv('data/test.csv')
    
    records = []
    
    #n = 42000*0.8
    n = 10000
    X, y = extract_data(data, n)
    activation = 'tanh'
    param_dict = {'batch_size': [100, 200], 'momentum': [0.9, 0.99 ], 'learning_rate_init':[0.001, 0.01, 0.1]}
    #param_dict = {'batch_size': [200], 'momentum': [0.9], 'learning_rate_init':[0.1]}
    for param in ParameterGrid(param_dict):       
        nn = MLPClassifier(algorithm='sgd', 
                           tol=float('-inf'),
                           warm_start = True,
                           max_iter=1, 
                           hidden_layer_sizes = [200],
                           random_state=RANDOM_STATE)
        #nn_params = {'algorithm': 'sgd', 'tol': float
        nn_params = nn.get_params()
        nn_params.update(param)
        nn.set_params(**nn_params)
        #nn = MLPClassifier(**nn_params)
        time_limits = list(range(1, 60, 60))
        try:
            evaluation_list = trainer_by_time(X, y, time_limits, nn)
        except:
            evaluation_list = [{}]
            
        for i in range(len(evaluation_list)):
            evaluation = evaluation_list[i]
            record = {}
            record['n'] = n
            record['time limit'] = time_limits[i]
            record.update(evaluation)  
            record.update(param)
            records.append(record)
        
        
    df = pd.DataFrame(records)
    cols = list(df.columns)
    keys = evaluation_list[0].keys()
    cols = [item for item in cols if item not in keys]
    cols += keys
    df = df.reindex(columns=cols)
    now = datetime.datetime.now()    
    result_file = open('result.txt', 'a')
    print(now,file=result_file)
    print(df)
    print(df,file=result_file)