def multi_SVM(needcv = False):
	NeedReFetch = NEED_REFETCH
	OnlyNeedReGenerate = ONLY_NEED_REGENERATE
	allGenreSongsTrain,allGenreSongsTest = fetchData_CM(NUM_NEED_PER_GENRE,GENRES,NeedReFetch,OnlyNeedReGenerate,USED_GENRES)
	# allGenreSongsTrain,allGenreSongsTest = featureSelection (allGenreSongsTrain,allGenreSongsTest,method = 'mean',testmode = True,n_features_to_select = 4)


	# assert(len(allGenreSongsTrain[0][0]) == 106)

	TrainX = []
	TrainY = []
	TestX = []
	TestY = []
	for i in range(sum(USED_GENRES)):
		for j in allGenreSongsTrain[i]:
			TrainX.append(j)
			TrainY.append(i)
		for k in allGenreSongsTest[i]:
			TestX.append(k)
			TestY.append(i)
	confuseMat = [[0 for i in range(sum(USED_GENRES))] for j in range(sum(USED_GENRES))];
	if not needcv:
		print "Start SVM training ... "
		model = SVC(probability=True,decision_function_shape='ovo',kernel = 'rbf',gamma = 0.0078125, C = 8)
		model.fit(TrainX,TrainY)
		print "Start SVM predicting ... "
		PredY = model.predict(TestX)
		for i in range(len(TestY)):
			confuseMat[TestY[i]][PredY[i]] += 1
		print(clfr(TestY, PredY))
	else:
		tuned_parameters = [															## remained to be play with
							{'kernel': ['rbf'], 'gamma': [2**i for i in range(-8,8)], 'C': [2**i for i in range(-8,8)]},
		 					# {'kernel': ['linear'], 'C': [2**i for i in range(-8,9,2)]},
		 					# {'kernel': ['poly'], 'gamma': [2**i for i in range(-8,9,2)], 'C': [2**i for i in range(-8,9,2)], 'degree':[2,3,4]},
		 					]
		print "Start SVM CV ... "
		clf = GSCV(SVC(decision_function_shape='ovo'), tuned_parameters, cv=5)
		clf.fit(TrainX, TrainY)


		print("Best parameters set found on development set:")
		print(clf.best_params_)
		# print("Grid scores on development set:")
		# print()
		# for params, mean_score, scores in clf.grid_scores_:
		# 	print("%0.4f (+/-%0.03f) for %r" % (mean_score, scores.std(), params))
		# print()

		print "Start SVM predicting ... "

		PredY = clf.predict(TestX)


		print(clfr(TestY, PredY))

		for i in range(len(TestY)):
			confuseMat[TestY[i]][PredY[i]] += 1

	return confuseMat

def main():
    print 'MIFS'
    filename = ['../data/arcene.mat', '../data/gisette.mat', '../data/madelon.mat']
    for f_num in range(len(filename)):
        print filename[f_num]
        mat = scipy.io.loadmat(filename[f_num])
        X = mat['X']    # data
        y = mat['Y']    # label
        y = y[:, 0]
        X = X.astype(float)
        n_sample, n_features = X.shape
        # split data
        ss = cross_validation.KFold(n_sample, n_folds=10, shuffle=True)
        # choose SVM as the classifier
        clf = SVC()
        num_fea = np.linspace(5, 300, 60)
        correct = np.zeros(len(num_fea))
        for train, test in ss:
            # select features
            F = MIFS.mifs(X[train], y[train], n_selected_features=300)
            for n in range(len(num_fea)):
                fea_idx = F[0:num_fea[n]]
                features = X[:, fea_idx]
                clf.fit(features[train], y[train])
                y_predict = clf.predict(features[test])
                acc = accuracy_score(y[test], y_predict)
                correct[n] += acc
        correct.astype(float)
        correct /= 10
        for i in range(len(num_fea)):
            print num_fea[i], correct[i]

def buildAndEvaluateSvm(p_kernel):
    model = SVC(kernel=p_kernel)

    model.fit(features, target)
    expected = target
    predicted = model.predict(features)
    print("SCORE: %f" % metrics.roc_auc_score(expected,predicted))

class PcaGmm(BaseEstimator):
    def __init__(self, X_all,
                 pca_components = 12, gmm_components = 4,
                 covariance_type = "full", min_covar = 0.1,
                 gamma = 0, C = 1.0):
        self.pca_components = pca_components
        self.gmm_components = gmm_components
        self.covariance_type = covariance_type
        self.min_covar = min_covar
        self.gamma = gamma
        self.C = C
        self.X_all = X_all
        X_all = X_all[:, :pca_components]
        self.gmm = GMM(n_components = gmm_components,
                       covariance_type = covariance_type,
                       min_covar = min_covar)
        self.gmm.fit(X_all)
    def fit(self, X, y):
        X = X[:, :self.pca_components]
        X = self.gmm.predict_proba(X)
        self.svm = SVC(C = self.C, gamma = self.gamma)
        self.svm.fit(X, y)
    def predict(self, X):
        X = X[:, :self.pca_components]
        return self.svm.predict(self.gmm.predict_proba(X))
    def score(self, X, y):
        y_pred = self.predict(X)
        return accuracy_score(y, y_pred)
    def transform(self, X, y = None):
        X = X[:, :self.pca_components]
        return self.gmm.predict_proba(X)
    def __str__(self):
        return "PCA(%d)-GMM(%d, %s, %f)-SVM(C=%f, gamma=%f)" % (self.pca_components, self.gmm_components,self.covariance_type, self.min_covar,self.C, self.gamma)

def cvalidate():
    from sklearn import cross_validation

    trainset = np.genfromtxt(open('train','r'), delimiter=' ')
    targetset = np.genfromtxt(open('target','r'), delimiter=' ')
    X = np.array([x[0:64] for x in trainset])
    y = np.array([x for x in targetset])
    #print X,y
    X_train, X_test, y_train, y_test = cross_validation.train_test_split(X, y, test_size = 0.3, random_state = 0)

    #X_train, X_test = decomposition_pca(X_train, X_test)
    

    X_train, X_test = decomposition_pca(X_train, X_test)
    c_range = 10.0 ** np.arange(6.5,7.5,1)
    gamma_range = 10.0 ** np.arange(-2.5,0.5,1)
    #parameters = {'kernel':['rbf'], 'C':c_range} 
    parameters = {'kernel':['rbf'], 'C':c_range,  'gamma':gamma_range} 
    svr = SVC(kernel = 'rbf', C = 0.72, gamma = 0.299)

    #clf = grid_search.GridSearchCV(svr, parameters)

    #print clf.estimator
    ##clf = Pipeline([('scale', Scaler()), ('svm', SVC())])

    svr.fit(X_train, y_train)
    print svr.score(X_test, y_test)

    def test_perm(self):
        X, y = datasets.make_classification(n_samples=20, n_features=5, n_informative=2)
        n_perms = 2
        rnd = 0
        # = With EPAC
        wf = Perms(SVC(kernel="linear"), n_perms=n_perms, permute="y", random_state=rnd, reducer=None)
        r_epac = wf.top_down(X=X, y=y)
        # = With SKLEARN
        clf = SVC(kernel="linear")
        r_sklearn = list()
        for perm in Permutations(n=y.shape[0], n_perms=n_perms, random_state=rnd):
            y_p = y[perm, :]
            clf.fit(X, y_p)
            r_sklearn.append(clf.predict(X))
        key2cmp = "y" + conf.SEP + conf.PREDICTION

        # = Comparison
        for iperm in range(n_perms):
            comp = np.all(np.asarray(r_epac[iperm][key2cmp]) == np.asarray(r_sklearn[iperm]))
            self.assertTrue(comp, u"Diff Perm: EPAC vs sklearn")
        # test reduce
        for iperm in range(n_perms):
            r_epac_reduce = wf.reduce().values()[iperm][key2cmp]
            comp = np.all(np.asarray(r_epac_reduce) == np.asarray(r_sklearn[iperm]))
            self.assertTrue(comp, u"Diff Perm: EPAC reduce")

def get_optimize_result(training_data, validation_data, important_cols_result):
    """ Get the number of cols that gets the best score """
    last_score = 0.0
    new_score = 0.0
    number_of_cols = 1
    decreases = 0
    optimal_result = {'score': 0.0, 'number_of_cols': 1}
    # Extract labels from data frames
    training_data_label, training_data = separate_labels(training_data)
    validation_data_label, validation_data = separate_labels(validation_data)
    while True:
        cols = important_cols_result.index[0: number_of_cols]
        # Fit models and test
        clf = SVC()
        clf.fit(training_data.iloc[:, cols], training_data_label)
        predictions = clf.predict(validation_data.iloc[:, cols])
        new_score = accuracy_score(validation_data_label, predictions)
        print(new_score)
        if new_score < optimal_result['score']:
            optimal_result['score'] = new_score
            optimal_result['number_of_cols'] = number_of_cols
            print(optimal_result)
        if last_score > new_score:
            decreases += 1
            if decreases > 5:
                break
        last_score = new_score
        number_of_cols += 5
    cols = important_cols_result.index[0: number_of_cols]
    print(optimal_result)
    export_optimal_result(training_data.iloc[:, cols].columns)

def main() :
    np.random.seed(1234)

    # read the tweets and its labels
    dictionary = extract_dictionary('../data/tweets.txt')
    X = extract_feature_vectors('../data/tweets.txt', dictionary)
    y = read_vector_file('../data/labels.txt')

    metric_list = ["accuracy", "f1_score", "auroc"]

    ### ========== TODO : START ========== ###
    # part 1: split data into training (training + cross-validation) and testing set
    s1 = slice(0, 560, 1)
    s2 = slice(560, 630, 1)
    training_data = X[s1]
    test_data = X[s2]

    training_label = y[s1]
    test_label = y[s2]

    # part 2: create stratified folds (5-fold CV)
    kf = StratifiedKFold(training_label, n_folds=5)
    # part 2: for each metric, select optimal hyperparameter for linear-kernel SVM using CV
    for metric in metric_list:
        val = select_param_linear(training_data, training_label, kf, metric)
        print("max c: " + str(val))

    # part 3: train linear-kernel SVMs with selected hyperparameters
    Model = SVC(kernel='linear', C=10)
    Model.fit(training_data, training_label)
    # part 3: report performance on test data
    for metric in metric_list:
        perf = performance_test(Model, test_data, test_label, metric)
        print 'Performance of ' + str(metric) + ': ' + str(perf)

def fitall(X,y):
    m1 =  RandomForestClassifier(n_estimators=500)
    m2 =  LogisticRegression()
    m3 =  SVC(probability=True)
    lib = [m1, m2, m3]
    m4 = sl.SuperLearner(lib, loss = "nloglik")
    return m1.fit(X,y), m2.fit(X,y), m3.fit(X,y), m4.fit(X, y)

def classify_rbf_tf(train_vector, train_label):
    classifier_RBFSvc_tf = SVC(kernel='rbf', gamma=2)
    rbf_clf_tf = classifier_RBFSvc_tf.fit(train_vector, train_label)
    save_clf2 = open("RBFSvcTf.pickle","wb")
    pickle.dump(rbf_clf_tf,save_clf2)
    save_clf2.close()
    return rbf_clf_tf

def classify_poly_tf(train_vector, train_label):
    classifier_polySvc_tf = SVC(kernel='poly', degree=2, gamma=2)
    poly_clf_tf = classifier_polySvc_tf.fit(train_vector, train_label)
    save_clf3 = open("polySvcTf.pickle","wb")
    pickle.dump(poly_clf_tf,save_clf3)
    save_clf3.close()
    return poly_clf_tf

def classify_linear_tf(train_vector, train_label):
    classifier_linearSvc_tf = SVC(kernel='linear')
    linear_clf_tf = classifier_linearSvc_tf.fit(train_vector, train_label)
    save_clf1 = open("linearSvcTf.pickle","wb")
    pickle.dump(linear_clf_tf,save_clf1)
    save_clf1.close()
    return linear_clf_tf

def experiment_three_feature_selection(svm,train_set,test_set,c_star):

    # Find the weight vectors
    weight_vectors = svm.coef_
    accuracies = np.zeros(58)

    # Sort the indexes of the weight vectors, then flip to largest to smallest.
    sorted_index = np.argsort(weight_vectors)
    reverse_index = np.fliplr(sorted_index)[0]

    # Randomizing the indices for the experiment
    np.random.shuffle(reverse_index)
    for i in range(2,len(reverse_index)+1):
        selector = reverse_index[:i]
        split_train = train_set[:,selector]
        split_test = test_set[:,selector]

        # Create the new SVM
        SVM_one = SVC(kernel='linear', C=c_star, probability=True)
        SVM_one.fit(split_train, train_set[:, -1])
        new_predictions = SVM_one.predict(split_test)

        # Calculate the accuracy, precision, and recall of the SVM
        accuracy = metrics.accuracy_score(test_set[:,-1], new_predictions)
        accuracies[i] = accuracy

        # Was not entirely sure if this part was needed.
        # Reshuffled this way every time it was a randomized set of m features.
        # np.random.shuffle(reverse_index)
    return accuracies

def experiment_two_feature_selection(svm,train_set,test_set,c_star):

    # Find the weight vectors
    weight_vectors = svm.coef_
    # Absolute these for argsort to accurately find the max to min vector
    weight_vectors = np.absolute(weight_vectors)
    accuracies = np.zeros(58)

    # Sort the indexes of the weight vectors, then flip to largest to smallest.
    sorted_index = np.argsort(weight_vectors)
    reverse_index = np.fliplr(sorted_index)[0]

    # Print for the Experiment 2 to find best 5 features.
    print("Top 5 weight vectors for Experiment 2: ", reverse_index[:5])
    for i in range(2,len(reverse_index)+1):
        selector = reverse_index[:i]
        split_train = train_set[:,selector]
        split_test = test_set[:,selector]    # Create the new SVM
        SVM_one = SVC(kernel='linear', C=c_star, probability=True)
        SVM_one.fit(split_train, train_set[:, -1])
        new_predictions = SVM_one.predict(split_test)

        # Calculate the accuracy, precision, and recall of the SVM
        accuracy = metrics.accuracy_score(test_set[:,-1], new_predictions)
        accuracies[i] = accuracy
    return accuracies

def svm_solver(train_data, train_label, validation, test, dimreduce, convertbinary) :
    """
    """
    logging.info ('begin to train the svm classifier')

    # train_data = train_data[:100,:]
    # validation = validation[:100,:]
    # test = test[:100,:]
    # train_label = train_label[:100]
    train_data, validation, test = dimreduce(train_data, train_label, validation, test)
    # print new_train_data.shape
    train_data, validation, test = convertbinary(train_data, validation, test)

    """
    svc = SVC ()
    params_rbf = {"kernel": ['rbf'],
             "class_weight": ['auto'],
             "C": [0.1 ,0.2 ,0.3 ,0.5 ,1, 2, 3, 5, 10],
             "gamma": [0.01, 0.03,  0.05, 0.1, 0.2, 0.3, 0.5],
             "tol": 10.0** -np.arange(1, 5),
             "random_state": [1000000007]}
    logging.info ("Hyperparameter opimization using RandomizedSearchCV...")
    rand_search_result = RandomizedSearchCV (svc, param_distributions = params_rbf, n_jobs = -1, cv = 3, n_iter = 30)
    # rand_search_result = GridSearchCV (svc , param_grid = params_rbf , n_jobs = 8  , cv = 3)
    rand_search_result.fit (train_data , train_label)
    params = tools.report (rand_search_result.grid_scores_)
    """
    params = {'kernel': 'poly', 'C': 0.1, 'random_state': 1000000007, 'tol': 0.001, 'gamma': 0.1, 'class_weight': 'auto'}
    svc = SVC (probability = True, **params)

    svc.fit (train_data , train_label)
    evaluate.get_auc (svc.predict_proba (validation)[:,1])
    return svc.predict_proba (test)[:,1]

def main():
    print '[INFO, time: %s] Getting Data....' % (time.strftime('%H:%M:%S'))
    testing_file = file('test.p', 'r')
    training_file = file('train.p', 'r')

    train = pickle.load(training_file)
    test = pickle.load(testing_file)

    testing_file.close()
    training_file.close()
    
    trainX = train[:,:-1]
    trainy = train[:,-1]
    
    testX = test[:,:-1]
    testy = test[:,-1]

    print '[INFO, time: %s] Downsampling ...' % (time.strftime('%H:%M:%S'))
    trainX = downsample_features(trainX)
    testX = downsample_features(testX)

    print '[INFO, time: %s] Fitting %s ...' % (time.strftime('%H:%M:%S'), 'SVM - rbf kernel (i.e. gaussian) with default paramenters')
    clf = SVC()
    clf.fit(trainX, trainy)

    print '[INFO, time: %s] Making Predictions...' % (time.strftime('%H:%M:%S'))
    prediction = clf.predict(testX)
    print '[RESULT, time: %s] accuracy = %f' % (time.strftime('%H:%M:%S'),accuracy_score(testy, prediction))

    def test_cv(self):
        X, y = datasets.make_classification(n_samples=20, n_features=5, n_informative=2)
        n_folds = 2

        # = With EPAC
        wf = CV(SVC(kernel="linear"), n_folds=n_folds, reducer=ClassificationReport(keep=True))
        r_epac = wf.top_down(X=X, y=y)

        # = With SKLEARN
        clf = SVC(kernel="linear")
        r_sklearn = list()
        for idx_train, idx_test in StratifiedKFold(y=y, n_folds=n_folds):
            # idx_train, idx_test  = cv.__iter__().next()
            X_train = X[idx_train, :]
            X_test = X[idx_test, :]
            y_train = y[idx_train, :]
            clf.fit(X_train, y_train)
            r_sklearn.append(clf.predict(X_test))

        # = Comparison
        key2cmp = "y" + conf.SEP + conf.TEST + conf.SEP + conf.PREDICTION
        for icv in range(n_folds):
            comp = np.all(np.asarray(r_epac[0][key2cmp]) == np.asarray(r_sklearn[0]))
            self.assertTrue(comp, u"Diff CV: EPAC vs sklearn")

        # test reduce
        r_epac_reduce = wf.reduce().values()[0][key2cmp]
        comp = np.all(np.asarray(r_epac_reduce) == np.asarray(r_sklearn))
        self.assertTrue(comp, u"Diff CV: EPAC reduce")

def svc_classifier(X_train, categories,X_test, test_categories):
    from sklearn.svm import SVC
    svm_classifier = SVC(C=100, gamma=0.1).fit(X_train, categories)
    y_svm_predicted = svm_classifier.predict(X_test)
    print '\n Here is the classification report for support vector machine classiffier:'
    print metrics.classification_report(test_categories, y_svm_predicted)
    to_latex(test_categories, y_svm_predicted)  

    def predict(self):
        if self.stage != Stage.SIMILARITY:
            print '没有计算相似度'
        import numpy as np
        
        utiles.clearList(self.predictPC)
        utiles.clearList(self.evaluation)
        for i in range(len(self.forest)):
            similarityPCList = []
            for sim in self.similarity[i]:
                similarityPCList.append(int(sim.powerConsume*100000))
            similarity_X = np.array(self.similarityList[i])
            similarity_y = np.array(similarityPCList)
#            print similarity_y
            predictModel = SVC()
            predictModel.fit(similarity_X, similarity_y)
            
            forest_X = np.array(self.forestList[i])
            forest_X = forest_X.reshape(1,-1)
            predicted = predictModel.predict(forest_X)
            
            self.forest[i].powerConsume = float(predicted[0]/100000.0)
            self.predictPC.append(float(predicted[0]/100000.0))
        
        for i in range(len(self.expect)):
            self.evaluation.append((self.expect[i]-self.predictPC[i])/self.expect[i])

def columnSvm(data, columns):
	loo = cross_validation.LeaveOneOut(len(data))
	totalAccurateNum = 0
	singleExpression = [0] * 7
	singleExpressionAccurate = [0] * 7
	eachAccuracy = [0.0] * 7
	for i in range(1, 8):
		singleExpression[i - 1] = sum(data[:, 43] == i)
	
	for trainIndex, testIndex in loo:
		xTrain, yTrain = data[trainIndex, 0 : 43], data[trainIndex, 43]
		xTest, yTest = data[testIndex, 0 : 43], data[testIndex, 43]
		xTrain = xTrain[:, columns]
		xTest = xTest[:, columns]
		svm = SVC(kernel = "rbf", C = 4.0, gamma = 0.01)
		svm.fit(xTrain, yTrain)
		yPredict = svm.predict(xTest)
		
		if yTest == yPredict :
			totalAccurateNum += 1
			singleExpressionAccurate[yTest[0] - 1] += 1
		else :
			print yTest, yPredict
	print "total accuracy: ", float(totalAccurateNum) / len(data)
	for i in range(0, 7):
		eachAccuracy[i] = float(singleExpressionAccurate[i]) / singleExpression[i]
		print i+1, eachAccuracy[i]
	
	return totalAccurateNum, eachAccuracy