def leave_one_out_cv(gram_matrix, labels, alg = 'SVM'):
    """
    leave-one-out cross-validation
    """
    scores = []
    preds = []
    loo = sklearn.cross_validation.LeaveOneOut(len(labels))
    for train_index, test_index in loo:
        X_train, X_test = gram_matrix[train_index][:,train_index], gram_matrix[test_index][:, train_index]
        y_train, y_test = labels[train_index], labels[test_index]
        if(alg == 'SVM'):
            svm = sklearn.svm.SVC(kernel = 'precomputed')
            svm.fit(X_train, y_train)
            preds += svm.predict(X_test).tolist()
            score = svm.score(X_test, y_test)
        elif(alg == 'kNN'):
            knn = sklearn.neighbors.KNeighborsClassifier()
            knn.fit(X_train, y_train)
            preds += knn.predict(X_test).tolist()
            score = knn.score(X_test, y_test)
        scores.append(score)

    print "Mean accuracy: %f" %(np.mean(scores))
    print "Stdv: %f" %(np.std(scores))

    return preds, scores

def svm_iterkernel(train_data, train_labels, test_data, test_labels, op_name_dir):


	label_set=np.unique(train_labels)

	if op_name_dir != ('None' or 'none'):
		fo=open(op_name_dir,'a')

	predict_list={}
	for kernel in ['linear']: #, 'poly', 'rbf']:
		t0=time.time()
		svm = SVC(C=1., kernel=kernel, cache_size=10240)
		svm.fit(train_data, train_labels)
		prediction=svm.predict(test_data)
		predict_list[kernel]=prediction
		pred_acc_tot =(float(np.sum(prediction == test_labels)))/len(test_labels)
		print time.time() - t0, ',kernel = '+kernel, ',pred acc = '+str(round(pred_acc_tot*100))
		if op_name_dir != ('None' or 'none'):
			fo.write('time='+str(time.time() - t0)+'sec,kernel='+kernel+',pred acc='+str(round(pred_acc_tot*100))+'\n')
		for lab_unq in label_set:	
			pred_acc=(prediction == lab_unq) & (test_labels == lab_unq)
			pred_acc=float(pred_acc.sum())/(len(test_labels[test_labels == lab_unq]))
			print 'pred_'+str(lab_unq)+','+str(round(pred_acc*100))	
			if op_name_dir != ('None' or 'none'):
				fo.write('pred_'+str(lab_unq)+','+str(round(pred_acc*100))+'\n')

	if op_name_dir != ('None' or 'none'):
		fo.close()

	return predict_list

def trainSVM(filteredFaces, labels, subjects, e):
    uniqueSubjects = np.unique(subjects)
    accuracies = []
    masterK = filteredFaces.dot(filteredFaces.T)
    for testSubject in uniqueSubjects:
        idxs = np.nonzero(subjects != testSubject)[0]
        someFilteredFacesTrain = filteredFaces[idxs]
        someLabels = labels[idxs]
        y = someLabels == e
        K = masterK[idxs, :]
        K = K[:, idxs]
        svm = sklearn.svm.SVC(kernel="precomputed")
        svm.fit(K, y)

        idxs = np.nonzero(subjects == testSubject)[0]
        someFilteredFaces = filteredFaces[idxs]
        someLabels = labels[idxs]
        y = someLabels == e
        yhat = svm.decision_function(someFilteredFaces.dot(someFilteredFacesTrain.T))

        if len(np.unique(y)) > 1:
            auc = sklearn.metrics.roc_auc_score(y, yhat)
        else:
            auc = np.nan
        print "{}: {}".format(testSubject, auc)
        accuracies.append(auc)
    accuracies = np.array(accuracies)
    accuracies = accuracies[np.isfinite(accuracies)]
    print np.mean(accuracies), np.median(accuracies)

def train():
	training_set=[]
	training_labels=[]
	os.chdir("/Users/muyunyan/Desktop/EC500FINAL/logo/")
	counter=0
	a=os.listdir(".")
	for i in a:
	 os.chdir(i)
	 print(i)
	 for d in os.listdir("."):
		 img = cv2.imread(d)
		 res=cv2.resize(img,(250,250))
		 gray_image = cv2.cvtColor(res, cv2.COLOR_BGR2GRAY)
		 xarr=np.squeeze(np.array(gray_image).astype(np.float32))
		 m,v=cv2.PCACompute(xarr)
		 arr= np.array(v)
		 flat_arr= arr.ravel()
		 training_set.append(flat_arr)
		 training_labels.append(i)
	 os.chdir("..")
	 trainData=training_set
	 responses=training_labels
	 svm = svm.SVC()
	 svm.fit(trainData,responses)
	 return svm

def run_model(train_data, train_labels, test_data, test_labels):
    '''
    Algorithm which will take in a set of training text and labels to train a bag of words model
    This model is then used with a logistic regression algorithm to predict the labels for a second set of text
    Method modified from code available at:
    https://www.kaggle.com/c/word2vec-nlp-tutorial/details/part-1-for-beginners-bag-of-words
    Args:
        train_data_text: Text training set.  Needs to be iterable
        train_labels: Training set labels
        test_data_text: The text to
    Returns:
        pred_labels: The predicted labels as determined by logistic regression
    '''

    #use Logistic Regression to train a model
    svm = SVC()

    # we create an instance of Neighbours Classifier and fit the data.
    svm.fit(train_data, train_labels)

    #Now that we have something trained we can check if it is accurate with the test set
    pred_labels = svm.predict(test_data)
    perform_results = performance_metrics.get_perform_metrics(test_labels, pred_labels)

    #Perform_results is a dictionary, so we should add other pertinent information to the run
    perform_results['vector'] = 'Bag_of_Words'
    perform_results['alg'] = 'Support_Vector_Machine'

    return pred_labels, perform_results

def trainOneSVM(masterK, y, subjects):
    Cs = 1.0 / np.array([0.1, 0.5, 2.5, 12.5, 62.5, 312.5])
    # Cs = 10. ** np.arange(-5, +6)/2.
    uniqueSubjects, subjectIdxs = np.unique(subjects, return_inverse=True)
    highestAccuracy = -float("inf")
    NUM_MINI_FOLDS = 4
    for C in Cs:  # For each regularization value
        # print "C={}".format(C)
        accuracies = []
        for i in range(NUM_MINI_FOLDS):  # For each test subject
            testIdxs = np.nonzero(subjectIdxs % NUM_MINI_FOLDS == i)[0]
            trainIdxs = np.nonzero(subjectIdxs % NUM_MINI_FOLDS != i)[0]
            if len(np.unique(y[testIdxs])) > 1:
                K = masterK[trainIdxs, :]
                K = K[:, trainIdxs]
                svm = sklearn.svm.SVC(kernel="precomputed", C=C)
                svm.fit(K, y[trainIdxs])

                K = masterK[testIdxs, :]
                K = K[:, trainIdxs]  # I.e., need trainIdxs dotted with testIdxs
                accuracy = sklearn.metrics.roc_auc_score(y[testIdxs], svm.decision_function(K))
                # print accuracy
                accuracies.append(accuracy)
        if np.mean(accuracies) > highestAccuracy:
            highestAccuracy = np.mean(accuracies)
            bestC = C
    svm = sklearn.svm.SVC(kernel="precomputed", C=bestC)
    svm.fit(masterK, y)
    return svm

def main():
    data = pickle.load(open('../submodular_20.pickle'))
    train, train_labels, test, test_labels = Load20NG()
    vectorizer = sklearn.feature_extraction.text.CountVectorizer(binary=True,
            lowercase=False) 
    vectorizer.fit(train + test)                                                          
    train_vectors = vectorizer.transform(train)
    test_vectors = vectorizer.transform(test)                                             
    svm = sklearn.svm.SVC(probability=True, kernel='rbf', C=10,gamma=0.001)               
    svm.fit(train_vectors, train_labels)                                                  
    
    json_ret = {}
    json_ret['class_names'] = ['Atheism', 'Christianity']
    json_ret['instances'] = []
    explanations = data['explanations']['20ng']['svm']
    idxs = data['submodular_idx']['20ng']['svm'][:10]
    for i in idxs:
        json_obj = {}
        json_obj['id'] = i
        idx = i
        instance = test_vectors[idx]
        json_obj['true_class'] = test_labels[idx]
        json_obj['c1'] = {}
        json_obj['c1']['predict_proba'] = list(svm.predict_proba(test_vectors[idx])[0])
        exp = explanations[idx]
        json_obj['c1']['exp'] = exp 
        json_obj['c1']['data'] = get_pretty_instance(test[idx], exp, vectorizer)
        json_ret['instances'].append(json_obj)
    import json
    open('static/exp2_local.json', 'w').write('data = %s' % json.dumps(json_ret))

def q20():
    X, y = load_data('/Users/pjhades/code/lab/ml/train.dat')
    y = set_binlabel(y, 0)

    # init hit counts
    gammas = [1, 10, 100, 1000, 10000]
    hits = {}
    for gamma in gammas:
        hits[gamma] = 0

    repeat = 100
    for round in range(repeat):
        print('round {0}/{1}'.format(round, repeat), end=', ')

        err_min = 1
        gamma_min = max(gammas) + 1

        X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=1000)
        for gamma in gammas:
            svm = sklearn.svm.SVC(C=0.1, kernel='rbf', gamma=gamma)
            svm.fit(X_train, y_train)
            err = get_error(svm, X_val, y_val)
            if err < err_min or (err == err_min and gamma < gamma_min):
                err_min = err
                gamma_min = gamma
        hits[gamma_min] += 1
        print('gamma={0}'.format(gamma_min))

    for gamma in gammas:
        print('{0} hits {1} times'.format(gamma, hits[gamma]))

def q15():
    X_train, y_train = load_data('/Users/pjhades/code/lab/ml/train.dat')
    y = set_binlabel(y_train, 0)

    svm = sklearn.svm.SVC(C=0.01, kernel='linear') 
    svm.fit(X_train, y)
    print(linalg.norm(svm.coef_))

def k_fold_cv(gram_matrix, labels, folds = 10, alg = 'SVM', shuffle = True):
    """
    K-fold cross-validation
    """
    pdb.set_trace()
    scores = []
    preds = []
    loo = sklearn.cross_validation.KFold(len(labels), folds, shuffle = shuffle, random_state = random.randint(0,100))
    #loo = sklearn.cross_validation.LeaveOneOut(len(labels))
    for train_index, test_index in loo:
        X_train, X_test = gram_matrix[train_index][:,train_index], gram_matrix[test_index][:, train_index]
        y_train, y_test = labels[train_index], labels[test_index]
        if(alg == 'SVM'):
            svm = sklearn.svm.SVC(kernel = 'precomputed')
            svm.fit(X_train, y_train)
            preds += svm.predict(X_test).tolist()
            score = svm.score(X_test, y_test)
        elif(alg == 'kNN'):
            knn = sklearn.neighbors.KNeighborsClassifier()
            knn.fit(X_train, y_train)
            preds += knn.predict(X_test).tolist()
            score = knn.score(X_test, y_test)

        scores.append(score)

    print "Mean accuracy: %f" %(np.mean(scores))
    print "Stdv: %f" %(np.std(scores))

    return preds, scores

def svm_train(X,y,k):
	C_range = 10.0 ** np.arange(-2, 9)
	gamma_range = 10.0 ** np.arange(-5, 4)
	param_grid = dict(gamma=gamma_range, C=C_range)
	cv = StratifiedKFold(y=y,n_folds=k)
	svm = GridSearchCV(SVC(), param_grid=param_grid, cv=cv)
	svm.fit(X,y)
	return svm

def svm_liblinear_solver(X, y, C, tol=1e-6, max_iter=100, verbose=False):
    svm = sklearn.svm.LinearSVC(loss='hinge', tol=tol, C=C, verbose=verbose,
                                intercept_scaling=10, max_iter=max_iter)
    now = time.clock()
    svm.fit(X, y)
    res_time = time.clock() - now
    return {'w0': svm.intercept_[0],
            'w': svm.coef_.copy()[0],
            'time': res_time}

def trainSVM(svm, sv, y):
	print "\ntraining SVM"
	# cross validate 5 times
	scores = cross_val_score(svm, sv, y, cv=5)
	print scores

	# fit the data to the labels
	svm.fit(sv, y)
	return svm

def q16_17():
    X_train, y_train = load_data('/Users/pjhades/code/lab/ml/train.dat')

    for goal in [0, 2, 4, 6, 8]:
        y = set_binlabel(y_train, goal)
        svm = sklearn.svm.SVC(C=0.01, kernel='poly', degree=2, coef0=1, gamma=1)
        svm.fit(X_train, y)
        ein = get_error(svm, X_train, y)
        print('{0} vs not {0}, ein={1}'.format(goal, ein), end=', ')
        print('sum of alphas={0}'.format(np.sum(np.abs(svm.dual_coef_))))

def q19():
    X_train, y_train = load_data('/Users/pjhades/code/lab/ml/train.dat')
    X_test, y_test = load_data('/Users/pjhades/code/lab/ml/test.dat')

    y_train = set_binlabel(y_train, 0)
    y_test = set_binlabel(y_test, 0)

    for gamma in [10000, 1000, 1, 10, 100]:
        svm = sklearn.svm.SVC(C=0.1, kernel='rbf', gamma=gamma)
        svm.fit(X_train, y_train)
        print('gamma={0:<10}, Eout={1}'.format(gamma, get_error(svm, X_test, y_test)))

def hw1q18():
    print "----------------------------------------"
    print "         Homework 1 Question 18         "
    print "----------------------------------------"

    Y_train_0 = (Y_train == 0).astype(int)
    Y_test_0 = (Y_test == 0).astype(int)

    print "in the training set:"
    print "n(+) =", np.count_nonzero(Y_train_0 == 1), "n(-) =", np.count_nonzero(Y_train_0 == 0)

    print "in the test set:"
    print "n(+) =", np.count_nonzero(Y_test_0 == 1), "n(-) =", np.count_nonzero(Y_test_0 == 0)

    for C in (0.001, 0.01, 0.1, 1, 10):
        svm = sklearn.svm.SVC(C=C, kernel="rbf", gamma=100, tol=1e-7, shrinking=True, verbose=False)
        svm.fit(X_train, Y_train_0)

        print "----------------------------------------"
        print "C =", C

        support = svm.support_
        coef = svm.dual_coef_[0]
        b = svm.intercept_[0]

        print "nSV =", len(support)
        Y_predict = svm.predict(X_test)

        print "in the prediction:"
        print "n(+) =", np.count_nonzero(Y_predict == 1), "n(-) =", np.count_nonzero(Y_predict == 0)

        print "E_out =", np.count_nonzero(Y_test_0 != Y_predict)
        print

        fig = plt.figure()
        plt.suptitle("C =" + str(C))
        plt.subplot(311)
        plt.title("Training data: green +, red -")
        plot_01(X_train, Y_train_0)
        plt.tick_params(axis="x", labelbottom="off")

        plt.subplot(312)
        plt.title("Prediction on test data: green +, red -")
        plot_01(X_test, Y_predict)
        plt.tick_params(axis="x", labelbottom="off")

        plt.subplot(313)
        plt.title("Support vectors: blue")
        plt.plot(X_train[:, 0], X_train[:, 1], "r.")
        plt.plot(X_train[support, 0], X_train[support, 1], "b.")

    plt.show()

def q18():
    X_train, y_train = load_data('/Users/pjhades/code/lab/ml/train.dat')
    X_test, y_test = load_data('/Users/pjhades/code/lab/ml/test.dat')

    y_train = set_binlabel(y_train, 0)
    y_test = set_binlabel(y_test, 0)

    for C in [0.001, 0.01, 0.1, 1, 10]:
        svm = sklearn.svm.SVC(C=C, kernel='rbf', gamma=100)
        svm.fit(X_train, y_train)

        print('C={0}'.format(C))
        print('# support vectors =', np.sum(svm.n_support_))
        print('Eout =', get_error(svm, X_test, y_test))

    def runSVM(self):
        """
        Runs the SVM on 5 different splits of cross validation data
        """
        for train, test in self.kf:
            svm = self.models["SVM"]

            train_set, train_labels = self.getCurrFoldTrainData(train)
            test_set, test_labels = self.getCurrFoldTestData(test)
            svm.fit(train_set, train_labels)

            preds = svm.predict(test_set)
            acc = self.getAccuracy(test_labels, preds)
            print "(SVM) Percent correct is", acc

def hw1q15():
    svm = sklearn.svm.SVC(C=0.01, kernel="linear", shrinking=False, verbose=True)

    X_train_0 = X_train
    Y_train_0 = (Y_train == 0).astype(int)

    svm.fit(X_train_0, Y_train_0)

    w = svm.coef_[0]
    b = svm.intercept_[0]

    print "w =", w
    print "norm(w) =", np.linalg.norm(w, ord=2)
    print "b =", b

def trainTest():

    data2010, labels2010 = read_tac('2010')
    data2011, labels2011 = read_tac("2011")

    #classifiers
    gnb = naive_bayes.GaussianNB()
    svm = svm.SVC(kernel = "linear")
    logReg = linear_model.LogisticRegression()


    gnb.fit(data2010, labels2010)
    svm.fit(data2010, labels2010)
    logReg.fit(data2010, labels2010)

    gnbPrediction = gnb.predict(data2011)
    svmPrediction = svm.predict(data2011)
    logRegPrediction = logReg.predict(data2011)

    gnbAccuracy = accuracy(labels2011, gnbPrediction)
    svmAccuracy = accuracy(labels2011, svmPrediction)
    logRegAccuracy = accuracy(labels2011, logRegPrediction)

    confusionMatrix = metrics.confusion_matrix(labels2011, logRegPrediction)

    print "Results:"
    print "Gaussian Naive Bayes: " 
    print gnbAccuracy
    print "Support Vector Machine: " 
    print svmAccuracy
    print "Logistic Regression: " 
    print logRegAccuracy
    print confusionMatrix

    fh.write("Results:" + "\n")
    fh.write("Gaussian Naive Bayes: "  + "\n")
    fh.write(gnbAccuracy + "\n")
    fh.write("Support Vector Machine: "  + "\n")
    fh.write(svmAccuracy + "\n")
    fh.write("Logistic Regression: "  + "\n")
    fh.write(logRegAccuracy + "\n")
    for i in confusionMatrix:
        fh.write(str(i))
        fh.write("\n")
    fh.write("-------------------------------------------------\n")
    fh.write("\n\n")