%% Machine Learning Online Class - Exercise 2: Logistic Regression

%

%  Instructions

%  ------------

%

%  This file contains code that helps you get started on the logistic

%  regression exercise. You will need to complete the following functions

%  in this exericse:

%

%     sigmoid.m

%     costFunction.m

%     predict.m

%     costFunctionReg.m

%

%  For this exercise, you will not need to change any code in this file,

%  or any other files other than those mentioned above.

%

%% Initialization

clear ; close all; clc

%% Load Data

%  The first two columns contains the exam scores and the third column

%  contains the label.

data = load('ex2data1.txt');

X = data(:, [1, 2]); y = data(:, 3);

%% ==================== Part 1: Plotting ====================

%  We start the exercise by first plotting the data to understand the

%  the problem we are working with.

fprintf(['Plotting data with + indicating (y = 1) examples and o ' ...

         'indicating (y = 0) examples.\n']);

plotData(X, y);

% Put some labels

hold on;

% Labels and Legend

xlabel('Exam 1 score')

ylabel('Exam 2 score')

% Specified in plot order

legend('Admitted', 'Not admitted')

hold off;

fprintf('\nProgram paused. Press enter to continue.\n');

pause;

%% ============ Part 2: Compute Cost and Gradient ============

%  In this part of the exercise, you will implement the cost and gradient

%  for logistic regression. You neeed to complete the code in

%  costFunction.m

%  Setup the data matrix appropriately, and add ones for the intercept term

[m, n] = size(X);

% Add intercept term to x and X_test

X = [ones(m, 1) X];

% Initialize fitting parameters

initial_theta = zeros(n + 1, 1);

% Compute and display initial cost and gradient

[cost, grad] = costFunction(initial_theta, X, y);

fprintf('Cost at initial theta (zeros): %f\n', cost);

fprintf('Expected cost (approx): 0.693\n');

fprintf('Gradient at initial theta (zeros): \n');

fprintf(' %f \n', grad);

fprintf('Expected gradients (approx):\n -0.1000\n -12.0092\n -11.2628\n');

% Compute and display cost and gradient with non-zero theta

test_theta = [-24; 0.2; 0.2];

[cost, grad] = costFunction(test_theta, X, y);

fprintf('\nCost at test theta: %f\n', cost);

fprintf('Expected cost (approx): 0.218\n');

fprintf('Gradient at test theta: \n');

fprintf(' %f \n', grad);

fprintf('Expected gradients (approx):\n 0.043\n 2.566\n 2.647\n');

fprintf('\nProgram paused. Press enter to continue.\n');

pause;

%% ============= Part 3: Optimizing using fminunc  =============

%  In this exercise, you will use a built-in function (fminunc) to find the

%  optimal parameters theta.

%  Set options for fminunc

options = optimset('GradObj', 'on', 'MaxIter', 400);

%  Run fminunc to obtain the optimal theta

%  This function will return theta and the cost

[theta, cost] = fminunc(@(t)(costFunction(t, X, y)), initial_theta, options);

% Print theta to screen

fprintf('Cost at theta found by fminunc: %f\n', cost);

fprintf('Expected cost (approx): 0.203\n');

fprintf('theta: \n');

fprintf(' %f \n', theta);

fprintf('Expected theta (approx):\n');

fprintf(' -25.161\n 0.206\n 0.201\n');

% Plot Boundary

plotDecisionBoundary(theta, X, y);

% Put some labels

hold on;

% Labels and Legend

xlabel('Exam 1 score')

ylabel('Exam 2 score')

% Specified in plot order

legend('Admitted', 'Not admitted')

hold off;

fprintf('\nProgram paused. Press enter to continue.\n');

pause;

%% ============== Part 4: Predict and Accuracies ==============

%  After learning the parameters, you'll like to use it to predict the outcomes

%  on unseen data. In this part, you will use the logistic regression model

%  to predict the probability that a student with score 45 on exam 1 and

%  score 85 on exam 2 will be admitted.

%

%  Furthermore, you will compute the training and test set accuracies of

%  our model.

%

%  Your task is to complete the code in predict.m

%  Predict probability for a student with score 45 on exam 1

%  and score 85 on exam 2 

prob = sigmoid([1 45 85] * theta);

fprintf(['For a student with scores 45 and 85, we predict an admission ' ...

         'probability of %f\n'], prob);

fprintf('Expected value: 0.775 +/- 0.002\n\n');

% Compute accuracy on our training set

p = predict(theta, X);

fprintf('Train Accuracy: %f\n', mean(double(p == y)) * 100);

fprintf('Expected accuracy (approx): 89.0\n');

fprintf('\n');

function plotData(X, y)

%PLOTDATA Plots the data points X and y into a new figure

%   PLOTDATA(x,y) plots the data points with + for the positive examples

%   and o for the negative examples. X is assumed to be a Mx2 matrix.

% Create New Figure

  figure; hold on;

% Instructions: Plot the positive and negative examples on a

%               2D plot, using the option 'k+' for the positive

%               examples and 'ko' for the negative examples.

  pos = find(y==1); neg = find(y == 0);

  plot(X(pos, 1), X(pos, 2), 'k+','LineWidth', 2, 'MarkerSize', 7);

  plot(X(neg, 1), X(neg, 2), 'ko', 'MarkerFaceColor', 'y', 'MarkerSize', 7);

  hold off;

end

function g = sigmoid(z)

%SIGMOID Compute sigmoid function

%   g = SIGMOID(z) computes the sigmoid of z.

% You need to return the following variables correctly

  g = ones(size(z)) ./ (1 + exp(-1 * z));

end

function [J, grad] = costFunction(theta, X, y)

  %COSTFUNCTION Compute cost and gradient for logistic regression

  %   J = COSTFUNCTION(theta, X, y) computes the cost of using theta as the

  %   parameter for logistic regression and the gradient of the cost

  %   w.r.t. to the parameters.

  % Initialize some useful values

  m = length(y); % number of training examples

  % You need to return the following variables correctly

  J = 0;

  grad = zeros(size(theta));

  % Instructions: Compute the cost of a particular choice of theta.

  %               You should set J to the cost.

  %               Compute the partial derivatives and set grad to the partial

  %               derivatives of the cost w.r.t. each parameter in theta

  %

  % Note: grad should have the same dimensions as theta

  %

  %  % use intrator to compue J

  %  for i = 1:m

  %    theta_X = X(i,:) * theta;

  %    h = 1 / (1 + exp(-1 * theta_X));

  %    J += y(i) * log(h) + (1 - y(i)) * log(1 - h);

  %  end

  %  J /= -1 * m;

  % use matrix to compute gradient

  h = sigmoid(X * theta);

  J = (y' * log(h) + (1 - y)' * log(1 - h)) / (-1 * m);

  #J /= -1 * m;

  grad = X' * (h - y) / m;

end

 from __future__ import division

 import numpy as np

 import random

 import matplotlib.pyplot as plt

 def train(X, Y, iterateNum=10000000, alpha=0.003):

     '''

     :param X: 训练样本的特征集

     :param Y: 训练样本的标签

     :param iterateNum: 梯度下降的迭代次数

     :param alpha: 学习率

     :return:theta

     '''

     m, n = np.shape(X)

     theta = np.zeros((n + 1, 1))

     # 在第一列添加x0

     X_new = np.c_[np.ones(m), X]

     for i in range(iterateNum):

         m = np.shape(X_new)[0]

         h = h_function(X_new, theta)

         theta -= alpha * (np.dot(X_new.T, h - Y) / m)

         if i % 100000 == 0:

             print('\t---------iter=' + str(i) + ', J(θ)=' + str(J_function(X_new, Y, theta)))

     print( str(J_function(X_new, Y, theta)))

     return theta

 def h_function(X, theta):

     return sigmoid(np.dot(X, theta))

 def sigmoid(X):

     return 1 / (1 + np.exp(-X ))

 # 计算J(θ)

 def J_function(X, Y, theta):

     h = h_function(X, theta)

     J_1 = np.dot(Y.T, np.log(h))

     J_2 = np.dot(1 - Y.T, np.log(1 - h))

     m = np.shape(X)[0]

     J = (-1 / m) * (J_1 + J_2)

     return J

 def predict(x, theta):

     if h_function(x, theta) >= 0.5:

         return 1

     else:

         return 0

 # 归一化处理

 def normalization(X):

     m, n = np.shape(X)

     X_new = np.zeros((m, n))

     for j in range(n):

         max = np.max(X[:,j])

         min = np.min(X[:,j])

         d_value = max - min

         for i in range(m):

             X_new[i, j] = (X[i, j] - min) / d_value

     return X_new

 def plot_datas(X, Y, theta):

     plt.figure()

     # 绘制分隔直线 g = 0

     x1 = [0, 1]

     x2 = [(-1 / theta[2]) * (theta[0] + theta[1] * x1[0]),

           (-1 / theta[2]) * (theta[0] + theta[1] * x1[1])]

     plt.xlabel('x1')

     plt.ylabel('x2')

     plt.plot(x1, x2, color='b')

     # 绘制数据点

     admit_x1, admit_x2 = [],[]

     not_admit_x1, not_admit_x2 = [],[]

     for i in range(len(X)):

         if (Y[i] == 1):

             admit_x1.append(X[i][0])

             admit_x2.append(X[i][1])

         else:

             not_admit_x1.append(X[i][0])

             not_admit_x2.append(X[i][1])

     plt.scatter(admit_x1, admit_x2, color='g')

     plt.scatter(not_admit_x1, not_admit_x2, marker='x', color='r')

     plt.legend(['logistic line', 'Admitted', 'Not admitted'])

     plt.show()

 if __name__ == '__main__':

     train_datas = np.loadtxt('ex2data1.txt', delimiter=',')

     X = train_datas[:,[0, 1]]

     X = normalization(X)

     Y = train_datas[:,[2]]

     theta = train(X, Y)

     print(theta)

     plot_datas(X, Y, theta)

 from sklearn.linear_model import LogisticRegression

 import numpy as np

 if __name__ == '__main__':

     train_datas = np.loadtxt("ex2data1.txt", delimiter=',')

     X_train = train_datas[:,[0, 1]]

     Y_train = train_datas[:,[2]]

     logistic = LogisticRegression()

     logistic.fit(X_train, Y_train)

     theta = [logistic.intercept_[0], logistic.coef_[0]]

     print(theta)