"""

def __init__(self, examples=None, attrs=None, attr_names=None, target=-1, inputs=None,

values=None, distance=mean_boolean_error, name='', source='', exclude=()):

"""

self.name = name

self.source = source

self.values = values

self.distance = distance

self.got_values_flag = bool(values)

# initialize .examples from string or list or data directory

if isinstance(examples, str):

self.examples = parse_csv(examples)

elif examples is None:

self.examples = parse_csv(open_data(name + '.csv').read())

else:

self.examples = examples

# attrs are the indices of examples, unless otherwise stated.

if self.examples is not None and attrs is None:

attrs = list(range(len(self.examples[0])))

self.attrs = attrs

# initialize .attr_names from string, list, or by default

if isinstance(attr_names, str):

self.attr_names = attr_names.split()

else:

self.attr_names = attr_names or attrs

self.set_problem(target, inputs=inputs, exclude=exclude)

def set_problem(self, target, inputs=None, exclude=()):

"""

self.target = self.attr_num(target)

exclude = list(map(self.attr_num, exclude))

if inputs:

self.inputs = remove_all(self.target, inputs)

else:

self.inputs = [a for a in self.attrs if a != self.target and a not in exclude]

if not self.values:

self.update_values()

self.check_me()

def check_me(self):

"""Check that my fields make sense."""

assert len(self.attr_names) == len(self.attrs)

assert self.target in self.attrs

assert self.target not in self.inputs

assert set(self.inputs).issubset(set(self.attrs))

if self.got_values_flag:

# only check if values are provided while initializing DataSet

list(map(self.check_example, self.examples))

def add_example(self, example):

"""Add an example to the list of examples, checking it first."""

self.check_example(example)

self.examples.append(example)

def check_example(self, example):

"""Raise ValueError if example has any invalid values."""

if self.values:

for a in self.attrs:

if example[a] not in self.values[a]:

raise ValueError('Bad value {} for attribute {} in {}'

.format(example[a], self.attr_names[a], example))

def attr_num(self, attr):

"""Returns the number used for attr, which can be a name, or -n .. n-1."""

if isinstance(attr, str):

return self.attr_names.index(attr)

elif attr < 0:

return len(self.attrs) + attr

else:

return attr

def update_values(self):

"""Recompute ``self.values`` (the list of distinct values per attribute) from the examples."""

self.values = list(map(unique, zip(*self.examples)))

def sanitize(self, example):

"""Return a copy of example, with non-input attributes replaced by None."""

return [attr_i if i in self.inputs else None for i, attr_i in enumerate(example)][:-1]

def classes_to_numbers(self, classes=None):

"""Converts class names to numbers."""

if not classes:

# if classes were not given, extract them from values

classes = sorted(self.values[self.target])

for item in self.examples:

item[self.target] = classes.index(item[self.target])

def remove_examples(self, value=''):

"""Remove examples that contain given value."""

self.examples = [x for x in self.examples if value not in x]

self.update_values()

def split_values_by_classes(self):

"""Split values into buckets according to their class."""

buckets = defaultdict(lambda: [])

target_names = self.values[self.target]

for v in self.examples:

item = [a for a in v if a not in target_names] # remove target from item

buckets[v[self.target]].append(item) # add item to bucket of its class

return buckets

def find_means_and_deviations(self):

"""

target_names = self.values[self.target]

feature_numbers = len(self.inputs)

item_buckets = self.split_values_by_classes()

means = defaultdict(lambda: [0] * feature_numbers)

deviations = defaultdict(lambda: [0] * feature_numbers)

for t in target_names:

# find all the item feature values for item in class t

features = [[] for _ in range(feature_numbers)]

for item in item_buckets[t]:

for i in range(feature_numbers):

features[i].append(item[i])

# calculate means and deviations fo the class

for i in range(feature_numbers):

means[t][i] = mean(features[i])

deviations[t][i] = stdev(features[i])

return means, deviations

def __repr__(self):

r"""

lines = [line for line in input.splitlines() if line.strip()]

"""

if hasattr(predict, 'predict'):

predict = predict.predict

examples = examples or dataset.examples

if len(examples) == 0:

return 0.0

right = 0

for example in examples:

desired = example[dataset.target]

output = predict(dataset.sanitize(example))

if output == desired:

right += 1

"""

if hasattr(predict, 'predict'):

predict = predict.predict

"""

examples = dataset.examples

if test_split is None:

train = examples[:start] + examples[end:]

val = examples[start:end]

else:

total_size = len(examples)

val_size = int(total_size * test_split)

train_size = total_size - val_size

train = examples[:train_size]

val = examples[train_size:total_size]

"""

errs = []

size = 1

while True:

errT, errV = cross_validation(learner, dataset, size, k, trials)

# check for convergence provided err_val is not empty

if errT and not np.isclose(errT[-1], errT, rtol=1e-6):

best_size = 0

min_val = np.inf

i = 0

while i < size:

if errs[i] < min_val:

min_val = errs[i]

best_size = i

i += 1

return learner(dataset, best_size)

errs.append(errV)

"""

k = k or len(dataset.examples)

if trials > 1:

trial_errT = 0

trial_errV = 0

for t in range(trials):

errT, errV = cross_validation(learner, dataset, size, k, trials)

trial_errT += errT

trial_errV += errV

return trial_errT / trials, trial_errV / trials

else:

fold_errT = 0

fold_errV = 0

n = len(dataset.examples)

examples = dataset.examples

random.shuffle(dataset.examples)

for fold in range(k):

train_data, val_data = train_test_split(dataset, fold * (n // k), (fold + 1) * (n // k))

dataset.examples = train_data

# pass `size` only to learners that take it (model selection); the

# plain learners used by e.g. compare() have a (dataset)-only signature

h = learner(dataset, size) if size is not None else learner(dataset)

fold_errT += err_ratio(h, dataset, train_data)

fold_errV += err_ratio(h, dataset, val_data)

# reverting back to original once test is completed

dataset.examples = examples

"""Return a list of (training-set size, mean accuracy) pairs, obtained by

if sizes is None:

sizes = list(range(2, len(dataset.examples) - trials, 2))

def score(learner, size):

random.shuffle(dataset.examples)

return cross_validation(learner, dataset, size, trials)

"""

most_popular = mode([e[dataset.target] for e in dataset.examples])

def predict(example):

"""Always return same result: the most popular from the training set."""

return most_popular

"""

def __init__(self, attr, attr_name=None, default_child=None, branches=None):

"""Initialize by saying what attribute this node tests."""

self.attr = attr

self.attr_name = attr_name or attr

self.default_child = default_child

self.branches = branches or {}

def __call__(self, example):

"""Given an example, classify it using the attribute and the branches."""

attr_val = example[self.attr]

if attr_val in self.branches:

return self.branches[attr_val](example)

else:

# return default class when attribute is unknown

return self.default_child(example)

def add(self, val, subtree):

"""Add a branch. If self.attr = val, go to the given subtree."""

self.branches[val] = subtree

def display(self, indent=0):

"""Print this subtree, showing the tested attribute and each branch, indented by ``indent``."""

name = self.attr_name

print('Test', name)

for (val, subtree) in self.branches.items():

print(' ' * 4 * indent, name, '=', val, '==>', end=' ')

subtree.display(indent + 1)

def __repr__(self):

"""A leaf of a decision tree holds just a result."""

def __init__(self, result):

self.result = result

def __call__(self, example):

return self.result

def display(self):

"""Print the result stored at this leaf."""

print('RESULT =', self.result)

def __repr__(self):

"""[Figure 18.5] Learn a decision tree by recursively splitting the examples on

target, values = dataset.target, dataset.values

def decision_tree_learning(examples, attrs, parent_examples=()):

if len(examples) == 0:

return plurality_value(parent_examples)

if all_same_class(examples):

return DecisionLeaf(examples[0][target])

if len(attrs) == 0:

return plurality_value(examples)

A = choose_attribute(attrs, examples)

tree = DecisionFork(A, dataset.attr_names[A], plurality_value(examples))

for (v_k, exs) in split_by(A, examples):

subtree = decision_tree_learning(exs, remove_all(A, attrs), examples)

tree.add(v_k, subtree)

return tree

def plurality_value(examples):

"""

popular = argmax_random_tie(values[target], key=lambda v: count(target, v, examples))

return DecisionLeaf(popular)

def count(attr, val, examples):

"""Count the number of examples that have example[attr] = val."""

return sum(e[attr] == val for e in examples)

def all_same_class(examples):

"""Are all these examples in the same target class?"""

class0 = examples[0][target]

return all(e[target] == class0 for e in examples)

def choose_attribute(attrs, examples):

"""Choose the attribute with the highest information gain."""

return argmax_random_tie(attrs, key=lambda a: information_gain(a, examples))

def information_gain(attr, examples):

"""Return the expected reduction in entropy from splitting by attr."""

def I(examples):

return information_content([count(target, v, examples) for v in values[target]])

n = len(examples)

remainder = sum((len(examples_i) / n) * I(examples_i) for (v, examples_i) in split_by(attr, examples))

return I(examples) - remainder

def split_by(attr, examples):

"""Return a list of (val, examples) pairs for each val of attr."""

return [(v, [e for e in examples if e[attr] == v]) for v in values[attr]]

"""

attrs = dataset.inputs

target = dataset.target

# the largest conjunction tried; the default lets a test pin down a single

# example, so a consistent list exists whenever the data is not contradictory

max_size = max_test_size or len(attrs)

def passes(example, test):

"""Does the example satisfy every literal of the (conjunctive) test?"""

return all(example[attr] == val for attr, val in test)

def find_examples(examples):

"""Find the smallest test selecting a non-empty subset of examples that

literals = sorted({(attr, e[attr]) for e in examples for attr in attrs}, key=str)

for size in range(1, max_size + 1):

for test in itertools.combinations(literals, size):

# a test may constrain each attribute at most once

if len({attr for attr, _ in test}) != size:

continue

matched = [e for e in examples if passes(e, test)]

if matched and len({e[target] for e in matched}) == 1:

return test, matched[0][target], matched

return None, None, None

def decision_list_learning(examples):

if not examples:

return [((), None)] # catch-all: the empty test matches any example

test, outcome, matched = find_examples(examples)

if test is None:

raise ValueError('DecisionListLearner: examples are not separable '

'(contradictory examples sharing identical attributes)')

return [(test, outcome)] + decision_list_learning([e for e in examples if e not in matched])

def predict(example):

"""Return the outcome of the first test the example satisfies."""

for test, outcome in predict.decision_list:

if passes(example, test):

return outcome

predict.decision_list = decision_list_learning(list(dataset.examples))

"""k-NearestNeighbor: the k nearest neighbors vote."""

def predict(example):

"""Find the k closest items, and have them vote for the best."""

# the enumerate() index is a tiebreaker so that equal-distance examples

# are never compared directly (examples may be numpy arrays, e.g. MNIST

# images, which are not orderable -> "truth value is ambiguous")

best = heapq.nsmallest(k, ((dataset.distance(e, example), i, e)

for i, e in enumerate(dataset.examples)))

return mode(e[dataset.target] for (d, i, e) in best)

"""

idx_i = dataset.inputs

idx_t = dataset.target

examples = dataset.examples

num_examples = len(examples)

# X transpose: the actual value of each input feature across the examples

X_col = [[example[i] for example in examples] for i in idx_i] # vertical columns of X

# add dummy

ones = [1 for _ in range(len(examples))]

X_col = [ones] + X_col

# initialize random weights

num_weights = len(idx_i) + 1

w = random_weights(min_value=-0.5, max_value=0.5, num_weights=num_weights)

for epoch in range(epochs):

err = []

# pass over all examples

for example in examples:

x = [1] + [example[i] for i in idx_i]

y = np.dot(w, x)

t = example[idx_t]

err.append(t - y)

# update weights

for i in range(len(w)):

w[i] = w[i] + learning_rate * (np.dot(err, X_col[i]) / num_examples)

def predict(example):

x = [1] + [example[i] for i in idx_i]

return np.dot(w, x)

"""

idx_i = dataset.inputs

idx_t = dataset.target

examples = dataset.examples

num_examples = len(examples)

# X transpose: the actual value of each input feature across the examples

X_col = [[example[i] for example in examples] for i in idx_i] # vertical columns of X

# add dummy

ones = [1 for _ in range(len(examples))]

X_col = [ones] + X_col

# initialize random weights

num_weights = len(idx_i) + 1

w = random_weights(min_value=-0.5, max_value=0.5, num_weights=num_weights)

for epoch in range(epochs):

err = []

h = []

# pass over all examples

for example in examples:

x = [1] + [example[i] for i in idx_i]

y = sigmoid(np.dot(w, x))

h.append(sigmoid_derivative(y))

t = example[idx_t]

err.append(t - y)

# update weights

for i in range(len(w)):

buffer = [x * y for x, y in zip(err, h)]

w[i] = w[i] + learning_rate * (np.dot(buffer, X_col[i]) / num_examples)

def predict(example):

x = [1] + [example[i] for i in idx_i]

return sigmoid(np.dot(w, x))

"""

if hidden_layer_sizes is None:

hidden_layer_sizes = [3]

i_units = len(dataset.inputs)

o_units = len(dataset.values[dataset.target])

# construct a network

raw_net = network(i_units, hidden_layer_sizes, o_units, activation)

learned_net = BackPropagationLearner(dataset, raw_net, learning_rate, epochs, activation)

def predict(example):

# input nodes

i_nodes = learned_net[0]

# activate input layer

for v, n in zip(example, i_nodes):

n.value = v

# forward pass

for layer in learned_net[1:]:

for node in layer:

inc = [n.value for n in node.inputs]

in_val = dot_product(inc, node.weights)

node.value = node.activation(in_val)

# hypothesis

o_nodes = learned_net[-1]

prediction = find_max_node(o_nodes)

return prediction

"""

# initialise weights

for layer in net:

for node in layer:

node.weights = random_weights(min_value=-0.5, max_value=0.5, num_weights=len(node.weights))

examples = dataset.examples

# As of now dataset.target gives an int instead of list,

# Changing dataset class will have effect on all the learners.

# Will be taken care of later.

o_nodes = net[-1]

i_nodes = net[0]

o_units = len(o_nodes)

idx_t = dataset.target

idx_i = dataset.inputs

n_layers = len(net)

inputs, targets = init_examples(examples, idx_i, idx_t, o_units)

for epoch in range(epochs):

# iterate over each example

for e in range(len(examples)):

i_val = inputs[e]

t_val = targets[e]

# activate input layer

for v, n in zip(i_val, i_nodes):

n.value = v

# forward pass

for layer in net[1:]:

for node in layer:

inc = [n.value for n in node.inputs]

in_val = dot_product(inc, node.weights)

node.value = node.activation(in_val)

# initialize delta

delta = [[] for _ in range(n_layers)]

# compute outer layer delta

# error for the MSE cost function

err = [t_val[i] - o_nodes[i].value for i in range(o_units)]

# calculate delta at output

if node.activation == sigmoid:

delta[-1] = [sigmoid_derivative(o_nodes[i].value) * err[i] for i in range(o_units)]

elif node.activation == relu:

delta[-1] = [relu_derivative(o_nodes[i].value) * err[i] for i in range(o_units)]

elif node.activation == tanh:

delta[-1] = [tanh_derivative(o_nodes[i].value) * err[i] for i in range(o_units)]

elif node.activation == elu:

delta[-1] = [elu_derivative(o_nodes[i].value) * err[i] for i in range(o_units)]

elif node.activation == leaky_relu:

delta[-1] = [leaky_relu_derivative(o_nodes[i].value) * err[i] for i in range(o_units)]

else:

return ValueError("Activation function unknown.")

# backward pass

h_layers = n_layers - 2

for i in range(h_layers, 0, -1):

layer = net[i]

h_units = len(layer)

nx_layer = net[i + 1]

# weights from each ith layer node to each i + 1th layer node

w = [[node.weights[k] for node in nx_layer] for k in range(h_units)]

if activation == sigmoid:

delta[i] = [sigmoid_derivative(layer[j].value) * dot_product(w[j], delta[i + 1])

for j in range(h_units)]

elif activation == relu:

delta[i] = [relu_derivative(layer[j].value) * dot_product(w[j], delta[i + 1])

for j in range(h_units)]

elif activation == tanh:

delta[i] = [tanh_derivative(layer[j].value) * dot_product(w[j], delta[i + 1])

for j in range(h_units)]

elif activation == elu:

delta[i] = [elu_derivative(layer[j].value) * dot_product(w[j], delta[i + 1])

for j in range(h_units)]

elif activation == leaky_relu:

delta[i] = [leaky_relu_derivative(layer[j].value) * dot_product(w[j], delta[i + 1])

for j in range(h_units)]

else:

return ValueError("Activation function unknown.")

# update weights

for i in range(1, n_layers):

layer = net[i]

inc = [node.value for node in net[i - 1]]

units = len(layer)

for j in range(units):

layer[j].weights = vector_add(layer[j].weights,

scalar_vector_product(learning_rate * delta[i][j], inc))

"""Logistic Regression, NO hidden layer"""

i_units = len(dataset.inputs)

o_units = len(dataset.values[dataset.target])

hidden_layer_sizes = []

raw_net = network(i_units, hidden_layer_sizes, o_units)

learned_net = BackPropagationLearner(dataset, raw_net, learning_rate, epochs)

def predict(example):

o_nodes = learned_net[1]

# forward pass

for node in o_nodes:

in_val = dot_product(example, node.weights)

node.value = node.activation(in_val)

# hypothesis

return find_max_node(o_nodes)

"""

def __init__(self, activation=sigmoid, weights=None, inputs=None):

self.weights = weights or []

self.inputs = inputs or []

self.value = None

"""

layers_sizes = [input_units] + hidden_layer_sizes + [output_units]

net = [[NNUnit(activation) for _ in range(size)] for size in layers_sizes]

n_layers = len(net)

# make connection

for i in range(1, n_layers):

for n in net[i]:

for k in net[i - 1]:

n.inputs.append(k)

n.weights.append(0)

"""Split examples into input and target dicts keyed by example index.

inputs, targets = {}, {}

for i, e in enumerate(examples):

# input values of e

inputs[i] = [e[i] for i in idx_i]

if o_units > 1:

# one-hot representation of e's target

t = [0 for i in range(o_units)]

t[e[idx_t]] = 1

targets[i] = t

else:

# target value of e

targets[i] = [e[idx_t]]

"""Support Vector Classifier trained in dual form by solving a quadratic

def __init__(self, kernel=linear_kernel, C=1.0, verbose=False):

self.kernel = kernel

self.C = C # hyper-parameter

self.sv_idx, self.sv, self.sv_y = np.zeros(0), np.zeros(0), np.zeros(0)

self.alphas = np.zeros(0)

self.w = None

self.b = 0.0 # intercept

self.verbose = verbose

def fit(self, X, y):

"""

# In QP formulation (dual): m variables, 2m+1 constraints (1 equation, 2m inequations)

self.solve_qp(X, y)

sv = self.alphas > 1e-5

self.sv_idx = np.arange(len(self.alphas))[sv]

self.sv, self.sv_y, self.alphas = X[sv], y[sv], self.alphas[sv]

if self.kernel == linear_kernel:

self.w = np.dot(self.alphas * self.sv_y, self.sv)

for n in range(len(self.alphas)):

self.b += self.sv_y[n]

self.b -= np.sum(self.alphas * self.sv_y * self.K[self.sv_idx[n], sv])

self.b /= len(self.alphas)

return self

def solve_qp(self, X, y):

"""

m = len(y) # m = n_samples

self.K = self.kernel(X) # gram matrix

P = self.K * np.outer(y, y)

q = -np.ones(m)

lb = np.zeros(m) # lower bounds

ub = np.ones(m) * self.C # upper bounds

A = y.astype(np.float64) # equality matrix

b = np.zeros(1) # equality vector

# imported lazily: SVM training is the only thing that needs the optional

# qpsolvers/cvxopt dependency, so the rest of the module (trees, kNN, naive

# Bayes, neural nets, ...) stays importable without it

from qpsolvers import solve_qp

self.alphas = solve_qp(P, q, A=A, b=b, lb=lb, ub=ub, solver='cvxopt',

sym_proj=True, verbose=self.verbose)

def predict_score(self, X):

"""

if self.w is None:

return np.dot(self.alphas * self.sv_y, self.kernel(self.sv, X)) + self.b

return np.dot(X, self.w) + self.b

def predict(self, X):

"""

"""Support Vector Regressor trained in dual form by solving a quadratic

def __init__(self, kernel=linear_kernel, C=1.0, epsilon=0.1, verbose=False):

self.kernel = kernel

self.C = C # hyper-parameter

self.epsilon = epsilon # epsilon insensitive loss value

self.sv_idx, self.sv = np.zeros(0), np.zeros(0)

self.alphas_p, self.alphas_n = np.zeros(0), np.zeros(0)

self.w = None

self.b = 0.0 # intercept

self.verbose = verbose

def fit(self, X, y):

"""

# In QP formulation (dual): m variables, 2m+1 constraints (1 equation, 2m inequations)

self.solve_qp(X, y)

sv = np.logical_or(self.alphas_p > 1e-5, self.alphas_n > 1e-5)

self.sv_idx = np.arange(len(self.alphas_p))[sv]

self.sv, sv_y = X[sv], y[sv]

self.alphas_p, self.alphas_n = self.alphas_p[sv], self.alphas_n[sv]

if self.kernel == linear_kernel:

self.w = np.dot(self.alphas_p - self.alphas_n, self.sv)

for n in range(len(self.alphas_p)):

self.b += sv_y[n]

self.b -= np.sum((self.alphas_p - self.alphas_n) * self.K[self.sv_idx[n], sv])

self.b -= self.epsilon

self.b /= len(self.alphas_p)

return self

def solve_qp(self, X, y):

"""

#

m = len(y) # m = n_samples

self.K = self.kernel(X) # gram matrix

P = np.vstack((np.hstack((self.K, -self.K)), # alphas_p, alphas_n

np.hstack((-self.K, self.K)))) # alphas_n, alphas_p

q = np.hstack((-y, y)) + self.epsilon

lb = np.zeros(2 * m) # lower bounds

ub = np.ones(2 * m) * self.C # upper bounds

A = np.hstack((np.ones(m), -np.ones(m))) # equality matrix

b = np.zeros(1) # equality vector

from qpsolvers import solve_qp # lazy: see SVC.solve_qp above

alphas = solve_qp(P, q, A=A, b=b, lb=lb, ub=ub, solver='cvxopt',