de[v|b]log

ShellScript, Coffee, iOS/OSX Dev
Origin: Himajinworks.
About.

やったこと

  • まずは多層パーセプトロン(Multi Layer Perceptron; MLP)を試すということで、以下の参考記事/サンプルコードをベースにして実装を行った。
  • 次に隠れ層の層の数を増やせるようにコードを書き換えた。
    • 下記コードの114行目の配列に値を追加することで隠れ層のノード数を定義して層を追加することができる。
    • chainer.ChainListを利用した。
  • 問題
    • 1-Bit XOR
    • 4-Bit Parity

参考ページ

その他

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
#encoding: utf-8

import numpy as np
import chainer
from chainer import Function, gradient_check, Variable, optimizers, serializers, utils
from chainer import Link, Chain, ChainList
import chainer.functions as F
import chainer.links as L
import itertools
from collections import namedtuple

class MLP(ChainList):
    def __init__(self, nobias_flag, config):
        # Initialize MLP structure (ChainList)
        super(MLP, self).__init__()

        # Add input - hidden layers
        for n1, n2 in config.hidden_pairs:
            self.add_link(L.Linear(n1, n2, nobias = nobias_flag))

        # Add output layer
        self.add_link(
            L.Linear(
                config.n_hiddens[-1],
                config.n_output,
                nobias = nobias_flag
            )
        )

        # Set some params
        self.nobias_flag = nobias_flag

    def __call__(self, x):
        # Compute forward direction
        h   = x
        idx = 0
        while idx < self.__len__() - 1:
            layer = self.__getitem__(idx)
            idx += 1
            h = F.sigmoid(layer(h))

        output_layer = self.__getitem__(idx)
        return output_layer(h)

    def dump(self):
        print(self.hidden_layer.W.data)
        if not self.nobias_flag:
            print(self.hidden_layer.b.data)

        print(self.output_layer.W.data)
        if not self.nobias_flag:
            print(self.output_layer.b.data)

class Classifier(Chain):
    def __init__(self, predictor):
        super(Classifier, self).__init__(predictor = predictor)

    def __call__(self, x, t):
        y = self.predictor(x)
        self.loss = F.softmax_cross_entropy(y, t)
        self.accuracy = F.accuracy(y, t)
        return self.loss, self.accuracy

def gen_config(n_input, n_hiddens, n_output):
    # Construct pair of hidden layer config
    a, b = itertools.tee([n_input] + n_hiddens)
    next(b, None)
    hidden_pairs = itertools.izip(a, b)

    MLPConfig = namedtuple('MLPConfig', 'n_input n_hiddens n_output hidden_pairs')
    config = MLPConfig(n_input, n_hiddens, n_output, hidden_pairs)
    return config

# Test task
def problem_set(name):
    Dataset = namedtuple('Dataset', 'data label')
    xor_dataset = Dataset(
        [
            [0, 0], [1, 0], [0, 1], [1, 1]
        ], [
            0, 1, 1, 0
        ]
    )

    parity_4bits_dataset = Dataset(
        [
            [0, 0, 0, 0], [0, 0, 0, 1], [0, 0, 1, 0], [0, 0, 1, 1],
            [0, 1, 0, 0], [0, 1, 0, 1], [0, 1, 1, 0], [0, 1, 1, 1],
            [1, 0, 0, 0], [1, 0, 0, 1], [1, 0, 1, 0], [1, 0, 1, 1],
            [1, 1, 0, 0], [1, 1, 0, 1], [1, 1, 1, 0], [1, 1, 1, 1]
        ], [
            0, 1, 1, 0,
            1, 0, 0, 1,
            1, 0, 0, 1,
            0, 1, 1, 0
        ]
    )

    dataset = Dataset([], [])
    if name == 'xor':
        dataset = xor_dataset
    else:
        dataset = parity_4bits_dataset

    np_data = Variable(np.array(dataset.data, dtype = np.float32))
    np_label = Variable(np.array(dataset.label, dtype = np.int32))
    return np_data, np_label

# Prepare problem
x, t = problem_set('parity')

# Configuration for each layer dimensions
input_dim   = len(x.data[0])
hidden_dims = [20]
output_dim  = 2
config      = gen_config(input_dim, hidden_dims, output_dim)

# Create model and optimizer
model     = Classifier(MLP(False, config))
#optimizer = optimizers.Adam()
optimizer = optimizers.AdaDelta()
optimizer.setup(model)

loss_value = 100000
n_iteration = 0
while loss_value > 1e-5:
    # Training
    model.zerograds()
    loss, accuracy = model(x, t)
    loss.backward()
    optimizer.update()

    n_iteration += 1
    if n_iteration % 1000 == 0:
        # Show progress
        y = F.softmax(model.predictor(x))

        print("Summary: iter = {0}, loss = {1}, accuracy = {2}".format(n_iteration, loss.data, accuracy.data))
        print("Output: {0}".format(y.data))
        print("Result: {0}".format(y.data.argmax(1))