machine-learning/data_loaders.py at master · anmaxvl/machine-learning · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
import numpy as np

import pandas.io.data as web
from pandas.io.api import read_csv

class StockDataLoader(object):
    pass


class TrafficDataLoader(object):
    def __init__(self, filename, max_norm):
        self._raw_data = None

        try:
            self._raw_data = read_csv(filename, header=0, index_col=0)
        except Exception as e:
            print str(e)
            return

        self._index = []
        self._values = []

        self._values = np.reshape(self._raw_data.values, (-1))
        self._index = self._raw_data.index

        self._normalize(max_norm)


    def _normalize(self, max_norm):
        logged = np.log(self._values)
        min_val = min(logged)
        max_val = max(logged)
        normalized = max_norm*(logged - min_val)/(max_val-min_val)
        self._values = normalized


    def eval_1st_diffs(self, start, n_steps):
        _1st_diffs = []

        for i in range(start, start+n_steps):
            if i < 1:
                _1st_diffs.append(0)
            else:
                _1st_diffs.append(self._values[i]-self._values[i-1])

        return _1st_diffs


    def eval_2nd_diffs(self, start, n_steps):
        _2nd_diffs = []

        for i in range(start, start+n_steps):
            if i < 2:
                _2nd_diffs.append(0)
            else:
                diff1 = self._values[i]-self._values[i-1]
                diff2 = self._values[i-1] - self._values[i-2]

                _2nd_diffs.append(diff1-diff2)

        return _2nd_diffs


    def get_rnn_input(self, config):
        start = config.start
        n_steps = config.n_steps
        use_1st_diffs = config.use_1st_diffs
        use_2nd_diffs = config.use_2nd_diffs
        window_size = config.window_size
        lag = config.lag

        seq_input = []

        seq_target = self._values[start+lag:start+lag+n_steps]

        for i in range(window_size):
            lagged_input = self._values[start:start+n_steps]
            seq_input.append(lagged_input)
            if use_1st_diffs:
                _1st_diffs = self.eval_1st_diffs(start+i, n_steps)
                seq_input.append(_1st_diffs)
            if use_2nd_diffs:
                _2nd_diffs = self.eval_2nd_diffs(start+i, n_steps)
                seq_input.append(_2nd_diffs)

        seq_input_np = np.array(seq_input).T
        seq_target_np = np.array([seq_target,]).T

        return seq_input_np, seq_target_np


    @property
    def index(self):
        return self._index

    @property
    def values(self):
        return self._values

    @property
    def raw_data(self):
        return self._raw_data