refactoring and cleaning

.idea/SequenceGenerator.iml

+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$">
+      <excludeFolder url="file://$MODULE_DIR$/venv" />
+      <excludeFolder url="file://$MODULE_DIR$/venv1" />
+    </content>
+    <orderEntry type="jdk" jdkName="Python 3.5" jdkType="Python SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+  <component name="TestRunnerService">
+    <option name="PROJECT_TEST_RUNNER" value="Unittests" />
+  </component>
+</module>
\ No newline at end of file

.idea/misc.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.5" project-jdk-type="Python SDK" />
+</project>
\ No newline at end of file

.idea/modules.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/SequenceGenerator.iml" filepath="$PROJECT_DIR$/.idea/SequenceGenerator.iml" />
+    </modules>
+  </component>
+</project>
\ No newline at end of file

.idea/vcs.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>
\ No newline at end of file

detectors/adwin_detector.py

-import numpy
+import numpy as np
 import math
 from adwin_win_list import AdWinList
 from detector import ChangeDetector
@@ -29,15 +29,14 @@ class AdwinDetector(ChangeDetector):
         else:
             return 0;
 
-    def update(self, new_signal_value):
-        super(AdwinDetector, self).update(new_signal_value)
-        self.insertElement(new_signal_value)
+    def update(self, new_value):
+        super(AdwinDetector, self).update(new_value)
+        x = np.mean(new_value)
+        self.insertElement(x)
         self.compressBuckets()
         self.est_ = self.getEstimation()
 
-        #return self.checkDrift()
-
-    def check_stopping_rules(self, new_signal_value):
+    def check_stopping_rules(self, new_value):
         self.rules_triggered = False
         if self.checkDrift():
             self.rules_triggered = True
@@ -53,23 +52,6 @@ class AdwinDetector(ChangeDetector):
             self.bucketNumber=0;
             self.est_ = 0
 
-    def printInfo(self):
-        it = self.bucketList.tail
-        if it is None:
-            print("It None")
-
-        i = self.lastBucketRow
-
-        while True:
-            for k in range(it.size-1, -1, -1):
-                print(str(i)+" ["+str(it.sum[k])+" de "+str(self.bucketSize(i))+"],")
-
-            print()
-            it = it.prev
-            i -= 1
-            if it is None:
-                break;
-
     def length(self):
         return self.W_
 

detectors/cusum_detector.py

@@ -10,18 +10,16 @@ class CusumDetector(ChangeDetector):
         self.mean_ = 0
         self.sum_ = 0
         self.n = 0
-        #self.p_x_ = 0
 
-    def update(self, new_signal_value):
-        super(CusumDetector, self).update(new_signal_value)
-        x = np.mean(new_signal_value)
+    def update(self, new_value):
+        super(CusumDetector, self).update(new_value)
+        x = new_value
         self.n += 1
         self.mean_ = self.mean_ + (x - self.mean_) / self.n;
         self.sum_ = self.sum_ + x - self.mean_ - self.delta;
 
-    def check_stopping_rules(self, new_signal_value):
+    def check_stopping_rules(self, new_value):
         self.rules_triggered = False
-        #print("sum:", np.sum(self.sum_))
         if np.abs(np.sum(self.sum_)) > self.lambd:
             self.rules_triggered = True
             self.n = 0

detectors/detector.py

@@ -5,14 +5,14 @@ from collections import defaultdict
 class ChangeDetector(object):
     def __init__(self):
         self.rules_triggered = False
-        self.signal_size = 0
-        self.signal = []
+        self.sequence_size = 0
+        self.sequence = []
 
-    def update(self, new_signal_value):
-        self.signal.append(new_signal_value)
-        self.signal_size += 1
+    def update(self, new_value):
+        self.sequence.append(new_value)
+        self.sequence_size += 1
 
-    def check_stopping_rules(self, new_signal_value):
+    def check_stopping_rules(self, new_value):
         pass
 
     def get_parameters(self):
@@ -23,36 +23,36 @@ class ChangeDetector(object):
 
         return parameters_dict
 
-    def step(self, new_signal_value):
-        self.update(new_signal_value)
-        self.check_stopping_rules(new_signal_value)
+    def step(self, new_value):
+        self.update(new_value)
+        self.check_stopping_rules(new_value)
 
         return self.get_parameters()
 
 class OnlineSimulator(object):
-    def __init__(self, change_detector, signal):
-        self.signal = signal
+    def __init__(self, change_detector, sequence):
+        self.sequence = sequence
         self.change_detector = change_detector
-        self.signal_size = len(signal)
-        self.stops = []
+        self.sequence_size = len(sequence)
+        self.detected_change_points = []
 
     def get_detected_changes(self):
-        return self.stops
+        return self.detected_change_points
 
     def run(self, plot=True, **kwargs):
-        signal = self.signal
+        sequence = self.sequence
         detector = self.change_detector
 
         parameters_history = defaultdict(list)
 
-        for i, value in enumerate(signal):
+        for i, value in enumerate(sequence):
             res = detector.step(value)
 
             for k, v in res.items():
                 parameters_history[k].append(v)
 
             if detector.rules_triggered is True:
-                self.stops.append(i)
+                self.detected_change_points.append(i)
 
         def dict_to_arrays(ddict):
             new_dict = {}
@@ -68,8 +68,8 @@ class OnlineSimulator(object):
 
         return detector.rules_triggered
 
-    def display_results(self, signal_name='Sequence', **kwargs):
-        signal = self.signal
+    def display_results(self, sequence_name='Sequence', **kwargs):
+        sequence = self.sequence
         detector = self.change_detector
         parameters_history = self.parameters_history
 
@@ -77,32 +77,32 @@ class OnlineSimulator(object):
         fig, axes = plt.subplots(nrows=plotcount, ncols=1, sharex=True,
                                  figsize=(12, plotcount*3))
 
-        # Plot the signal
+        # Plot the sequence
         if plotcount > 1:
             ax = axes[0]
         elif plotcount == 1:
             ax = axes
 
-        ax.plot(signal, 'b.')
-        ax.plot(signal, 'b-', alpha=0.15)
+        ax.plot(sequence, 'b.')
+        ax.plot(sequence, 'b-', alpha=0.25)
 
-        ax.set_title(signal_name)
+        ax.set_title(sequence_name)
 
         ax.set_ylim(
-            np.nanmin(signal)*.5,
-            np.nanmax(signal)*1.5)
-        ax.set_xlim(0, len(signal))
+            np.nanmin(sequence)*.5,
+            np.nanmax(sequence)*1.5)
+        ax.set_xlim(0, len(sequence))
         xl = ax.get_xticks()
-        ticks = xl - int(2/3 * len(signal))
+        ticks = xl - int(2/3 * len(sequence))
 
         ax.set_xticklabels(ticks)
 
-        # Plot a horizontal line where the stop_point is indicated
-        for s in self.stops:
+        # Plot a horizontal line where the change_point is detected
+        for s in self.detected_change_points:
             ax.vlines(x=s, ymin=0, ymax=ax.get_ylim()[1],
                   colors='r', linestyles='dotted')
 
-        # Plot each
+        # Plot each parameter
         for ii, (res_name, res_values) in enumerate(parameters_history.items()):
             ax = axes[ii+1]
             ax.plot(res_values, 'g.', alpha=0.7)
@@ -111,8 +111,8 @@ class OnlineSimulator(object):
             # ax.set_ylim(
             #     np.nanmin(res_values)*0.5,
             #     np.nanmax(res_values)*1.5)
-            for s in self.stops:
-                ax.vlines(x=s, ymin=0, ymax=ax.get_ylim()[1],
+            for p in self.detected_change_points:
+                ax.vlines(x=p, ymin=0, ymax=ax.get_ylim()[1],
                       colors='r', linestyles='dotted')
 
         plt.show()

detectors/mean_detector.py

@@ -8,20 +8,22 @@ class MeanDetector(ChangeDetector):
         self.threshold = threshold
         self.total_val = 0
         self.diff_ = np.nan
+        self.n = 0
 
-    def update(self, new_signal_value):
-        super(MeanDetector, self).update(new_signal_value)
-        self.total_val += new_signal_value
-        self.mean_ = self.total_val / self.signal_size
-        self.diff_ = np.absolute(self.mean_ - new_signal_value)
+    def update(self, new_value):
+        super(MeanDetector, self).update(new_value)
+        self.total_val += new_value
+        self.n += 1
+        self.mean_ = self.total_val / self.n
+        self.diff_ = np.absolute(self.mean_ - new_value)
 
-    def check_stopping_rules(self, new_signal_value):
+    def check_stopping_rules(self, new_value):
         threshold_level = self.mean_ * self.threshold
         self.rules_triggered = False
 
         if self.diff_ > threshold_level:
             self.rules_triggered = True
             self.total_val = 0
-            self.signal_size = 0
+            self.n = 0
             self.mean_ = 0
             self.diff_ = 0

detectors/page_hinkley_detector.py

-import matplotlib.pyplot as plt
 import numpy as np
 import scipy as sp
 from scipy import signal
@@ -14,31 +13,19 @@ class PageHinkleyDetector(ChangeDetector):
         self.x_mean_ = 0
         self.sum_ = 0
         self.n = 0
-        self.sig = []
-        self.signa = []
         self.rules_triggered = False
 
-    def update(self, new_signal_value):
-        super(PageHinkleyDetector, self).update(new_signal_value)
-        self.sig.append(new_signal_value)
-        self.signa.append(new_signal_value)
+    def update(self, new_value):
+        super(PageHinkleyDetector, self).update(new_value)
         self.n += 1
-        #if self.n % 10 == 0:
-        #    self.sig = sp.signal.medfilt(self.sig,5).tolist()
-        #    self.signa = sp.signal.medfilt(self.signa,5).tolist()
-
-        x = np.mean(new_signal_value)
+        x = new_value
         self.x_mean_ = self.x_mean_ + (x - self.x_mean_) / self.n
-        self.x_mean_ = np.mean(self.sig)
         self.sum_ = self.sum_ * self.alpha + x - self.x_mean_ - self.delta
 
-    def check_stopping_rules(self, new_signal_value):
+    def check_stopping_rules(self, new_value):
         self.rules_triggered = False
-        #if self.n % 5 == 0 and np.abs(self.sum_) > self.lambd:
         if np.abs(self.sum_) > self.lambd:
             self.rules_triggered = True
-            plt.plot(self.sig)
             self.n = 0
             self.x_mean_ = 0
             self.sum_ = 0
-            self.sig = []

detectors/zscore_detector.py

@@ -10,9 +10,8 @@ class ZScoreDetector(ChangeDetector):
     def __init__(self, window_size = 100, threshold=0.05):
         super( ZScoreDetector, self ).__init__()
         self.threshold = threshold
-        #self.signal = []
         self.window_size = window_size
-        self.k = 0  # total signal_size
+        self.k = 0  # total seq_size
         self.g_mean_ = 0.0  # global mean
         self.s_ = 0.0  # for Welford's method. variance = s / (k + 1)
         self.window = deque(maxlen = window_size)
@@ -20,39 +19,36 @@ class ZScoreDetector(ChangeDetector):
         self.window_mean_ = 0.0
         self.g_std_ = 0.0
 
-    def update(self, new_signal_value):
-        super(ZScoreDetector, self).update(new_signal_value)
+    def update(self, new_value):
+        super(ZScoreDetector, self).update(new_value)
 
-        #self.signal = sp.signal.medfilt(self.signal,5).tolist()
-        x = new_signal_value
+        x = new_value
         self.window.append(x)
-        x = np.mean(x)
 
         # Calculate global statistics using welford's method
-        oldm = self.g_mean_
-        newm = oldm + (x - oldm) / (self.k + 1)
-        s = self.s_ + (x - newm) * (x - oldm)
+        old_mean = self.g_mean_
+        new_mean = old_mean + (x - old_mean) / (self.k + 1)
+        s = self.s_ + (x - new_mean) * (x - new_mean)
 
-        g_mean_ = newm  # Global mean
+        g_mean_ = new_mean  # Global mean
         g_std = np.sqrt(s / (self.k+1))  # Global std
 
         w_mean = np.mean(self.window)  # Window mean
         w_std = np.std(self.window)  # Window std
         self.window_mean_ = w_mean
 
-        std_diff = (g_std - w_std) / g_std
-        SE = g_std / np.sqrt(self.window_size)
-        mean_diff = (g_mean_ - w_mean) / g_mean_
+        self.std_diff_ = (g_std - w_std) / g_std
+        self.SE_ = g_std / np.sqrt(self.window_size)
+        self.mean_diff_ = (g_mean_ - w_mean) / g_mean_
 
-        self.z_score_ = (w_mean - g_mean_) / SE
-        #self.z_score_ = (w_mean - g_std) / SE
+        self.z_score_ = (w_mean - g_mean_) / self.SE_
 
         self.g_mean_ = g_mean_
         self.g_std_ = g_std
         self.s_ = s
         self.k += 1
 
-    def reset(self, x):
+    def reset(self):
         self.k = 0
         self.g_mean_ = 0
         self.s_ = 0
@@ -61,11 +57,8 @@ class ZScoreDetector(ChangeDetector):
         self.g_std_ = 0
         self.window.clear()
 
-    def check_stopping_rules(self, new_signal_value):
+    def check_stopping_rules(self, new_value):
         self.rules_triggered = False
         if np.absolute(self.z_score_) > self.threshold:
             self.rules_triggered = True
-            self.reset(new_signal_value)
-
-        if self.signal_size % 1000 == 0:
-            print(self.signal_size)
+            self.reset()

detectors/zscore_rules.py

@@ -70,7 +70,7 @@ class ZScoreDetectorRules(ChangeDetector):
             self.rules_triggered = True
 
             prev = 0 if len(self.stops) == 0 else self.stops[-1][1]
-            self.stops.append((prev, self.signal_size, int(round(self.g_mean_))))
+            self.stops.append((prev, self.k, int(round(self.g_mean_))))
 
             #print(self.stops)
 

plot_sequence.py → generators/plot_sequence.py

@@ -81,5 +81,5 @@ else:
         j += 1
     #plt.plot(signal)
     axarr[0].legend((pLegend), list(colors.keys()), bbox_to_anchor=(1, 1.0))
-    plt.savefig("plots/plot_" + seqName.rsplit('/')[1][:-4] + '.png')
+    plt.savefig("../plots/plot_" + seqName.rsplit('/')[1][:-4] + '.png')
     plt.show()

seqgen.py → generators/seqgen.py

@@ -2,7 +2,6 @@ import argparse
 import sequence_generator as sg
 import matplotlib.pyplot as plt
 import numpy
-import math
 import ast
 import time
 

sequence_generator.py → generators/sequence_generator.py

@@ -43,7 +43,7 @@ def plotSequence(axarr, seq, domain, attrName, curr_state):
 
 def saveToCsv(listOfConfigs, sequences):
     timestr = time.strftime("%Y_%m_%d-%H.%M.%S")
-    with open('sequences/sequence_'+timestr+'.csv','w') as csv_file:
+    with open('../sequences/sequence_'+timestr+'.csv','w') as csv_file:
         writer = csv.writer(csv_file, delimiter=',')
         #print(listOfConfigs)
         writer.writerow([config['attr_name'] for config in listOfConfigs])

test_generator.py → generators/test_generator.py

-import sequence_generator as sg
+from generators import sequence_generator as sg
 import matplotlib.pyplot as plt
-import numpy
-import math
 
 domain1 = ['a', 'b','z']
 domain2 = ['c', 'd']