Code refactor

.gitignore

@@ -1,4 +1,6 @@
 .idea/
 venv/
 data/
-__pycache__/
+__pycache__/
+*.csv
+TESTS

Pipfile

@@ -15,7 +15,7 @@ falcon = "*"
 gunicorn = "*"
 scikit-learn = "*"
 pandas = "*"
-learn-utils = "*"
+matplotlib = "*"
 
 [requires]
 python_version = "3.6"

api/api.py

@@ -44,6 +44,3 @@ def server():
     api.add_route('/operators', OperatorRoute())
 
     StandaloneApplication(application, options).run()
-
-
-

api/app.py

@@ -21,5 +21,3 @@ def load_config(self):
 
     def load(self):
         return self.application
-
-

api/json.py

@@ -0,0 +1,20 @@
+import json
+
+from cassandra.cqlengine.query import ModelQuerySet
+
+
+class CassandraDataEncoder(json.JSONEncoder):
+    def default(self, cassandra_object: ModelQuerySet):
+        keys = []
+        for v in cassandra_object.model.__dict__['_defined_columns'].items():
+            keys.append(v[0])
+
+        result = []
+        for row in cassandra_object:
+            parsed_row = {}
+            for key in keys:
+                parsed_row[key] = row[key]
+
+            result.append(parsed_row)
+
+        return result

api/routes/dataset_key_route.py

@@ -1,6 +1,7 @@
 import json
 import falcon
 
+from api.json import CassandraDataEncoder
 from db.models.DataSetMap import DataSetMap
 
 
@@ -10,16 +11,8 @@ def on_get(req, resp):
         try:
             operator_id = req.params['operator_id']
             result = DataSetMap.objects.filter(operator_id=operator_id)
-            body = []
-            for data_set_map in result:
-                body.append({
-                    'operator_id': data_set_map.operator_id,
-                    'acronym': data_set_map.acronym,
-                    'kpi_name': data_set_map.kpi_name,
-                    'has_enough': data_set_map.has_enough
-                })
 
-            resp.body = json.dumps(body, ensure_ascii=False)
+            resp.body = json.dumps(result, ensure_ascii=False, cls=CassandraDataEncoder)
             resp.status = falcon.HTTP_200
         except KeyError as err:
             missing_field = str(err)

api/routes/dataset_route.py

@@ -2,6 +2,7 @@
 import falcon
 import time_uuid
 
+from api.json import CassandraDataEncoder
 from db.models.DataPoint import DataPoint
 
 

api/routes/kpi_definition_route.py

@@ -33,5 +33,3 @@ def on_get(req, resp):
             missing_field = str(err)
             resp.body = json.dumps({'error': "{} is required".format(missing_field)})
             resp.status = falcon.HTTP_400
-
-

api/routes/operator_route.py

@@ -1,21 +1,14 @@
 import json
 import falcon
 
+from api.json import CassandraDataEncoder
 from db.models.Operator import Operator
 
 
 class OperatorRoute(object):
     @staticmethod
-    def on_get(req, resp):
+    def on_get(_, resp):
         result = Operator.objects.all()
 
-        body = []
-        for operator_row in result:
-            body.append({
-                'operator_id': operator_row.operator_id,
-                'operator_name': "OPERATOR"
-            })
-
-        resp.body = json.dumps(body, ensure_ascii=False)
+        resp.body = json.dumps(result, ensure_ascii=False, cls=CassandraDataEncoder)
         resp.status = falcon.HTTP_200
-

api/routes/outlier_route.py

@@ -31,17 +31,34 @@ def on_get(req, resp):
 
             final_result = []
             if method == 'sim':
-                final_result = detector.knn_mean_sim(similarity=3)
+                final_result, _ = detector.knn_mean_sim(similarity=5)
 
-            if method == 'dist':
-                final_result = detector.knn_mean_dist()
+            if method == 'dist_f':
+                final_result, _ = detector.knn_mean_dist(variant='fixed')
+
+            if method == 'dist_c':
+                final_result, _ = detector.knn_mean_dist(variant='calculated', distance_ratio=4)
 
             if method == 'reg':
-                final_result = detector.regression_line()
+                final_result, _ = detector.regression_line(tolerance=0.5)
+
+            if method == 'isolation_forest':
+                final_result, _ = detector.isolation_forest()
+
+            if method == 'random_split':
+                final_result, _ = detector.rnd_split()
+
+            if method == 'fusion':
+                final_result = detector.fusion({
+                    'clusters': 30,
+                    'distance_ratio': 4,
+                    'similarity': 5,
+                    'precision': 0.5,
+                    'tolerance': 0.5
+                })['data']['fusion_result']
 
             resp.body = json.dumps(final_result, ensure_ascii=False)
             resp.status = falcon.HTTP_200
         except ValueError as err:
             resp.body = json.dumps({'error': "{}".format(str(err))})
             resp.status = falcon.HTTP_400
-

cli/cli.py

@@ -8,6 +8,7 @@
 from db.Database import Database
 from load_data.load_data import load_data
 from load_data.load_operators import load_operators
+from run_all.run_all import run_all
 
 
 @click.group()
@@ -41,9 +42,11 @@ def __count_dataset():
     count_dataset()
 
 
+@cli.command('run_all')
+def __run_all():
+    run_all()
+
+
 @cli.command('server')
 def __server():
     server()
-
-
-

count_dataset/count_dataset.py

@@ -1,15 +1,20 @@
 from db.models.DataPoint import DataPoint
 from db.models.DataSetMap import DataSetMap
+from db.models.Operator import Operator
 
 
-def count_dataset(limit=200):
+def count_dataset(limit=700):
     result = DataSetMap.objects.all()
 
     for data_set_map in result:
         operator_id = data_set_map.operator_id
         acronym = data_set_map.acronym
         kpi_name = data_set_map.kpi_name
         has_enough = len(DataPoint.objects.filter(operator_id=operator_id, acronym=acronym, kpi_name=kpi_name).limit(limit))
-        print(has_enough)
         DataSetMap.objects(operator_id=operator_id, acronym=acronym, kpi_name=kpi_name).update(has_enough=has_enough >= limit)
 
+        operator = Operator.objects(operator_id=operator_id).first()
+        if not operator.has_enough or operator.has_enough is None:
+            print(operator_id, has_enough >= limit)
+            Operator.objects(operator_id=operator_id).update(has_enough=has_enough >= limit)
+

db/Database.py

@@ -8,6 +8,7 @@
 from db.models.DataSetMap import DataSetMap
 from db.models.AcronymNameMap import AcronymNameMap
 from db.models.Operator import Operator
+from db.models.Report import Report
 
 
 class Database:
@@ -27,6 +28,7 @@ def sync():
         sync_table(DataSetMap)
         sync_table(AcronymNameMap)
         sync_table(Operator)
+        sync_table(Report)
 
     def query(self, query):
         session = self.cluster.connect()

db/models/Operator.py

@@ -4,6 +4,7 @@
 
 class Operator(Model):
     operator_id = Integer(primary_key=True)
+    has_enough = Boolean()
 
     __keyspace__ = "data_det"
     __table_name__ = "operators"

db/models/Report.py

@@ -0,0 +1,16 @@
+from cassandra.cqlengine.columns import *
+from cassandra.cqlengine.models import Model
+
+
+class Report(Model):
+    operator_id = BigInt(primary_key=True)
+    acronym = Text(primary_key=True)
+    kpi_name = Text(primary_key=True)
+    start_time = TimeUUID(primary_key=True)
+    method = Text()
+    params = Text()
+    count_outliers = BigInt()
+    end_time = TimeUUID()
+
+    __keyspace__ = "data_det"
+    __table_name__ = "reports"

load_data/load_data.py

@@ -26,14 +26,14 @@ def load_kpi_definitions(skip_if_exists=True):
     result = RawKpiDefinition.objects.all()
     filtered_result = filter_complex_kpis(result)
     for raw_kpi_definition in filtered_result:
-        d_id = raw_kpi_definition.id
+        definition_id = raw_kpi_definition.id
         description = raw_kpi_definition.text
         formula = raw_kpi_definition.formula
         tags = raw_kpi_definition.tags
         technology = raw_kpi_definition.technology
         unit = raw_kpi_definition.unit
 
-        KpiDefinition.create(id=d_id, description=description, formula=formula, tags=tags, technology=technology, unit=unit)
+        KpiDefinition.create(id=definition_id, description=description, formula=formula, tags=tags, technology=technology, unit=unit)
 
 
 def load_data_points(skip_if_exists=True):

load_data/load_operators.py

@@ -5,7 +5,7 @@
 def load_operators(skip_if_exists=True):
     print("Loading operators")
     if len(DataPoint.objects.limit(1)) is 0:
-        raise ResourceWarning("collection data_set_map is empty")
+        raise ResourceWarning("collection data_points is empty")
 
     if len(Operator.objects.limit(1)) is not 0 and skip_if_exists:
         print("Skipping due to skip_if_exists: True")