Merge branch 'dev'

Author: Nicolas@Thor

.gitignore

@@ -7,3 +7,5 @@ __pycache__
 *.csv
 # Plot files
 *.png
+# Log files
+*.log

config.yaml

@@ -18,4 +18,48 @@ options:
 windows: [30, 60, 120, 150, 180, 300, 360] # number of datapoints of the non-overlapping windows
 infected: [0, 1, 5, 6, 7, 8, 9] # positions of the infected datasets
 healthy: [2] # position of the healthy dataset
-functions: ['mean'] # aggragation functions
+functions: ['mean'] # aggragation functions
+
+
+# policy creation
+## general
+windowSize: 360 # define the window size for which the policy should be generated
+aggregateFunctions: ['mean', 'min', 'max']
+
+seed: 10 # random seed to obtain the same policies in the random approach
+randomPolicyCreation: False # enable random policy creation
+randomNumberOfPolicyRules: False 
+maxNumberOfPolicyRules: 6
+minNumberOfPolicyRules: 3
+exactNumberOfPolicyRules: 4
+
+completePolicyCreation: True # enable complete policy creation
+
+expertPolicyCreation: False
+
+#
+threshholds: [1, 3, 5]
+
+# policy selection
+interval: 60
+countMalwareIndicators: False
+countMalwareTypeIndicators: False
+countMalwareIndicatorsRelatively: True
+modes:
+- name: sensitive
+  letter: S
+  detectionTreshold: 0.05
+- name: average
+  letter: A
+  detectionTreshold: 0.05
+- name: robust
+  letter: R
+  detectionTreshold: 0.80
+mode:
+- name: robust
+  letter: R
+  detectionTreshold: 0.80
+
+
+dstatCommand: ['dstat', '-t', '--cpu', '--mem', '-d', '--disk-tps', '-n', '--tcp', '-y', '-p', '-N', 'eth0', '1', '2']
+ipFinderCommand: ['hostname', '-I']

observer/observer-dstat.sh

@@ -19,9 +19,9 @@ sudo apt-get install dstat
 '
 cd /root/MTDPolicy/data/csv/
 
-iterations=60
+iterations=1
 delay=1
-observations=30
+observations=1800
 for ((i = 0 ; i < $iterations ; i++)); do
     now=`date +%F-%H-%M-%S`
     suffix="-log.csv"

policyCreator.py

@@ -0,0 +1,139 @@
+import glob
+import pandas as pd
+import numpy as np
+import yaml
+import random
+import utils
+
+# load config
+with open('config.yaml') as stream:
+    config = yaml.safe_load(stream)
+
+windowSize = config['windowSize']
+randomPolicyCreation = config['randomPolicyCreation']
+randomNumberOfPolicyRules = config['randomNumberOfPolicyRules']
+minNumberOfPolicyRules = config['minNumberOfPolicyRules']
+maxNumberOfPolicyRules = config['maxNumberOfPolicyRules']
+exactNumberOfPolicyRules = config['exactNumberOfPolicyRules']
+completePolicyCreation = config['completePolicyCreation']
+expertPolicyCreation = config['expertPolicyCreation']
+aggregateFunctions = config['aggregateFunctions']  # avg, min, max
+
+AGGREGATEFUNCTION = aggregateFunctions[0]
+SEED = config['seed']
+POLICYCOLUMNS = utils.POLICYCOLUMNS
+
+# set seed
+random.seed(SEED)
+print(random.random())
+
+
+def createPolicy():
+
+    # load the csv with windowSize
+    filenames = [file for file in glob.glob(
+        './*.csv') if 'policy({}).csv'.format(windowSize) in file]
+    csvPolicy = pd.read_csv(filenames[0], header=None)
+
+    # postprocess: set header and group by malwaretype
+    csvPolicy.columns = POLICYCOLUMNS
+    # print(malwareGroup.get_group('httpbackdoor')) # DEBUG
+    malwareGroup = csvPolicy.groupby(['malware'])
+
+    # policy creation
+    policy = pd.DataFrame()
+
+    # random policy creation
+    # iterate over all malware groups and add some (random or defined) rules (row) for each malware type
+    if randomPolicyCreation == True:
+        method = 'random'
+        if randomNumberOfPolicyRules == True:
+            method += '({}-{})'.format(minNumberOfPolicyRules,
+                                       maxNumberOfPolicyRules)
+        else:
+            method += '({})'.format(exactNumberOfPolicyRules)
+
+        for malware in malwareGroup:
+            # malware is a tuple: (name, df)
+            rows = malware[1].shape[0]  # number of rows for that malware type
+
+            # random number of rules
+            if randomNumberOfPolicyRules == True:
+
+                # define random number between min/max number of policy rules
+                random.seed(SEED)
+                nRules = random.choice(
+                    [minNumberOfPolicyRules, maxNumberOfPolicyRules])
+
+            # defined number of rules
+            else:
+                nRules = exactNumberOfPolicyRules
+
+            # make sure we don't have more rules than rows
+            while(rows < nRules):
+                nRules -= 1
+
+            # print(malware[1].sample(n = nRules)) # DEBGUG
+            # add defined rules to policy
+            # todo check what happens if nRules > n when set
+            policy = policy.append(
+                malware[1].sample(n=nRules, random_state=SEED))
+            policy = policy.drop_duplicates(subset=['metric'])
+
+    # complete policy creation
+    # iterate over all malware groups and all rules (row) for each malware type
+    elif completePolicyCreation == True:
+        method = 'complete'
+        policy = csvPolicy
+
+    # expert policy creation
+    elif expertPolicyCreation == True:
+        method = 'expert'
+        pass
+        # to be done
+
+    # postprocessing
+    # remove aggregate function string for all rows
+    policy['metric'] = policy['metric'].str.replace(
+        '-{}'.format(AGGREGATEFUNCTION), '')
+    policyName = 'policy({})-{}-{}'.format(windowSize,
+                                           AGGREGATEFUNCTION, method)
+    policy.to_csv('{}.csv'.format(policyName), index=False)
+
+    return policy
+
+
+def malwareDistribution(policy):
+    # init
+    CNCMALWARE = utils.CNC
+    RKMALWALRE = utils.ROOTKIT
+    RWMALWARE = utils.RANSOMWARE
+    MALWARECATEGORIES = utils.MALWARECATEGORIES
+
+    conditions = [
+        (policy['malware'].isin(CNCMALWARE)),
+        (policy['malware'].isin(RKMALWALRE)),
+        (policy['malware'].isin(RWMALWARE))
+    ]
+    # classify each malware by type and add type column
+    policy['malwaretype'] = np.select(conditions, MALWARECATEGORIES)
+
+    # count different malware types and create a dict
+    malwareTypes = policy['malwaretype'].value_counts().index.tolist()
+    malwareOccurrences = policy['malwaretype'].value_counts().values.tolist()
+    malwareTypeOcc = {malwareTypes[i]: malwareOccurrences[i]
+                      for i in range(len(malwareTypes))}
+
+    # count total occurences of all malware types
+    totalOccurences = sum(malwareOccurrences)
+    '''
+    malwareTypeOcc: {
+        'Rootkit': 3
+        'CnC': 7
+        'Ransomware: 3
+    }
+    totalOccurences: 13
+    array([0.53846154, 0.23076923, 0.23076923])
+    '''
+
+    return [malwareTypeOcc, totalOccurences, np.divide(malwareOccurrences, totalOccurences)]

prePolicy.py

@@ -0,0 +1,44 @@
+import glob
+import os
+from cv2 import detail_BestOf2NearestRangeMatcher
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import seaborn as sns
+import statsmodels.api as sm
+import shutil
+import csv
+import yaml
+import random
+
+COLS = ['malware', 'metric', 'sign', 'threshold']
+with open('config.yaml') as stream:
+    config = yaml.safe_load(stream)
+
+print(config['policy'])
+
+file = [i for i in glob.glob('./*.csv') if str(config['policy']) in i]
+
+thresholds = pd.read_csv(file[0], header = None)
+thresholds.columns = COLS
+th = thresholds.groupby(['malware'])
+
+#print(th.get_group('httpbackdoor'))
+
+policy = pd.DataFrame()
+for malware in th:
+    if config['random'] == True:
+        # malware is a tuple: (name, df)
+        rows = malware[1].shape[0]
+        nRules = random.choice([config['MIN_TH'], config['MAX_TH']])
+        while(rows < nRules):
+            nRules = random.choice([config['MIN_TH'], config['MAX_TH']])
+
+    else:
+        nRules = config['NUMBER_TH']
+        print('false')
+    print(malware[1].sample(n = nRules))
+    policy = policy.append(malware[1].sample(n = nRules))
+
+print(policy)
+policy.to_csv('policy.csv', index=False)