diff --git a/github_updater.py b/github_updater.py index 525dd44..dd9fc6e 100644 --- a/github_updater.py +++ b/github_updater.py @@ -1,17 +1,16 @@ import os +import requests import shutil import tempfile import zipfile -import requests - class GithubUpdater: """ Interface for obtaining latest source codes from GitHub repository """ - def __init__(self, repo_owner: str, repo_name: str): + def __init__(self, repo_owner, repo_name): # type: (str, str) -> None """ Initializes the GitHub updater :param repo_owner: owner of the GitHub repository to update from @@ -20,21 +19,21 @@ def __init__(self, repo_owner: str, repo_name: str): self.url_release = "https://api.github.com/repos/%s/%s/releases/latest" % (repo_owner, repo_name) self.url_master_zip = "https://github.com/%s/%s/archive/master.zip" % (repo_owner, repo_name) - def get_latest_release_data(self) -> dict: + def get_latest_release_data(self): # type: () -> dict """ Uses GitHub API to obtain all data about latest release and return them as a dict :return: all data about latest release and as a dict """ return requests.get(self.url_release).json() - def get_latest_release_tag(self) -> str: + def get_latest_release_tag(self): # type: () -> str """ Fetches the tag of the latest release :return: the tag of the latest release from GitHub repository """ return self.get_latest_release_data()['tag_name'] - def get_latest_release_zip(self, target_file: str) -> None: + def get_latest_release_zip(self, target_file): # type: (str) -> None """ Downloads the zip with a source code of the latest release from GitHub :param target_file: where to download the zip file to @@ -46,7 +45,7 @@ def get_latest_release_zip(self, target_file: str) -> None: if chunk: f.write(chunk) - def get_master(self, target_file: str) -> None: + def get_master(self, target_file): # type: (str) -> None """ Downloads the current master branch as a zip file :param target_file: where to download the master branch zip file to @@ -58,7 +57,7 @@ def get_master(self, target_file: str) -> None: if chunk: f.write(chunk) - def get_and_extract_newest_release_to_directory(self, target_directory: str) -> None: + def get_and_extract_newest_release_to_directory(self, target_directory): # type: (str) -> None """ Downloads and extracts newest release to specified directory :param target_directory: here will be the code of the newest release placed @@ -68,7 +67,7 @@ def get_and_extract_newest_release_to_directory(self, target_directory: str) -> self.get_latest_release_zip(zip_file) self._extract_zip(zip_file, target_directory) - def extract_master(self, target_directory: str) -> None: + def extract_master(self, target_directory): # type: (str) -> None """ Downloads and extracts code from master branch to specified directory :param target_directory: here will be the code from master branch placed @@ -79,7 +78,7 @@ def extract_master(self, target_directory: str) -> None: self._extract_zip(zip_file, target_directory) @staticmethod - def _extract_zip(zip_path: str, target_directory: str) -> None: + def _extract_zip(zip_path, target_directory): # type: (str, str) -> None """ Extract zip with source code to target directory and deletes the zip file :param zip_path: zip file to be extracted diff --git a/http_socket_client.py b/http_socket_client.py index f52a9c6..5843654 100644 --- a/http_socket_client.py +++ b/http_socket_client.py @@ -1,235 +1,234 @@ -import json -import time -from threading import Thread - -import requests - - -class HSocket: - """ - Client HSocket that communicates with the remote server HSocket - """ - - def __init__(self, host, auto_connect=True): # type: (str, bool) -> None - """ - Initializes the HSocket - :param host: the host server URL (eg. if HSocket server is running on https://example.com/hsocket/ - then pass as host just "https://example.com") - :param auto_connect: if set to True, immediately performs a connection to the host - """ - self.host = host - self._listeners = {} # event name: function to call upon firing of the event - - self.__thread = None # background thread for communicating with server - self.connected = False # indicated if the connection with the server is stable - self.__connectedFired = False # when listener for connected is not defined yet, it will be fired right after - # definition if this is still False - self._connecting = False # indicates if we are at least connecting to the server - self.sid = None # local socket's id from server - - self._last_message_time = time.time() # time of last message we got from the server - self._fetch_msg_max_time = 10.0 # maximum time between fetching new messages from the server - - if auto_connect: - self.connect() - - def connect(self): # type: () -> None - """ - Performs a connection to the server - Fires 'connect' event upon successful connection - :return: None - """ - if self._connecting: - return - - self._connecting = True - - class HSocketRecevierThread(Thread): - """ - Thread that handles messages from the server - """ - # noinspection PyMethodParameters - def run(__): - while self._connecting: # as long as we are at least connecting to the server, fetch new messages - msg = self._get_message() - if (msg is None or msg.get('action', '') == 'disconnect') and self.connected: - # there was an error in communication or we are ordered to disconnect for now - self.disconnect(reconnect=True) # disconnect for now, but retry later - elif msg is None: # invalid message. Skip. - continue - elif msg.get('action', '') == 'connect': # server processed our request and had decided to connect - # us. Accept a new socket ID from the server and run "connect" event - self.sid = msg['sid'] - self.connected = True - self._run_listener('connect') - elif msg.get('action', '') == 'event': # server is firing an event on us - # run the appropriate listener - self._run_listener(msg['name'], msg['data']) - elif msg.get('action', '') == 'set_max_msg_interval': # server orders us to set a new maximum time - # between asking for new messages - self.set_retry_interval(float(msg['data'])) - - # start the background communication thread - self.__thread = HSocketRecevierThread() - self.__thread.start() - - def disconnect(self, reconnect=False): # type: (bool) -> None - """ - Disconnect from the server - :param reconnect: if set to True then after 30 seconds we will try to reconnect to the server - :return: None - """ - if not self._connecting: - return - - # reset everything - self.__thread = None - self._connecting = False - self.__connectedFired = False - self.sid = None - - if self.connected: - # if we are connected, inform the server about our disconnection - try: - requests.post(self.host + '/hsocket/', params={'sid': self.sid}, data={'action': 'disconnect'}, - timeout=5) - except requests.exceptions.ConnectionError or requests.exceptions.ConnectTimeout: - pass - except requests.exceptions.ReadTimeout: - pass - self.connected = False - self._run_listener('disconnect') - - if reconnect: - # if enabled, run the reconnection countdown in background - def f_reconnect(): - time.sleep(30) - self.connect() - - AsyncExecuter(f_reconnect).start() - - def on(self, event_name, func): # type: (str, "function") -> None - """ - Sets a new listener for an event - :param event_name: name of the event that the listener shall listen for - :param func: function fired upon calling of this event. Calls are performed like func(event_data) - :return: None - """ - item = self._listeners.get(event_name, []) - item.append(func) - self._listeners[event_name] = item - - if event_name == 'connect' and self.connected and not self.__connectedFired: - self._run_listener(event_name) - - def emit(self, event_name, data=None): # type: (str, any) -> None - """ - Fire an event with specified data - :param event_name: Name of the event to fire on the server - :param data: data passed to the fired function - :return: None - """ - if not self.connected: - return - try: - requests.post(self.host + '/hsocket/', params={'sid': self.sid}, data={'action': 'event', - 'name': event_name, - 'data': data}) - except requests.exceptions.ConnectionError: - self.disconnect(reconnect=True) - - def set_retry_interval(self, interval: float) -> None: - """ - Sets the maximum time in seconds before asking the server for new messages - :param interval: maximum time in seconds before asking the server for new messages - :return: None - """ - self._fetch_msg_max_time = interval - - def _get_message(self) -> dict or None: - """ - Waits until the message from server for this client is available or some error occurs and then returns - the fetched message or None on fail - :return: fetched message from the server or None on connection fail - """ - try: - while True: - request = requests.get(self.host + '/hsocket/', params=None if self.sid is None else {'sid': self.sid}, - timeout=10) - if request.status_code not in [200, 404]: - self.disconnect(reconnect=True) - return - data = request.json() - - if data.get('action', '') != 'retry': # if the message was a real message, save the time - # we have gathered it - if data.get('action', '') != 'set_max_msg_interval': - self._last_message_time = time.time() - break - time.sleep(min(self._fetch_msg_max_time, max(1.0, time.time() - self._last_message_time))) - return data - except requests.exceptions.ConnectionError: - self.disconnect(reconnect=True) - except json.decoder.JSONDecodeError: - raise HSocketException("This is not a http-socket server") - except requests.exceptions.Timeout: - pass - - def _run_listener(self, event_name, data=None): # type: (str, any) -> None - """ - Runs asynchronously all listeners for specified event - :param event_name: name of the event listeners to run - :param data: data to pass to the listening functions - :return: None - """ - if event_name == 'connect': - self.__connectedFired = True - for listener in self._listeners.get(event_name, []): - AsyncExecuter(listener, data).start() - - -class AsyncExecuter(Thread): - """ - Executes a function asynchronously - """ - - def __init__(self, func, data=None): # type: ("function", any) -> None - """ - Initializes the data for asynchronous execution. - The execution itself must be then started by using .start() - :param func: function to execute - :param data: data passed to the executed function - """ - Thread.__init__(self) - self.func = func - self.data = data - - def run(self): - self.func() if self.data is None else self.func(self.data) - - -class HSocketException(Exception): - pass - - -# If run directly, perform a quick test -if __name__ == '__main__': - sock = HSocket('http://127.0.0.1:5000') - - - def connect(): - print('Connected') - - - def disconnect(): - print('Disconnected') - - - def hello(msg): - print('Got:', msg) - sock.emit('helloBack', 'You too, sir') - - - sock.on('hello', hello) - sock.on('connect', connect) - sock.on('disconnect', disconnect) +import json +import requests +import time +from threading import Thread + + +class HSocket: + """ + Client HSocket that communicates with the remote server HSocket + """ + + def __init__(self, host, auto_connect=True): # type: (str, bool) -> None + """ + Initializes the HSocket + :param host: the host server URL (eg. if HSocket server is running on https://example.com/hsocket/ + then pass as host just "https://example.com") + :param auto_connect: if set to True, immediately performs a connection to the host + """ + self.host = host + self._listeners = {} # event name: function to call upon firing of the event + + self.__thread = None # background thread for communicating with server + self.connected = False # indicated if the connection with the server is stable + self.__connectedFired = False # when listener for connected is not defined yet, it will be fired right after + # definition if this is still False + self._connecting = False # indicates if we are at least connecting to the server + self.sid = None # local socket's id from server + + self._last_message_time = time.time() # time of last message we got from the server + self._fetch_msg_max_time = 10.0 # maximum time between fetching new messages from the server + + if auto_connect: + self.connect() + + def connect(self): # type: () -> None + """ + Performs a connection to the server + Fires 'connect' event upon successful connection + :return: None + """ + if self._connecting: + return + + self._connecting = True + + class HSocketRecevierThread(Thread): + """ + Thread that handles messages from the server + """ + # noinspection PyMethodParameters + def run(__): + while self._connecting: # as long as we are at least connecting to the server, fetch new messages + msg = self._get_message() + if (msg is None or msg.get('action', '') == 'disconnect') and self.connected: + # there was an error in communication or we are ordered to disconnect for now + self.disconnect(reconnect=True) # disconnect for now, but retry later + elif msg is None: # invalid message. Skip. + continue + elif msg.get('action', '') == 'connect': # server processed our request and had decided to connect + # us. Accept a new socket ID from the server and run "connect" event + self.sid = msg['sid'] + self.connected = True + self._run_listener('connect') + elif msg.get('action', '') == 'event': # server is firing an event on us + # run the appropriate listener + self._run_listener(msg['name'], msg['data']) + elif msg.get('action', '') == 'set_max_msg_interval': # server orders us to set a new maximum time + # between asking for new messages + self.set_retry_interval(float(msg['data'])) + + # start the background communication thread + self.__thread = HSocketRecevierThread() + self.__thread.start() + + def disconnect(self, reconnect=False): # type: (bool) -> None + """ + Disconnect from the server + :param reconnect: if set to True then after 30 seconds we will try to reconnect to the server + :return: None + """ + if not self._connecting: + return + + # reset everything + self.__thread = None + self._connecting = False + self.__connectedFired = False + self.sid = None + + if self.connected: + # if we are connected, inform the server about our disconnection + try: + requests.post(self.host + '/hsocket/', params={'sid': self.sid}, data={'action': 'disconnect'}, + timeout=5) + except requests.exceptions.ConnectionError or requests.exceptions.ConnectTimeout: + pass + except requests.exceptions.ReadTimeout: + pass + self.connected = False + self._run_listener('disconnect') + + if reconnect: + # if enabled, run the reconnection countdown in background + def f_reconnect(): + time.sleep(30) + self.connect() + + AsyncExecuter(f_reconnect).start() + + def on(self, event_name, func): # type: (str, "function") -> None + """ + Sets a new listener for an event + :param event_name: name of the event that the listener shall listen for + :param func: function fired upon calling of this event. Calls are performed like func(event_data) + :return: None + """ + item = self._listeners.get(event_name, []) + item.append(func) + self._listeners[event_name] = item + + if event_name == 'connect' and self.connected and not self.__connectedFired: + self._run_listener(event_name) + + def emit(self, event_name, data=None): # type: (str, any) -> None + """ + Fire an event with specified data + :param event_name: Name of the event to fire on the server + :param data: data passed to the fired function + :return: None + """ + if not self.connected: + return + try: + requests.post(self.host + '/hsocket/', params={'sid': self.sid}, data={'action': 'event', + 'name': event_name, + 'data': data}) + except requests.exceptions.ConnectionError: + self.disconnect(reconnect=True) + + def set_retry_interval(self, interval): # type: (float) -> None + """ + Sets the maximum time in seconds before asking the server for new messages + :param interval: maximum time in seconds before asking the server for new messages + :return: None + """ + self._fetch_msg_max_time = interval + + def _get_message(self): # type: () -> dict or None + """ + Waits until the message from server for this client is available or some error occurs and then returns + the fetched message or None on fail + :return: fetched message from the server or None on connection fail + """ + try: + while True: + request = requests.get(self.host + '/hsocket/', params=None if self.sid is None else {'sid': self.sid}, + timeout=10) + if request.status_code not in [200, 404]: + self.disconnect(reconnect=True) + return + data = request.json() + + if data.get('action', '') != 'retry': # if the message was a real message, save the time + # we have gathered it + if data.get('action', '') != 'set_max_msg_interval': + self._last_message_time = time.time() + break + time.sleep(min(self._fetch_msg_max_time, max(1.0, time.time() - self._last_message_time))) + return data + except requests.exceptions.ConnectionError: + self.disconnect(reconnect=True) + except json.decoder.JSONDecodeError: + raise HSocketException("This is not a http-socket server") + except requests.exceptions.Timeout: + pass + + def _run_listener(self, event_name, data=None): # type: (str, any) -> None + """ + Runs asynchronously all listeners for specified event + :param event_name: name of the event listeners to run + :param data: data to pass to the listening functions + :return: None + """ + if event_name == 'connect': + self.__connectedFired = True + for listener in self._listeners.get(event_name, []): + AsyncExecuter(listener, data).start() + + +class AsyncExecuter(Thread): + """ + Executes a function asynchronously + """ + + def __init__(self, func, data=None): # type: ("function", any) -> None + """ + Initializes the data for asynchronous execution. + The execution itself must be then started by using .start() + :param func: function to execute + :param data: data passed to the executed function + """ + Thread.__init__(self) + self.func = func + self.data = data + + def run(self): + self.func() if self.data is None else self.func(self.data) + + +class HSocketException(Exception): + pass + + +# If run directly, perform a quick test +if __name__ == '__main__': + sock = HSocket('http://127.0.0.1:5000') + + + def connect(): + print('Connected') + + + def disconnect(): + print('Disconnected') + + + def hello(msg): + print('Got:', msg) + sock.emit('helloBack', 'You too, sir') + + + sock.on('hello', hello) + sock.on('connect', connect) + sock.on('disconnect', disconnect) diff --git a/log_manipulator.py b/log_manipulator.py index bf1e746..2a562cf 100644 --- a/log_manipulator.py +++ b/log_manipulator.py @@ -1,156 +1,156 @@ -import re -import time -from datetime import datetime -from typing import List - - -class LogParser: - """ - Class for parsing information about attacks from log files - """ - - def __init__(self, file_log, rules, service_name=None): # type: (str, List[str], str) -> None - """ - Initialize the log parser - :param file_log: path to the file with logs - :param rules: list of string filters/rules - :param service_name: optional name of the service. If not specified then found attacks are not assigned to any - service - """ - self.file_log = file_log - self.rules = [rule if type(rule) == Rule else Rule(rule, service_name) for rule in rules] - - def parse_attacks(self, max_age=None): # type: (float) -> dict - """ - Parses the attacks from log file and returns them - :param max_age: optional, in seconds. If attack is older as this then it is ignored - :return: dictionary. Key is the IP that attacked and value is list of dictionaries with data about every attack - """ - attacks = {} - with open(self.file_log, 'r') as f: - log_lines = f.read().splitlines() - for log_line in log_lines: - for rule in self.rules: - variables = rule.get_variables(log_line) - if variables is not None: - if max_age is not None and time.time() - max_age > variables['TIMESTAMP']: - break - attacker_ip = variables['IP'] - del variables['IP'] - item = attacks.get(attacker_ip, []) - item.append(variables) - attacks[attacker_ip] = item - break - - return attacks - - def get_habitual_offenders(self, min_attack_attempts, attack_attempts_time, max_age=None, attacks=None): - # type: (int, int, int, dict) -> dict - """ - Finds IPs that had performed more than allowed number of attacks in specified time range - :param min_attack_attempts: minimum allowed number of attacks in time range to be included - :param attack_attempts_time: the time range in which all of the attacks must have occurred in seconds - :param max_age: optional, in seconds. If attack is older as this then it is ignored - :param attacks: optional. If None, then the value of self.parse_attacks(max_age) is used - :return: dictionary. Key is the IP that attacked more or equal than min_attack_attempts times and - value is list of dictionaries with data about every attack in specified time range - """ - attacks = self.parse_attacks(max_age) if attacks is None else attacks - habitual_offenders = {} - - for ip, attack_list in attacks.items(): - for attack in attack_list: - attacks_in_time_range = [] - for attack2 in attack_list: - attack_time_delta = attack2['TIMESTAMP'] - attack['TIMESTAMP'] - if 0 <= attack_time_delta <= attack_attempts_time: - attacks_in_time_range.append(attack2) - if len(attacks_in_time_range) > min_attack_attempts: - break - if len(attacks_in_time_range) >= min_attack_attempts: - habitual_offenders[ip] = attack_list - - return habitual_offenders - - -class Rule: - """ - Rule or filter that can be tested on a line from config line. If this rule/filter fits, than it can parse - variables from that line - """ - - def __init__(self, filter_string: str, service_name=None): - """ - Initializes this rule/filter - :param filter_string: string representation of this rule/filter with all variables stated as %VAR_NAME% - :param service_name: optional name of the service. If not specified then found attacks are not assigned to any - service - """ - self.__service_name = service_name - self.__rule_variables = re.findall("%.*?%", filter_string) - - # Generate regex for rule detection - self.__rule_regex = filter_string - for reserved_char in list("\\+*?^$.[]{}()|/"): # escape reserved regex characters - self.__rule_regex = self.__rule_regex.replace(reserved_char, '\\' + reserved_char) - for variable in self.__rule_variables: # replace all variables with any regex characters - self.__rule_regex = self.__rule_regex.replace(variable, '(.+?)') - if self.__rule_regex.endswith('?)'): # disable lazy search for last variables so they are found whole - self.__rule_regex = self.__rule_regex[:-2] + ')' - - # Remove %'s from variable names - self.__rule_variables = [var[1:-1] for var in self.__rule_variables] - - def test(self, log_line: str) -> bool: - """ - Test this Rule against a line from log file if it fits - :param log_line: line from a log file - :return: True if it fits, False if this rule cannot be applied to this line - """ - return True if re.match(self.__rule_regex, log_line) else False - - def get_variables(self, log_line): # type: (str) -> dict or None - """ - Parses variables from log line that fits this rule - :param log_line: line from a log file - :return: None if this rule cannot be applied to this line, otherwise returns a dictionary with parsed variables - from this line - """ - data = {} - - # Parse all variables from log line - variable_search = re.match(self.__rule_regex, log_line) - if not variable_search: # this rule is not for this line - return None - # noinspection PyTypeChecker - for i, variable in enumerate(self.__rule_variables): - data[variable] = variable_search.group(i + 1).strip() - - if self.__service_name is not None: - data['SERVICE'] = self.__service_name - - date_format = '%Y %b %d %H:%M:%S' - date_string = None - if 'D:M' in data and 'D:D' in data and 'TIME' in data: - date_string = '%s %s %s %s' % (datetime.now().strftime('%Y'), data['D:M'], - data['D:D'], data['TIME']) - elif 'D:M' in data and 'D:D' in data: - date_string = '%s %s %s 00:00:00' % (datetime.now().strftime('%Y'), data['D:M'], data['D:D']) - elif 'TIME' in data: - # noinspection PyTypeChecker - date_string = datetime.now().strftime('%Y %b %d') + ' ' + data['TIME'] - - data['TIMESTAMP'] = time.time() if date_string is None else \ - time.mktime(datetime.strptime(date_string, date_format).timetuple()) - return data - - -# If launched directly, perform a quick proof of work in file debug.log -if __name__ == '__main__': - all_rules = ["%D:M% %D:D% %TIME% %IP% attacked on user %USER%"] - file = 'debug.log' - - parser = LogParser(file, all_rules) - offenders = parser.get_habitual_offenders(3, 100000) - for off_ip, off_attacks in offenders.items(): - print(off_ip + ':', off_attacks) +import re +import time +from datetime import datetime +from typing import List + + +class LogParser: + """ + Class for parsing information about attacks from log files + """ + + def __init__(self, file_log, rules, service_name=None): # type: (str, List[str], str) -> None + """ + Initialize the log parser + :param file_log: path to the file with logs + :param rules: list of string filters/rules + :param service_name: optional name of the service. If not specified then found attacks are not assigned to any + service + """ + self.file_log = file_log + self.rules = [rule if type(rule) == Rule else Rule(rule, service_name) for rule in rules] + + def parse_attacks(self, max_age=None): # type: (float) -> dict + """ + Parses the attacks from log file and returns them + :param max_age: optional, in seconds. If attack is older as this then it is ignored + :return: dictionary. Key is the IP that attacked and value is list of dictionaries with data about every attack + """ + attacks = {} + with open(self.file_log, 'r') as f: + log_lines = f.read().splitlines() + for log_line in log_lines: + for rule in self.rules: + variables = rule.get_variables(log_line) + if variables is not None: + if max_age is not None and time.time() - max_age > variables['TIMESTAMP']: + break + attacker_ip = variables['IP'] + del variables['IP'] + item = attacks.get(attacker_ip, []) + item.append(variables) + attacks[attacker_ip] = item + break + + return attacks + + def get_habitual_offenders(self, min_attack_attempts, attack_attempts_time, max_age=None, attacks=None): + # type: (int, int, int, dict) -> dict + """ + Finds IPs that had performed more than allowed number of attacks in specified time range + :param min_attack_attempts: minimum allowed number of attacks in time range to be included + :param attack_attempts_time: the time range in which all of the attacks must have occurred in seconds + :param max_age: optional, in seconds. If attack is older as this then it is ignored + :param attacks: optional. If None, then the value of self.parse_attacks(max_age) is used + :return: dictionary. Key is the IP that attacked more or equal than min_attack_attempts times and + value is list of dictionaries with data about every attack in specified time range + """ + attacks = self.parse_attacks(max_age) if attacks is None else attacks + habitual_offenders = {} + + for ip, attack_list in attacks.items(): + for attack in attack_list: + attacks_in_time_range = [] + for attack2 in attack_list: + attack_time_delta = attack2['TIMESTAMP'] - attack['TIMESTAMP'] + if 0 <= attack_time_delta <= attack_attempts_time: + attacks_in_time_range.append(attack2) + if len(attacks_in_time_range) > min_attack_attempts: + break + if len(attacks_in_time_range) >= min_attack_attempts: + habitual_offenders[ip] = attack_list + + return habitual_offenders + + +class Rule: + """ + Rule or filter that can be tested on a line from config line. If this rule/filter fits, than it can parse + variables from that line + """ + + def __init__(self, filter_string, service_name=None): # type: (str, str or None) -> None + """ + Initializes this rule/filter + :param filter_string: string representation of this rule/filter with all variables stated as %VAR_NAME% + :param service_name: optional name of the service. If not specified then found attacks are not assigned to any + service + """ + self.__service_name = service_name + self.__rule_variables = re.findall("%.*?%", filter_string) + + # Generate regex for rule detection + self.__rule_regex = filter_string + for reserved_char in list("\\+*?^$.[]{}()|/"): # escape reserved regex characters + self.__rule_regex = self.__rule_regex.replace(reserved_char, '\\' + reserved_char) + for variable in self.__rule_variables: # replace all variables with any regex characters + self.__rule_regex = self.__rule_regex.replace(variable, '(.+?)') + if self.__rule_regex.endswith('?)'): # disable lazy search for last variables so they are found whole + self.__rule_regex = self.__rule_regex[:-2] + ')' + + # Remove %'s from variable names + self.__rule_variables = [var[1:-1] for var in self.__rule_variables] + + def test(self, log_line): # type: (str) -> bool + """ + Test this Rule against a line from log file if it fits + :param log_line: line from a log file + :return: True if it fits, False if this rule cannot be applied to this line + """ + return True if re.match(self.__rule_regex, log_line) else False + + def get_variables(self, log_line): # type: (str) -> dict or None + """ + Parses variables from log line that fits this rule + :param log_line: line from a log file + :return: None if this rule cannot be applied to this line, otherwise returns a dictionary with parsed variables + from this line + """ + data = {} + + # Parse all variables from log line + variable_search = re.match(self.__rule_regex, log_line) + if not variable_search: # this rule is not for this line + return None + # noinspection PyTypeChecker + for i, variable in enumerate(self.__rule_variables): + data[variable] = variable_search.group(i + 1).strip() + + if self.__service_name is not None: + data['SERVICE'] = self.__service_name + + date_format = '%Y %b %d %H:%M:%S' + date_string = None + if 'D:M' in data and 'D:D' in data and 'TIME' in data: + date_string = '%s %s %s %s' % (datetime.now().strftime('%Y'), data['D:M'], + data['D:D'], data['TIME']) + elif 'D:M' in data and 'D:D' in data: + date_string = '%s %s %s 00:00:00' % (datetime.now().strftime('%Y'), data['D:M'], data['D:D']) + elif 'TIME' in data: + # noinspection PyTypeChecker + date_string = datetime.now().strftime('%Y %b %d') + ' ' + data['TIME'] + + data['TIMESTAMP'] = time.time() if date_string is None else \ + time.mktime(datetime.strptime(date_string, date_format).timetuple()) + return data + + +# If launched directly, perform a quick proof of work in file debug.log +if __name__ == '__main__': + all_rules = ["%D:M% %D:D% %TIME% %IP% attacked on user %USER%"] + file = 'debug.log' + + parser = LogParser(file, all_rules) + offenders = parser.get_habitual_offenders(3, 100000) + for off_ip, off_attacks in offenders.items(): + print(off_ip + ':', off_attacks) diff --git a/simple-guardian.py b/simple-guardian.py index 3ca410d..a84d045 100644 --- a/simple-guardian.py +++ b/simple-guardian.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import json import os +import requests import sqlite3 import subprocess import sys @@ -9,9 +10,7 @@ from queue import Queue from subprocess import Popen from threading import Thread, Lock -from typing import List, Dict - -import requests +from typing import List, Dict, Set import github_updater import log_manipulator @@ -52,10 +51,10 @@ PROFILES_DIR = os.path.join(CONFIG_DIR, 'profiles') # directory with profiles CONFIG = {} # dictionary with loaded config in main() -ONLINE_DATA: Dict[str, any] = {'loggedIn': False} # data about the online server, +ONLINE_DATA = {'loggedIn': False} # type: Dict[str, any] # data about the online server, PROFILES = {} # type: {str: dict} PROFILES_LOCK = Lock() # lock used when manipulating with profiles in async -VERSION_TAG = "1.01" # tag of current version +VERSION_TAG = "1.02" # tag of current version class Database: @@ -159,7 +158,7 @@ class AppRunning: app_running = [True] @staticmethod - def is_running() -> bool: + def is_running(): # type: () -> bool """ Tests if the program should be running :return: True if the program should be running, False if it should terminate itself @@ -167,7 +166,7 @@ def is_running() -> bool: return len(AppRunning.app_running) > 0 @staticmethod - def set_running(val: bool): + def set_running(val): # type: (bool) -> None """ Sets if the program should be running :param val: True if the program should be running, False if it should terminate itself @@ -208,7 +207,7 @@ class IPBlocker: block_command_path = './blocker' # path to the executable that blocks the IPs @staticmethod - def list_blocked_ips() -> set: + def list_blocked_ips(): # type: () -> Set[str] """ Lists the blocked IPs from database :return: set of the blocked IPs @@ -340,7 +339,7 @@ def init(): CONFIG['updater']['githubRepo']) @staticmethod - def update_available() -> bool: + def update_available(): # type: () -> bool """ Check the most recent version tag from the server and compare it to the VERSION_TAG variable :return: True if local and remote version tags differ, False if they are the same @@ -348,7 +347,7 @@ def update_available() -> bool: return VERSION_TAG != Updater._updater.get_latest_release_tag() @staticmethod - def get_latest_name() -> str: + def get_latest_name(): # type: () -> str """ Gets the name of the latest release tag :return: the name of the latest release tag @@ -450,7 +449,7 @@ def load_profiles(): # type: () -> None PROFILES_LOCK.release() -def pair_with_server(url: str) -> (bool, str): +def pair_with_server(url): # type: (str) -> (bool, str) """ Pairs this device with an account on the Simple Guardian server :param url: URL generated by creating a new device on Simple Guardian Server web