Merge pull request #455 from InQuest/merge-python38

Merge python38

.gitignore

@@ -4,7 +4,6 @@ parsetab.py
 .jshintrc
 flask/
 env/
-db_data/
 .DS_Store
 *.db
 *.pyc

Dockerfile

@@ -1,11 +1,9 @@
-FROM ubuntu:18.04
+FROM ubuntu:20.04
 
 # Update OS Packages, Install OS Dependencies (Do this in one line to ensure Update always happens)
 RUN apt-get update && \
-    apt-get install -y curl && \
-    curl -sL https://deb.nodesource.com/setup_10.x | bash - && \
-    apt-get install -y git libsqlite3-dev python2.7 python-pip nodejs libffi-dev libssl-dev mysql-client \
-    libmysqlclient-dev python2.7-dev libpython2.7-dev file yara apt-transport-https ca-certificates \
+    DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true apt-get install -y git libsqlite3-dev python3.8 python3-pip npm libffi-dev libssl-dev mysql-client \
+    libmysqlclient-dev python3-dev libpython3-dev git yara apt-transport-https ca-certificates curl \
     software-properties-common libpcre3 libpcre3-dev
 
 # Setup UWSGI Installation
@@ -17,7 +15,9 @@ WORKDIR /opt/threatkb
 COPY package.json .bowerrc bower.json Gruntfile.js requirements.txt ./
 
 # Install Python Dependencies
-RUN python -m pip install --upgrade pip && python -m pip install -r requirements.txt
+RUN /usr/bin/pip3 install --upgrade pip & /usr/bin/pip3 install virtualenv
+RUN /usr/local/bin/virtualenv -p /usr/bin/python3.8 env
+RUN env/bin/pip3 install -r requirements.txt
 
 # Install Node Dependencies
 RUN npm install -g bower && bower install --allow-root

app/__init__.py

@@ -13,27 +13,31 @@
 import datetime
 import logging
 import os
-from distutils import util
-from slack_helper import SlackHelper
+import distutils
 
 
 app = Flask(__name__, static_url_path='')
-cache = Cache(app, config={'CACHE_TYPE': 'simple'})
+app.config.from_object("config")
+
+if app.config.get('REDIS_CACHE_URL', None):
+    cache = Cache(app, config={'CACHE_TYPE': 'redis', 'CACHE_REDIS_URL': app.config.get('REDIS_CACHE_URL')})
+else:
+    cache = Cache(app, config={'CACHE_TYPE': 'simple'})
 cache.init_app(app)
 app.secret_key = "a" * 24  # os.urandom(24)
-app.config.from_object("config")
 
 auto = Autodoc(app)
 
-db = SQLAlchemy(app)
+#db = SQLAlchemy(app)
+db = SQLAlchemy(app, engine_options={"pool_recycle": 600})
 login_manager = LoginManager()
 login_manager.init_app(app)
 bcrypt = Bcrypt(app)
 celery = None
+migrate = Migrate(app, db)
 
 ENTITY_MAPPING = {"SIGNATURE": 1, "DNS": 2, "IP": 3, "TASK": 4, "RELEASE": 5}
 ENTITY_MAPPING_URI = {1: "yara_rules", 2: "c2dns", 3: "c2ips", 4: "tasks", 5: "releases"}
-
 ACTIVITY_TYPE = {"ARTIFACT_CREATED": "Artifact Created",
                  "ARTIFACT_MODIFIED": "Artifact Modified",
                  "COMMENTS": 'Comment',
@@ -88,14 +92,10 @@ def set_celery_stuff(flask_app):
     if flask_app.config["MAX_MILLIS_PER_FILE_THRESHOLD"]:
         flask_app.config["MAX_MILLIS_PER_FILE_THRESHOLD"] = float(flask_app.config["MAX_MILLIS_PER_FILE_THRESHOLD"])
 
-
-print("app config: %s" % (str(app.config)))
-print("print env")
-for key, val in os.environ.iteritems():
-    print "%s=%s" % (key, val)
-
 set_celery_stuff(app)
 
+print(("app config: %s" % (str(app.config))))
+
 from app.celeryapp import make_celery
 
 celery = make_celery(app)
@@ -129,6 +129,7 @@ def set_celery_stuff(flask_app):
 from app.routes.errors import *
 from app.routes.activity_log import *
 from app.routes.macros import *
+from app.routes.countries import *
 
 from app.models import users
 from app.models import c2ip
@@ -293,6 +294,7 @@ def generate_app():
     from app.routes import errors
     from app.routes import activity_log
     from app.routes import macros
+    from app.routes import countries
 
     app.config['PRESERVE_CONTEXT_ON_EXCEPTION'] = False
 

app/models/access_keys.py

@@ -16,7 +16,7 @@ def to_dict(self):
         return dict(
             id=self.id,
             user=self.user.to_dict(),
-            token=self.token.decode('ascii'),
+            token=self.token,
             created=self.created.isoformat(),
             deleted=None if not self.deleted else self.deleted.isoformat(),
             status=self.status

app/models/activity_log.py

@@ -3,7 +3,7 @@
 from app import db, ENTITY_MAPPING, ACTIVITY_TYPE, ENTITY_MAPPING_URI, app
 from sqlalchemy import bindparam, inspect
 
-from app.models import users, cfg_settings, users
+from app.models import users, cfg_settings
 from app.models.cfg_states import Cfg_states
 import re
 from flask import request
@@ -16,7 +16,6 @@ class ActivityLog(db.Model):
     id = db.Column(db.Integer, primary_key=True, autoincrement=True)
     activity_type = db.Column(db.String(256))
     activity_text = db.Column(db.TEXT())
-    activity_text_short = db.Column(db.String(1000))
     activity_date = db.Column(db.DateTime(timezone=True))
 
     entity_type = db.Column(db.Integer(), index=True, nullable=False)
@@ -46,7 +45,7 @@ def get_modified_changes(target):
     inspection = inspect(target)
     attrs = class_mapper(target.__class__).column_attrs
 
-    changes = {"short": [], "long": []}
+    changes = []
     for attr in attrs:
         attr_hist = getattr(inspection.attrs, attr.key).history
         if attr_hist.has_changes():
@@ -61,11 +60,9 @@ def get_modified_changes(target):
                 after = attr_hist.added[0]
 
             if before != after:
-                changes["long"].append("'%s' changed from '%s' to '%s'" % (attr.key, before, after))
-                changes["short"].append("'%s'" % (attr.key))
+                changes.append("'%s' changed from '%s' to '%s'" % (attr.key, before, after))
 
-    changes["long"] = [re.sub("[^\x00-\x7F]", "", change) for change in changes["long"]]
-    changes["short"] = [re.sub("[^\x00-\x7F]", "", change) for change in changes["short"]]
+    changes = [re.sub("[^\x00-\x7F]", "", change) for change in changes]
     return changes
 
 

app/models/c2dns.py

@@ -7,7 +7,7 @@
 
 import app
 from app import db, current_user, ENTITY_MAPPING, ACTIVITY_TYPE
-from app.models.whitelist import Whitelist
+from app.models.whitelist import Whitelist, WhitelistException
 from app.models.metadata import Metadata, MetadataMapping
 from app.routes import tags_mapping
 from app.models.comments import Comments
@@ -70,8 +70,8 @@ def metadata_values(self):
     def save_metadata(self, metadata):
         for name, val in metadata.items():
             val = val if not type(val) == dict else val.get("value", None)
-            if not val:
-                continue
+            #if not val:
+            #    continue
 
             m = db.session.query(MetadataMapping).join(Metadata, Metadata.id == MetadataMapping.metadata_id).filter(
                 Metadata.key == name).filter(Metadata.artifact_type == ENTITY_MAPPING["DNS"]).filter(
@@ -85,6 +85,7 @@ def save_metadata(self, metadata):
                     Metadata.artifact_type == ENTITY_MAPPING["DNS"]).first()
                 db.session.add(MetadataMapping(value=val, metadata_id=m.id, artifact_id=self.id,
                                                created_user_id=current_user.id))
+                dirty = True
 
         try:
             db.session.commit()
@@ -123,7 +124,10 @@ def get_metadata_to_save(artifact, metadata, metadata_cache={}, user_cache={}):
                                                             created_user_id=current_user.id))
         return metadata_to_save
 
-    def to_dict(self, include_metadata=True, include_tags=True, include_comments=True):
+    def to_dict(self, include_metadata=True, include_tags=True, include_comments=True,
+                metadata_cache=None, users_cache=None, tags_mapping_cache=None,
+                comments_cache=None
+                ):
         d = dict(
             active=self.active,
             date_created=self.date_created.isoformat() if self.date_created else None,
@@ -135,26 +139,42 @@ def to_dict(self, include_metadata=True, include_tags=True, include_comments=Tru
             id=self.id,
             description=self.description,
             references=self.references,
-            created_user=self.created_user.to_dict(),
-            modified_user=self.modified_user.to_dict(),
-            owner_user=self.owner_user.to_dict() if self.owner_user else None,
         )
 
         if include_tags:
-            d["tags"] = tags_mapping.get_tags_for_source(self.__tablename__, self.id)
+            if tags_mapping_cache and tags_mapping_cache['c2dns'].get(self.id, None):
+                d["tags"] = tags_mapping_cache['c2dns'][self.id]
+            else:
+                d["tags"] = tags_mapping.get_tags_for_source(self.__tablename__, self.id)
 
         if include_comments:
-            d["comments"] = [comment.to_dict() for comment in Comments.query.filter_by(entity_id=self.id).filter_by(
-                entity_type=ENTITY_MAPPING["DNS"]).all()]
+            if comments_cache:
+                d["comments"] = comments_cache['c2dns'][self.id] if self.id in comments_cache['c2dns'] else []
+            else:
+                d["comments"] = [comment.to_dict() for comment in Comments.query.filter_by(entity_id=self.id).filter_by(
+                    entity_type=ENTITY_MAPPING["DNS"]).all()]
 
         if include_metadata:
-            metadata_values_dict = {}
-            metadata_keys = Metadata.get_metadata_keys("DNS")
-            metadata_values_dict = {m["metadata"]["key"]: m for m in
-                                    [entity.to_dict() for entity in self.metadata_values]}
-            for key in list(set(metadata_keys) - set(metadata_values_dict.keys())):
-                metadata_values_dict[key] = {}
-            d.update(dict(metadata=Metadata.get_metadata_dict("DNS"), metadata_values=metadata_values_dict))
+            if metadata_cache:
+                d["metadata"] = metadata_cache["DNS"][self.id]["metadata"] if metadata_cache["DNS"].get(self.id, None) and metadata_cache["DNS"][self.id].get("metadata", None) else {}
+                d["metadata_values"] = metadata_cache["DNS"][self.id]["metadata_values"] if metadata_cache["DNS"].get(self.id,None) and metadata_cache["DNS"][self.id].get("metadata_values", None) else {}
+            else:
+                metadata_values_dict = {}
+                metadata_keys = Metadata.get_metadata_keys("DNS")
+                metadata_values_dict = {m["metadata"]["key"]: m for m in
+                                        [entity.to_dict() for entity in self.metadata_values]}
+                for key in list(set(metadata_keys) - set(metadata_values_dict.keys())):
+                    metadata_values_dict[key] = {}
+                d.update(dict(metadata=Metadata.get_metadata_dict("DNS"), metadata_values=metadata_values_dict))
+
+        if users_cache:
+            d["created_user"] = users_cache.get(self.created_user_id, None)
+            d["modified_user"] = users_cache.get(self.modified_user_id, None)
+            d["owner_user"] = users_cache.get(self.owner_user_id, None)
+        else:
+            d["created_user"] = self.created_user.to_dict()
+            d["modified_user"] = self.modified_user.to_dict()
+            d["owner_user"] = self.owner_user.to_dict() if self.owner_user else None
 
         return d
 
@@ -206,120 +226,20 @@ def __repr__(self):
 
 @listens_for(C2dns, "before_insert")
 def run_against_whitelist(mapper, connect, target):
-    whitelist_enabled = cfg_settings.Cfg_settings.get_setting("ENABLE_DNS_WHITELIST_CHECK_ON_SAVE")
-    whitelist_states = cfg_settings.Cfg_settings.get_setting("WHITELIST_STATES")
-
-    if whitelist_enabled and distutils.util.strtobool(whitelist_enabled) and whitelist_states:
-
-        if target.state in whitelist_states.split(","):
-            domain_name = target.domain_name
-            target.domain_name = str(target.domain_name).lower()
-
-            abort_import = False
-
-            if not C2dns.WHITELIST_CACHE_LAST_UPDATE or not C2dns.WHITELIST_CACHE \
-                    or (time.time() - C2dns.WHITELIST_CACHE_LAST_UPDATE) > 60:
-                C2dns.WHITELIST_CACHE = Whitelist.query.all()
-                C2dns.WHITELIST_CACHE_LAST_UPDATE = time.time()
-
-            whitelists = C2dns.WHITELIST_CACHE
-
-            hit = None
-
-            for whitelist in whitelists:
-                wa = str(whitelist.whitelist_artifact)
-
-                try:
-                    ip = IPAddress(wa)
-                    continue
-                except ValueError:
-                    pass
-
-                try:
-                    cidr = IPNetwork(wa)
-                    continue
-                except ValueError:
-                    pass
-
-                try:
-                    regex = re.compile(wa)
-                    result = regex.search(domain_name)
-                    hit = wa
-                except:
-                    result = False
-
-                if result:
-                    abort_import = True
-                    break
-
-            if abort_import:
-                raise Exception('Failed Whitelist Validation on whitelist entry \'%s\'' % (hit))
-
-            if not current_user.admin:
-                release_state = cfg_states.Cfg_states.query.filter(cfg_states.Cfg_states.is_release_state > 0).first()
-                if release_state and target.state == release_state.state:
-                    abort(403)
-
+    if Whitelist.hits_whitelist(target.domain_name, target.state):
+        abort(412, f"Failed whitelist validation {target.domain_name}")
 
-@listens_for(C2dns, "before_update")
-def run_against_whitelist_before_update(mapper, connect, target):
-    whitelist_enabled = cfg_settings.Cfg_settings.get_setting("ENABLE_DNS_WHITELIST_CHECK_ON_SAVE")
-    whitelist_states = cfg_settings.Cfg_settings.get_setting("WHITELIST_STATES")
-
-    if whitelist_enabled and distutils.util.strtobool(whitelist_enabled) and whitelist_states:
-
-        if target.state in whitelist_states.split(","):
-            domain_name = target.domain_name
-            target.domain_name = str(target.domain_name).lower()
-
-            abort_import = False
-
-            if not C2dns.WHITELIST_CACHE_LAST_UPDATE or not C2dns.WHITELIST_CACHE \
-                or (time.time() - C2dns.WHITELIST_CACHE_LAST_UPDATE) > 60:
-                C2dns.WHITELIST_CACHE = Whitelist.query.all()
-                C2dns.WHITELIST_CACHE_LAST_UPDATE = time.time()
-
-            whitelists = C2dns.WHITELIST_CACHE
-
-            hit = None
-
-            for whitelist in whitelists:
-                wa = str(whitelist.whitelist_artifact)
-
-                try:
-                    ip = IPAddress(wa)
-                    continue
-                except ValueError:
-                    pass
-
-                try:
-                    cidr = IPNetwork(wa)
-                    continue
-                except ValueError:
-                    pass
-
-                try:
-                    regex = re.compile(wa)
-                    result = regex.search(domain_name)
-                    hit = wa
-                except:
-                    result = False
-
-                if result:
-                    abort_import = True
-                    break
-
-            if abort_import:
-                raise Exception('Failed Whitelist Validation on whitelist entry \'%s\'' % (hit))
-
-            if not current_user.admin:
-                release_state = cfg_states.Cfg_states.query.filter(cfg_states.Cfg_states.is_release_state > 0).first()
-                if release_state and target.state == release_state.state:
-                    abort(403)
+    if not current_user.admin:
+        release_state = cfg_states.Cfg_states.query.filter(cfg_states.Cfg_states.is_release_state > 0).first()
+        if release_state and target.state == release_state.state:
+            abort(403)
 
 
 @listens_for(C2dns, "before_update")
 def c2dns_before_update(mapper, connect, target):
+    if Whitelist.hits_whitelist(target.domain_name, target.state):
+        abort(412, f"Failed whitelist validation {target.domain_name}")
+
     if not current_user.admin:
         release_state = cfg_states.Cfg_states.query.filter(cfg_states.Cfg_states.is_release_state > 0).first()
         if release_state and target.state == release_state.state:
@@ -353,13 +273,11 @@ def dns_modified(mapper, connection, target):
                                       user_id=target.modified_user_id)
 
         changes = activity_log.get_modified_changes(target)
-        if changes["long"].__len__() > 0:
+        if changes.__len__() > 0:
             activity_log.log_activity(connection=connection,
                                       activity_type=list(ACTIVITY_TYPE.keys())[list(ACTIVITY_TYPE.keys()).index("ARTIFACT_MODIFIED")],
                                       activity_text="'%s' modified with changes: %s"
-                                                    % (target.domain_name, ', '.join(map(str, changes["long"]))),
-                                      activity_text_short="'%s' modified fields are: %s"
-                                                    % (target.domain_name, ', '.join(map(str, changes["short"]))),
+                                                    % (target.domain_name, ', '.join(map(str, changes))),
                                       activity_date=target.date_modified,
                                       entity_type=ENTITY_MAPPING["DNS"],
                                       entity_id=target.id,