Merge pull request AirbusDefenceAndSpace#2 from AirbusDefenceAndSpace…

…/https

HTTPS (#1)

README.md

@@ -114,6 +114,7 @@ POSIX storage :
 You can configure the server with the following environement variables : 
 
  * CADOR_PORT : port where the server run
+ * CADOR_USE_SSL : whether to use SSL or not (call the service with `https` scheme if activated)
  * PROCESSING_SERVER : address of the algorithm docker to bin with
  * KAFKA_BROKERS_REQUEST : queue_in to read request orders (only kafka is available for now). Not mandatory, if not provided only REST API is available
  * EOPAAS_JOB_REQUEST_TOPIC : queue_in topic, mandatory if queue_in specified

cador/app.py

@@ -2,6 +2,8 @@
 import asyncio
 import os
 
+from OpenSSL import crypto
+
 from sanic import Sanic
 from sanic_openapi import swagger_blueprint, openapi_blueprint
 
@@ -71,6 +73,7 @@ def start():
     # environment variables
     try:
         cador_port = os.environ['CADOR_PORT']
+        use_ssl = bool(os.environ.get('CADOR_USE_SSL'))
         algo_server = os.environ['PROCESSING_SERVER']
 
         input_brokers = os.environ.get('KAFKA_BROKERS_REQUEST')
@@ -101,7 +104,37 @@ def start():
     cador_config.CADOR_QUEUES['tags'] = KafkaTopic('tags', tags_brokers, tags_topic)
 
     log_app_conf()
-    cador_app.run(host='0.0.0.0', port=int(cador_port))
+
+    if use_ssl:
+        configure_ssl()
+        cador_app.run(host='0.0.0.0', port=int(cador_port), ssl={'cert': "/tmp/cert.pem", 'key': "/tmp/key.pem"})
+    else:
+        cador_app.run(host='0.0.0.0', port=int(cador_port))
+
+
+def configure_ssl() -> None:
+    # create a key pair
+    k = crypto.PKey()
+    k.generate_key(crypto.TYPE_RSA, 4096)
+    # create a self-signed cert
+    cert = crypto.X509()
+    cert.get_subject().C = "FR"
+    cert.get_subject().ST = "stateOrProvinceName"
+    cert.get_subject().L = "localityName"
+    cert.get_subject().O = "organizationName"
+    cert.get_subject().OU = "organizationUnitName"
+    cert.get_subject().CN = "commonName"
+    cert.get_subject().emailAddress = "cador@localhost"
+    cert.set_serial_number(0)
+    cert.gmtime_adj_notBefore(0)
+    cert.gmtime_adj_notAfter(10*365*24*60*60)
+    cert.set_issuer(cert.get_subject())
+    cert.set_pubkey(k)
+    cert.sign(k, 'sha512')
+    with open(os.path.join('tmp', "cert.pem"), "wt") as f:
+        f.write(crypto.dump_certificate(crypto.FILETYPE_PEM, cert).decode("utf-8"))
+    with open(os.path.join('tmp', "key.pem"), "wt") as f:
+        f.write(crypto.dump_privatekey(crypto.FILETYPE_PEM, k).decode("utf-8"))
 
 
 def log_app_conf():

cador/core/RequestService.py

@@ -18,7 +18,7 @@ def get_service_info():
         Get service information from describe, call the async routine
         """
         if not cador_config.cador_cache_service_description:
-            r = requests.get(cador_config.CADOR_ALGO_SRV + '/describe')
+            r = requests.get(cador_config.CADOR_ALGO_SRV + '/describe', verify=False)
             if r.status_code != 200:
                 message = 'Bad response : server algorithm send {0} with code {1}'.format(r.text, r.status_code)
                 raise ValueError(message)
@@ -55,12 +55,12 @@ async def async_processing(payload):
 
     @staticmethod
     async def async_post(url, payload):
-        r = await RequestService.async_exec(partial(requests.post, url, json=payload))
+        r = await RequestService.async_exec(partial(requests.post, url, json=payload, verify=False))
         return response.raw(r.content, status=r.status_code)
 
     @staticmethod
     async def async_get(url):
-        return await RequestService.async_exec(partial(requests.get, url))
+        return await RequestService.async_exec(partial(requests.get, url, verify=False))
 
     @staticmethod
     async def async_get_text(url):
@@ -101,7 +101,7 @@ async def async_get_file(url):
 
     @staticmethod
     async def async_delete(url):
-        return await RequestService.async_exec(partial(requests.delete, url))
+        return await RequestService.async_exec(partial(requests.delete, url, verify=False))
 
     @staticmethod
     async def async_delete_json(url):
@@ -111,7 +111,7 @@ async def async_delete_json(url):
         :return a Sanic response
         """
         try:
-            r = await RequestService.async_exec(partial(requests.delete, url))
+            r = await RequestService.async_exec(partial(requests.delete, url, verify=False))
             return response.json(r.json(), headers=RequestService.HEADERS, status=r.status_code)
         except Exception as e:
             log.exception(e)
@@ -126,7 +126,7 @@ async def async_put(url, payload):
         :return a Sanic response
         """
         try:
-            r = await RequestService.async_exec(partial(requests.put, url=url, json=payload))
+            r = await RequestService.async_exec(partial(requests.put, url=url, json=payload, verify=False))
             return response.json(r.json(), status=r.status_code)
         except Exception as e:
             log.exception(e)

cador/core/StorageService.py

@@ -52,10 +52,11 @@ def __init__(self, config: dict):
         self.container_folder = os.path.join(*parts[1:]) if len(parts) > 1 else ''
 
         self.s3 = boto3.resource('s3',
-                                 use_ssl=False,
+                                 use_ssl=config.get('ssl') or False,
                                  endpoint_url=(config['url']),
                                  aws_access_key_id=(config['key']),
-                                 aws_secret_access_key=(config['secret']))
+                                 aws_secret_access_key=(config['secret']),
+                                 verify=False)
 
     def init(self):
         try:

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:18.04
+FROM ubuntu:20.04
 
 ARG ENV
 

docker/Makefile

@@ -1,4 +1,4 @@
-VERSION=0.7.7-ubuntu
+VERSION=0.7.8-ubuntu
 
 all: build
 

requirements.txt

@@ -2,9 +2,10 @@ Sanic==0.8.3
 sanic-openapi
 requests-file
 jsonschema
-aiokafka==0.4.0
+aiokafka
 apache-libcloud
 trio
-kafka
+kafka-python
 shapely
 boto3
+pyopenssl

setup.py

@@ -2,7 +2,7 @@
 
 setup(
     name='cador',
-    version='0.7.7',
+    version='0.7.8',
     packages=find_packages(),
     maintainer='Airbus Defense & Space',
     description='Geo Processing Platform Cradle for Docker Algorithm',

tests/features/resources/cador/docker-compose-ssl.yml

@@ -0,0 +1,68 @@
+version: "3"
+services:
+  object-storage:
+    image: minio/minio
+    volumes:
+    - /home:/home
+    ports:
+    - "9000:9000"
+    environment:
+      MINIO_ACCESS_KEY:
+      MINIO_SECRET_KEY:
+    command: server /tmp
+    networks:
+    - private-network
+  processing:
+    image: algo-service:1.0.0.dev0
+    networks:
+    - private-network
+    ports:
+    - 8080:8080
+    volumes:
+    - /home:/home
+  cradle:
+    entrypoint: >
+      python3 -c "from cador import app; app.start()"
+    image: cador:0.7.8-ubuntu
+    networks:
+    - private-network
+    ports:
+    - 9443:8443
+    environment:
+      CADOR_PORT: 8443
+      CADOR_USE_SSL: "true"
+      PROCESSING_SERVER: https://processing:8080/api/v1
+      KAFKA_BROKERS_REQUEST: kafka:9092
+      EOPAAS_JOB_REQUEST_TOPIC: ipcloud_requests
+      # KAFKA_CONSUMER_GROUP: my_group
+      KAFKA_BROKERS_OUTPUT: kafka:9092
+      EOPAAS_JOB_OUTPUT_TOPIC: ipcloud_out
+      KAFKA_BROKERS_STATUS: kafka:9092
+      EOPAAS_STATUS_UPDATE_TOPIC: ipcloud_status
+      KAFKA_BROKERS_TAGS: kafka:9092
+      EOPAAS_JOB_TAGS_TOPIC: ipcloud_tags
+      CADOR_TIMEOUT_SECONDS: 5
+    volumes:
+    - /home:/home
+    depends_on:
+    - "object-storage"
+  zookeeper:
+    image: wurstmeister/zookeeper
+    networks:
+    - private-network
+    ports:
+    - "2181:2181"
+  kafka:
+    image: wurstmeister/kafka
+    networks:
+    - private-network
+    ports:
+    - "9092:9092"
+    environment:
+      KAFKA_ADVERTISED_HOST_NAME: kafka
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+    volumes:
+    - /var/run/docker.sock:/var/run/docker.sock
+networks:
+  private-network:
+    driver: bridge

tests/features/steps/steps_cador.py

@@ -91,7 +91,7 @@ def _generate_payload(path, image_prefix='file://'):
 
 
 def download_result(URL, link):
-    response = requests.get('{}{}'.format(URL, link))
+    response = requests.get('{}{}'.format(URL, link), verify=False)
     tmp_path = '/tmp/{}'.format(str(uuid.uuid4()))
     with open(tmp_path, 'wb') as fd:
         for chunk in response.iter_content(chunk_size=128):