Merge pull request #5 from ARGOeu/devel

Version 0.1

README.md

@@ -1,2 +1,79 @@
 # eosc-recommender-metrics
 A framework for counting the recommender metrics
+
+# Preprocessor v.1.0
+<p align="center">
+<a href="https://github.com/nikosT/Gisola">
+<img src="https://github.com/nikosT/eosc-recommender-metrics/blob/master/docs/Preprocessor.png" width="70%"/>
+</a>
+</p>
+
+# RS metrics v.1.0
+<p align="center">
+<a href="https://github.com/nikosT/Gisola">
+<img src="https://github.com/nikosT/eosc-recommender-metrics/blob/master/docs/RSmetrics.png" width="70%"/>
+</a>
+</p>
+
+
+
+
+# Dependencies
+1. Install Conda from here: https://docs.conda.io/projects/conda/en/latest/user-guide/install/linux.html. Tested on conda v 4.10.3.
+2. Run from terminal: `conda env create -f rsmetrics_env.yml`
+3. Run from terminal: `conda activate rsmetrics`
+4. Run from terminal: `chmod +x ./preprocessor.py ./rsmetrics.py`
+
+# Usage
+7. Run from terminal: `./preprocessor.py` in order to prepare the data for the RSmetrics
+8. Run from terminal: `./rsmetrics.py` to run RSmetrics
+
+
+## Reporting
+
+The reporting script generates an evalutation report in html format automatically served from a spawed localserver (default: http://localhost:8080)
+and automatically opens the default browser to present the report. 
+
+To execute the script issue:
+```
+chmod u+X ./report.py
+report.py
+```
+The script will automatically look for evaulation result files in the default folder `./data` and will output the report in the default folder: `./report`
+
+#### Additional script usage with parameters
+
+The `report.py` script can be used with the `--input` parameter: a path to a folder that the results from the evaluation process have been generated (default folder:`./data`). The report script can also take an `--output` parameter: a path to an output folder where the generated report will be served automatically.
+
+_Note:_ the script copies to the output folder all the necessary files such as `pre_metrics.json`, `metrics.json` as well as `report.html.prototype` renamed to `index.html` 
+
+```
+usage: report.py [-h] [-i STRING] [-o STRING] [-a STRING] [-p STRING]
+
+Generate report
+
+optional arguments:
+  -h, --help            show this help message and exit
+  -i STRING, --input STRING
+                        Input folder
+  -o STRING, --output STRING
+                        Output report folder
+  -a STRING, --address STRING
+                        Address to bind and serve the report
+  -p STRING, --port STRING
+                        Port to bind and serve the report
+```
+
+## Utilities
+
+### Get service catalog script (./get_service_catalog.py)
+
+This script contacts EOSC Marketplace catalog and generates a csv with a list of all available services, their name, id and url
+
+To execute the script issue:
+```
+chmod u+x ./get_service_catalog.py
+./get_service_catalog.py
+```
+
+_Tested with python 3.9_

config.yaml

@@ -0,0 +1,22 @@
+# Set the desired connector (e.g. MongoDB)
+Source:
+    MongoDB:
+        host: localhost
+        port: 27017
+        db: recommender_dev
+
+# Use the EOSC-Marketplace webpage
+# to associate page_id and service_id
+Marketplace:
+    download: true
+    path: ./page_map
+
+#Reward:
+#    transition: ./transition_rewards.csv
+
+# Calculate connector's metrics
+Metrics: true
+
+
+
+

environment.yml

@@ -0,0 +1,32 @@
+name: rsmetrics
+channels:
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=main
+  - _openmp_mutex=4.5=1_gnu
+  - ca-certificates=2022.3.29=h06a4308_0
+  - certifi=2021.10.8=py39h06a4308_2
+  - ld_impl_linux-64=2.35.1=h7274673_9
+  - libffi=3.3=he6710b0_2
+  - libgcc-ng=9.3.0=h5101ec6_17
+  - libgomp=9.3.0=h5101ec6_17
+  - libstdcxx-ng=9.3.0=hd4cf53a_17
+  - ncurses=6.3=h7f8727e_2
+  - openssl=1.1.1n=h7f8727e_0
+  - pip=21.2.4=py39h06a4308_0
+  - python=3.9.11=h12debd9_2
+  - readline=8.1.2=h7f8727e_1
+  - setuptools=58.0.4=py39h06a4308_0
+  - sqlite=3.38.2=hc218d9a_0
+  - tk=8.6.11=h1ccaba5_0
+  - tzdata=2022a=hda174b7_0
+  - wheel=0.37.1=pyhd3eb1b0_0
+  - xz=5.2.5=h7b6447c_0
+  - zlib=1.2.11=h7f8727e_4
+  - pip:
+    - charset-normalizer==2.0.12
+    - idna==3.3
+    - pymongo==4.0.2
+    - pyyaml==6.0
+    - requests==2.27.1
+    - urllib3==1.26.9

get_service_catalog.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+import requests
+from bs4 import BeautifulSoup
+import sys
+import csv
+import argparse
+
+
+
+# Main logic
+def main(args=None):
+    # call eosc marketplace with ample number of services per page: default = 1000
+    url = "https://marketplace.eosc-portal.eu/services?page=1&per_page={}".format(str(args.items))
+
+    print("Retrieving page: marketplace list of services... \nGrabbing url: {0}".format(url))
+    page = requests.get(url)
+
+    print("Page retrieved!\nGenerating results...")
+    soup = BeautifulSoup(page.content, 'html.parser')
+
+    # Find all h2 that contain the data-e2e attribute equal to service-id
+    results = soup.findAll("h2", {"data-e2e":"service-id"})
+    rows = []
+    # populate rows with each row = [service id, service name, service path]
+    for item in results:
+        a = item.findChildren("a",recursive=False)[0]
+        row = [int(item.attrs["data-service-id"]),item.text.strip(),a['href']]
+        rows.append(row)
+    # sort rows by id 
+    rows = sorted(rows, key=lambda x: x[0])
+
+    # output to csv
+    with open(args.output, "w") as f:
+        writer = csv.writer(f)
+        writer.writerows(rows)
+
+    print("File written to {}".format(args.output))
+
+
+# Parse arguments and call main 
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Retrieve service catalog from eosc marketplace")
+    parser.add_argument(
+        "-n", "--num-of-items", metavar="STRING", help="Number of items per page", required=False, dest="items", default="1000")
+    parser.add_argument(
+        "-o", "--output", metavar="STRING", help="Output csv file", required=False, dest="output", default="./service_catalog.csv")
+
+    # Parse the arguments
+    sys.exit(main(parser.parse_args()))