Updating Message Samples + MultiPoint

.github/workflows/ci.yml

@@ -0,0 +1,38 @@
+name: cv-manager
+
+on: [pull_request, push]
+
+jobs:
+  build_wzdx:
+    runs-on: ubuntu-latest
+    container:
+      image: python:3.12.2
+      options: --user root
+    steps:
+      - name: Checkout ${{ github.event.repository.name }}
+        uses: actions/checkout@v3
+
+      - name: Install Dependencies
+        run: |
+          apt-get update
+          apt-get -y install python3-coverage python3-pip python3-pytest
+
+      - name: Install Python Requirements
+        run: python3 -m pip install -r $GITHUB_WORKSPACE/requirements.txt
+
+      - name: Run Tests and Generate Coverage
+        continue-on-error: false
+        run: |
+          # Set PYTHONPATH and navigate to the tests directory
+          export PYTHONPATH=$PYTHONPATH:/usr/lib/python3/dist-packages
+          cd $GITHUB_WORKSPACE
+
+          # Run tests and generate coverage report
+          python3 -m coverage run -m pytest
+          python3 -m coverage xml --omit="/opt/*,/root/*,/tmp/*,/usr/*,/var/*,**/__init__.py"
+
+      - name: Archive Code Coverage Results
+        uses: actions/upload-artifact@v3
+        with:
+          name: build_api
+          path: cov.xml

.gitignore

@@ -8,7 +8,6 @@ icone_to_wzdx_test.geojson
 icone_wzdx_translated_output_message.geojson
 cotrip_translated_output_message.geojson
 cotrip_wzdx_translated_output_message.geojson
-.vscode/settings.json
 *.pyc
 *.geojson
 test-output.xml

.travis.yml

@@ -1,15 +1,13 @@
 language: python
 python:
-  - "3.6"
+  - '3.6'
 addons:
   sonarcloud:
-    organization: "cdot-cv"
+    organization: 'cdot-cv'
     token: ${SONAR_TOKEN}
 
-
 script:
   - coverage erase
-  - coverage run -m pytest 'test/'
+  - coverage run -m pytest 'tests/'
   - coverage xml -i
   - sonar-scanner
-

.vscode/settings.json

@@ -0,0 +1,56 @@
+{
+  "python.testing.unittestEnabled": false,
+  "python.testing.pytestEnabled": true,
+  "editor.defaultFormatter": "esbenp.prettier-vscode",
+  "editor.formatOnSave": true,
+  "editor.formatOnPaste": false,
+  "prettier.useEditorConfig": false,
+  "prettier.useTabs": false,
+  "prettier.configPath": ".vscode/.prettierrc.json",
+  "[python]": {
+    "editor.defaultFormatter": "ms-python.black-formatter",
+    "editor.formatOnSave": true
+  },
+  "cSpell.ignorePaths": ["*.json"],
+  "cSpell.words": [
+    "bdist",
+    "brng",
+    "cdot",
+    "codot",
+    "creationtime",
+    "Dataflows",
+    "dateutil",
+    "ellps",
+    "endtime",
+    "geopy",
+    "Geotab",
+    "hexidecimal",
+    "Hilgers",
+    "icone",
+    "inputfile",
+    "ipin",
+    "millis",
+    "navjoy",
+    "orjson",
+    "outputfile",
+    "pcms",
+    "pjson",
+    "polyline",
+    "polylines",
+    "protobuf",
+    "pypi",
+    "pyproj",
+    "pytest",
+    "pytz",
+    "restriping",
+    "rtdh",
+    "sdist",
+    "setuptools",
+    "srzmap",
+    "starttime",
+    "Trafficade",
+    "updatetime",
+    "USDOT",
+    "xmltodict"
+  ]
+}

README.md

@@ -70,7 +70,7 @@ python -m wzdx.raw_to_standard.{raw translator} inputfile.json --outputDir outpu
 Example usage:
 
 ```
-python -m wzdx.raw_to_standard.planned_events 'wzdx/sample_files/raw/planned_events/hwy_50.json'
+python -m wzdx.raw_to_standard.planned_events 'wzdx/sample_files/raw/planned_events/I70_resurfacing_2024_11_07.json'
 ```
 
 #### Standard to WZDx Conversion
@@ -82,7 +82,7 @@ python -m wzdx.standard_to_enhanced.{standard translator} inputfile.json --outpu
 Example usage:
 
 ```
-python -m wzdx.standard_to_enhanced.planned_events_translator 'wzdx/sample_files/standard/planned_events/standard_planned_event_OpenTMS-Event2702170538_eastbound.json'
+python -m wzdx.standard_to_enhanced.planned_events_translator 'wzdx/sample_files/standard/planned_events/standard_planned_event_OpenTMS-Event20643308360_westbound.json'
 ```
 
 ### Execution for iCone translator
@@ -96,7 +96,7 @@ python -m wzdx.raw_to_standard.icone inputfile.json --outputDir outputDirectory
 Example usage:
 
 ```
-python -m wzdx.raw_to_standard.icone 'wzdx/sample_files/raw/icone/incident_short.xml'
+python -m wzdx.raw_to_standard.icone 'wzdx/sample_files/raw/icone/icone_ftp_20241107-235100.xml'
 ```
 
 #### Standard to WZDx Conversion
@@ -108,7 +108,7 @@ python -m wzdx.standard_to_enhanced.icone_translator inputfile.json --outputFile
 Example usage:
 
 ```
-python -m wzdx.standard_to_enhanced.icone_translator 'wzdx/sample_files/standard/icone/standard_icone_1245_1633444335.json'
+python -m wzdx.standard_to_enhanced.icone_translator 'wzdx/sample_files/standard/icone/standard_icone_U13632784_20241107235100_1731023924_unknown.json'
 ```
 
 ### Execution for NavJoy 568 translator

setup.py

@@ -4,7 +4,7 @@
 import pathlib
 from setuptools import setup, find_packages
 
-VERSION = "1.3.0"
+VERSION = "1.3.1"
 DESCRIPTION = "CDOT Work Zone WZDx Translators"
 HERE = pathlib.Path(__file__).parent
 README = (HERE / "README.md").read_text()

tests/data/raw_to_standard/planned_events_test_expected_results.py

@@ -330,14 +330,14 @@
                     "MMax": 449.589,
                     "MMin": 0.0,
                     "Measure": 60.0,
-                    "Route": "070A",
+                    "Route": "070A_DEC",
                 },
                 "route_details_start": {
                     "Distance": 0.91,
                     "MMax": 449.589,
                     "MMin": 0.0,
                     "Measure": 50.0,
-                    "Route": "070A",
+                    "Route": "070A_DEC",
                 },
                 "condition_1": False,
                 "beginning_milepost": 50.0,

tests/data/standard_to_enhanced/planned_events_translator_data.py

@@ -1,4 +1,4 @@
-test_parse_work_zone_linestring_standard = {
+test_parse_work_zone_multipoint_standard = {
     "rtdh_timestamp": 1642036259.3099449,
     "rtdh_message_id": "42fe21b8-102b-43e8-8668-23c55334a201",
     "event": {
@@ -36,7 +36,7 @@
     },
 }
 
-test_parse_work_zone_linestring_expected = {
+test_parse_work_zone_multipoint_expected = {
     "id": "ce32f52d-1150-5bc9-a9df-7b5c71217d26",
     "type": "Feature",
     "properties": {
@@ -68,7 +68,7 @@
         ],
     },
     "geometry": {
-        "type": "LineString",
+        "type": "MultiPoint",
         "coordinates": [[-108.279106, 39.195663], [-108.218549, 39.302392]],
     },
 }
@@ -166,7 +166,7 @@
                 ],
             },
             "geometry": {
-                "type": "LineString",
+                "type": "MultiPoint",
                 "coordinates": [[-108.279106, 39.195663], [-108.218549, 39.302392]],
             },
         }

tests/standard_to_enhanced/planned_events_translator_test.py

@@ -38,11 +38,11 @@ def init_datetime_mocks(mock_dts):
 )
 @unittest.mock.patch("wzdx.standard_to_enhanced.navjoy_translator.datetime")
 @unittest.mock.patch("wzdx.tools.wzdx_translator.datetime")
-def test_parse_work_zone_linestring(mock_dt, mock_dt_3):
+def test_parse_work_zone_multipoint(mock_dt, mock_dt_3):
     init_datetime_mocks([mock_dt, mock_dt_3])
-    standard = planned_events_translator_data.test_parse_work_zone_linestring_standard
+    standard = planned_events_translator_data.test_parse_work_zone_multipoint_standard
     expected_feature = (
-        planned_events_translator_data.test_parse_work_zone_linestring_expected
+        planned_events_translator_data.test_parse_work_zone_multipoint_expected
     )
 
     test_feature = planned_events_translator.parse_work_zone(standard)

tests/tools/cdot_geospatial_api_test.py

@@ -78,10 +78,20 @@ def test_get_route_between_measures():
 
 
 def test_get_route_between_measures():
+    routeId = "070A"
+    startMeasure = 50
+    endMeasure = 60
+    actual = cdot_geospatial_api.GeospatialApi().get_route_between_measures(
+        routeId, startMeasure, endMeasure, compressed=True, adjustRoute=False
+    )
+    assert len(actual) == 108
+
+
+def test_get_route_between_measures_allow_reversal():
     routeId = "070A"
     startMeasure = 50
     endMeasure = 60
     actual = cdot_geospatial_api.GeospatialApi().get_route_between_measures(
         routeId, startMeasure, endMeasure, compressed=True
     )
-    assert len(actual) == 109
+    assert len(actual) == 81

wzdx/experimental_combination/attenuator.py

@@ -23,9 +23,7 @@ def main():
     geotab_avl = [json.loads(open(geotabFile, "r").read())]
     outputPath = output_dir + "/wzdx_attenuator_combined.geojson"
     if updateDates == "true":
-        geotab_avl[0]["avl_location"]["source"]["collection_timestamp"] = (
-            date_tools.get_current_ts_millis() / 1000
-        )
+        geotab_avl[0]["avl_location"]["source"]["collection_timestamp"] = datetime.now()
         wzdx[0]["features"][0]["properties"]["start_date"] = (
             date_tools.get_iso_string_from_datetime(datetime.now() - timedelta(days=2))
         )
@@ -107,9 +105,19 @@ def validate_dates(geotab: dict, wzdx: dict) -> bool:
     Returns:
         bool: Whether the Geotab date falls within the WZDx date range
     """
-    geotab_date = date_tools.get_unix_from_iso_string(
-        geotab["avl_location"]["source"]["collection_timestamp"]
-    )
+    if type(geotab["avl_location"]["source"]["collection_timestamp"]) == datetime:
+        geotab_date = date_tools.date_to_unix(
+            geotab["avl_location"]["source"]["collection_timestamp"]
+        )
+    elif type(geotab["avl_location"]["source"]["collection_timestamp"]) == str:
+        geotab_date = date_tools.get_unix_from_iso_string(
+            geotab["avl_location"]["source"]["collection_timestamp"]
+        )
+    else:
+        logging.error(
+            f"Invalid Geotab date format: avl_location.source.collection_timestamp is type {type(geotab['avl_location']['source']['collection_timestamp'])}"
+        )
+        return False
     logging.debug(f"Geotab: {json.dumps(geotab)}")
     if not geotab_date:
         geotab_date = (

wzdx/experimental_combination/icone.py

@@ -47,7 +47,9 @@ def main(outputPath="./tests/data/output/wzdx_icone_combined.json"):
     combined_events = get_combined_events(icone, wzdx)
 
     if len(combined_events) == 0:
-        print("No overlapping events found between WZDx and iCone data. See logs for more information.")
+        print(
+            "No overlapping events found between WZDx and iCone data. See logs for more information."
+        )
     else:
         with open(outputPath, "w+") as f:
             f.write(json.dumps(combined_events, indent=2))
@@ -383,7 +385,6 @@ def identify_overlapping_features_icone(
         logging.debug("No routes found for wzdx")
         return []
 
-    logging.error("Match iCone: " + str(icone_standard_msgs) + str(wzdx_msgs))
     # Step 3: Identify overlapping events
     for wzdx_route_id, wzdx_matched_msgs in wzdx_routes.items():
         matching_icone_routes = icone_routes.get(wzdx_route_id, [])

wzdx/monitoring/experimental/main.py

@@ -31,7 +31,7 @@ def upload_logs(contents, path, bucket_name):
             blob.upload_from_string(contents)
             logging.info(f"{path} uploaded to {bucket_name}")
         else:
-            logging.warn(f"blob already exists for {path}")
+            logging.warning(f"blob already exists for {path}")
         return True
     except Exception as e:
         logging.error(

wzdx/raw_to_standard/icone.py

@@ -20,9 +20,8 @@ def main():
     generated_messages = generate_standard_messages_from_string(input_file_contents)
 
     generated_files_list = []
-    features = json.loads(open(f"{output_dir}/icone_feature_collection.geojson").read())
+    features = []
 
-    wzdx_msg = {}
     for message in generated_messages:
         output_path = f"{output_dir}/icone_{message['event']['source']['id']}_{round(message['rtdh_timestamp'])}_{message['event']['detail']['direction']}.json"
         open(output_path, "w+").write(json.dumps(message, indent=2))

wzdx/raw_to_standard/navjoy_568.py

@@ -433,12 +433,12 @@ def validate_closure(obj: dict | OrderedDict) -> bool:
     start_time = date_tools.parse_datetime_from_iso_string(starttime_string)
     end_time = date_tools.parse_datetime_from_iso_string(endtime_string)
     if not start_time:
-        logging.error(
+        logging.warning(
             f"Invalid incident with id = {id}. Unsupported start time format: {start_time}"
         )
         return False
     elif endtime_string and not end_time:
-        logging.error(
+        logging.warning(
             f"Invalid incident with id = {id}. Unsupported end time format: {end_time}"
         )
         return False