From 2e23054fb49390701795830d104bc80dd35f5954 Mon Sep 17 00:00:00 2001 From: Hagellach37 Date: Tue, 17 Aug 2021 18:33:10 +0200 Subject: [PATCH 01/62] add change detection tutorial for building damage mapping in haiti --- ...orial_damaged_buildings_haiti_screens.json | 86 +++++++++++++++++++ ...rial_damaged_buildings_haiti_tasks.geojson | 12 +++ 2 files changed, 98 insertions(+) create mode 100644 mapswipe_workers/sample_data/change_detection/change_detection_tutorial_damaged_buildings_haiti_screens.json create mode 100644 mapswipe_workers/sample_data/change_detection/change_detection_tutorial_damaged_buildings_haiti_tasks.geojson diff --git a/mapswipe_workers/sample_data/change_detection/change_detection_tutorial_damaged_buildings_haiti_screens.json b/mapswipe_workers/sample_data/change_detection/change_detection_tutorial_damaged_buildings_haiti_screens.json new file mode 100644 index 000000000..67db73b25 --- /dev/null +++ b/mapswipe_workers/sample_data/change_detection/change_detection_tutorial_damaged_buildings_haiti_screens.json @@ -0,0 +1,86 @@ +{ "1": { + "hint" : { + "description" : "Compare the building outlines in the two images.", + "icon" : "swipe-left", + "title" : "There is severe damage visible." + }, + "instructions" : { + "description" : "Tap once whenever you spot damaged settlements. Swipe to learn more.", + "icon" : "tap-1", + "title" : "There is a destroyed building here." + }, + "success" : { + "description" : "Swipe to the next screen to look for more.", + "icon" : "check", + "title" : "You correctly mapped this task!" + } +}, +"2": { + "hint" : { + "description" : "Tap twice whenever you are not sure.", + "icon" : "swipe-left", + "title" : "Yellow indicates that settlements might have been damaged." + }, + "instructions" : { + "description" : "Tap the square twice to mark it yellow indicating that there might be damage.", + "icon" : "tap-2", + "title" : "This is difficult." + }, + "success" : { + "description" : "Swipe to the next screen to learn more.", + "icon" : "check", + "title" : "Nice. Use this whenever you are not sure." + } +}, +"3": { + "hint" : { + "description" : "The buildings look the same in the two images.", + "icon" : "swipe-left", + "title" : "No need to tap here." + }, + "instructions" : { + "description" : "Just swipe left to the next screen.", + "icon" : "swipe-left", + "title" : "No building damage visible here." + }, + "success" : { + "description" : "Swipe to keep on going", + "icon" : "check", + "title" : "Great. We can't spot damaged buildings here." + } +}, +"4": { + "hint" : { + "description" : "There might be other objects such as tents, but still the buildings are okay.", + "icon" : "swipe-left", + "title" : "Don't get distracted." + }, + "instructions" : { + "description" : "Again, just swipe left.", + "icon" : "swipe-left", + "title" : "Spontaneous shelters but not building damage." + }, + "success" : { + "description" : "Swipe to the next screen to learn more.", + "icon" : "check", + "title" : "Correct!" + } +}, +"5": { + "hint" : { + "description" : "Only map building damage.", + "icon" : "swipe-left", + "title" : "We’ve marked the correct answers" + }, + "instructions" : { + "description" : "If there are no buildings, just swipe left.", + "icon" : "swipe-left", + "title" : "No buildings at all" + }, + "success" : { + "description" : "Swipe to practice some more.", + "icon" : "check", + "title" : "Well done, you got it right!" + } +} +} diff --git a/mapswipe_workers/sample_data/change_detection/change_detection_tutorial_damaged_buildings_haiti_tasks.geojson b/mapswipe_workers/sample_data/change_detection/change_detection_tutorial_damaged_buildings_haiti_tasks.geojson new file mode 100644 index 000000000..655fbb885 --- /dev/null +++ b/mapswipe_workers/sample_data/change_detection/change_detection_tutorial_damaged_buildings_haiti_tasks.geojson @@ -0,0 +1,12 @@ +{ +"type": "FeatureCollection", +"name": "haiti_tutorial_tasks", +"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } }, +"features": [ +{ "type": "Feature", "properties": { "groupId": "g106", "taskId": "19-154730-235162", "tile_x": 154730, "tile_y": 235162, "tile_z": 19, "screen": 3, "reference": 0 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ -73.755340576171903, 18.2123937258254, 0.0 ], [ -73.754653930664105, 18.2123937258254, 0.0 ], [ -73.754653930664105, 18.2117414769682, 0.0 ], [ -73.755340576171903, 18.2117414769682, 0.0 ], [ -73.755340576171903, 18.2123937258254, 0.0 ] ] ] } }, +{ "type": "Feature", "properties": { "groupId": "g117", "taskId": "19-154731-235194", "tile_x": 154731, "tile_y": 235194, "tile_z": 19, "screen": 4, "reference": 0 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ -73.754653930664105, 18.191520551066599, 0.0 ], [ -73.753967285156193, 18.191520551066599, 0.0 ], [ -73.753967285156193, 18.1908682240724, 0.0 ], [ -73.754653930664105, 18.1908682240724, 0.0 ], [ -73.754653930664105, 18.191520551066599, 0.0 ] ] ] } }, +{ "type": "Feature", "properties": { "groupId": "g117", "taskId": "19-154741-235193", "tile_x": 154741, "tile_y": 235193, "tile_z": 19, "screen": 1, "reference": 1 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ -73.747787475585895, 18.192172875620201, 0.0 ], [ -73.747100830078097, 18.192172875620201, 0.0 ], [ -73.747100830078097, 18.191520551066599, 0.0 ], [ -73.747787475585895, 18.191520551066599, 0.0 ], [ -73.747787475585895, 18.192172875620201, 0.0 ] ] ] } }, +{ "type": "Feature", "properties": { "groupId": "g117", "taskId": "19-154743-235193", "tile_x": 154743, "tile_y": 235193, "tile_z": 19, "screen": 2, "reference": 2 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ -73.746414184570298, 18.192172875620201, 0.0 ], [ -73.7457275390625, 18.192172875620201, 0.0 ], [ -73.7457275390625, 18.191520551066599, 0.0 ], [ -73.746414184570298, 18.191520551066599, 0.0 ], [ -73.746414184570298, 18.192172875620201, 0.0 ] ] ] } }, +{ "type": "Feature", "properties": { "groupId": "g119", "taskId": "19-154723-235201", "tile_x": 154723, "tile_y": 235201, "tile_z": 19, "screen": 5, "reference": 0 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ -73.760147094726605, 18.1869542108583, 0.0 ], [ -73.759460449218807, 18.1869542108583, 0.0 ], [ -73.759460449218807, 18.1863018667818, 0.0 ], [ -73.760147094726605, 18.1863018667818, 0.0 ], [ -73.760147094726605, 18.1869542108583, 0.0 ] ] ] } } +] +} From a638389cceab9105037c0f7b2179baf8347c7a2c Mon Sep 17 00:00:00 2001 From: Hagellach37 Date: Mon, 1 Nov 2021 00:09:36 +0100 Subject: [PATCH 02/62] update tasks geojson because imagery has changed --- .../build_area_tutorial_tasks.geojson | 98 +++++++++---------- 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/mapswipe_workers/sample_data/build_area/build_area_tutorial_tasks.geojson b/mapswipe_workers/sample_data/build_area/build_area_tutorial_tasks.geojson index 5e5695052..32f0cb4e1 100644 --- a/mapswipe_workers/sample_data/build_area/build_area_tutorial_tasks.geojson +++ b/mapswipe_workers/sample_data/build_area/build_area_tutorial_tasks.geojson @@ -1,55 +1,55 @@ { "type": "FeatureCollection", -"name": "build_area_tutorial_tasks", +"name": "build_area_tutorial_tasks_updated", "crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } }, "features": [ -{ "type": "Feature", "properties": { "taskid": "18-64984-120438", "reference": 3, "category": "bad_clouds_1", "screen": 6, "tile_x": 64984, "tile_y": 120438, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ -90.75805664, 14.44797921 ], [ -90.75668335, 14.44797921 ], [ -90.75668335, 14.44664935 ], [ -90.75805664, 14.44664935 ], [ -90.75805664, 14.44797921 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-64984-120439", "reference": 0, "category": "bad_clouds_1", "screen": 6, "tile_x": 64984, "tile_y": 120439, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ -90.75805664, 14.44664935 ], [ -90.75668335, 14.44664935 ], [ -90.75668335, 14.44531948 ], [ -90.75805664, 14.44531948 ], [ -90.75805664, 14.44664935 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-64984-120437", "reference": 3, "category": "bad_clouds_1", "screen": 6, "tile_x": 64984, "tile_y": 120437, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ -90.75805664, 14.44930907 ], [ -90.75668335, 14.44930907 ], [ -90.75668335, 14.44797921 ], [ -90.75805664, 14.44797921 ], [ -90.75805664, 14.44930907 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-64985-120439", "reference": 3, "category": "bad_clouds_1", "screen": 6, "tile_x": 64985, "tile_y": 120439, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ -90.75668335, 14.44664935 ], [ -90.75531006, 14.44664935 ], [ -90.75531006, 14.44531948 ], [ -90.75668335, 14.44531948 ], [ -90.75668335, 14.44664935 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-64985-120437", "reference": 3, "category": "bad_clouds_1", "screen": 6, "tile_x": 64985, "tile_y": 120437, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ -90.75668335, 14.44930907 ], [ -90.75531006, 14.44930907 ], [ -90.75531006, 14.44797921 ], [ -90.75668335, 14.44797921 ], [ -90.75668335, 14.44930907 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-64985-120438", "reference": 3, "category": "bad_clouds_1", "screen": 6, "tile_x": 64985, "tile_y": 120438, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ -90.75668335, 14.44797921 ], [ -90.75531006, 14.44797921 ], [ -90.75531006, 14.44664935 ], [ -90.75668335, 14.44664935 ], [ -90.75668335, 14.44797921 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-208640-120212", "reference": 0, "category": "bad_no_imagery_1", "screen": 7, "tile_x": 208640, "tile_y": 120212, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 106.5234375, 14.74832321 ], [ 106.52481079, 14.74832321 ], [ 106.52481079, 14.74699516 ], [ 106.5234375, 14.74699516 ], [ 106.5234375, 14.74832321 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-208640-120213", "reference": 0, "category": "bad_no_imagery_1", "screen": 7, "tile_x": 208640, "tile_y": 120213, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 106.5234375, 14.74699516 ], [ 106.52481079, 14.74699516 ], [ 106.52481079, 14.74566711 ], [ 106.5234375, 14.74566711 ], [ 106.5234375, 14.74699516 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-208640-120214", "reference": 0, "category": "bad_no_imagery_1", "screen": 7, "tile_x": 208640, "tile_y": 120214, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 106.5234375, 14.74566711 ], [ 106.52481079, 14.74566711 ], [ 106.52481079, 14.74433904 ], [ 106.5234375, 14.74433904 ], [ 106.5234375, 14.74566711 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-208639-120213", "reference": 3, "category": "bad_no_imagery_1", "screen": 7, "tile_x": 208639, "tile_y": 120213, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 106.52206421, 14.74699516 ], [ 106.5234375, 14.74699516 ], [ 106.5234375, 14.74566711 ], [ 106.52206421, 14.74566711 ], [ 106.52206421, 14.74699516 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-208639-120214", "reference": 3, "category": "bad_no_imagery_1", "screen": 7, "tile_x": 208639, "tile_y": 120214, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 106.52206421, 14.74566711 ], [ 106.5234375, 14.74566711 ], [ 106.5234375, 14.74433904 ], [ 106.52206421, 14.74433904 ], [ 106.52206421, 14.74566711 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-208639-120212", "reference": 3, "category": "bad_no_imagery_1", "screen": 7, "tile_x": 208639, "tile_y": 120212, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 106.52206421, 14.74832321 ], [ 106.5234375, 14.74832321 ], [ 106.5234375, 14.74699516 ], [ 106.52206421, 14.74699516 ], [ 106.52206421, 14.74832321 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-157556-134648", "reference": 3, "category": "bad_no_imagery_2", "screen": 8, "tile_x": 157556, "tile_y": 134648, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 36.37023926, -4.90488679 ], [ 36.37161255, -4.90488679 ], [ 36.37161255, -4.90625506 ], [ 36.37023926, -4.90625506 ], [ 36.37023926, -4.90488679 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-157555-134649", "reference": 3, "category": "bad_no_imagery_2", "screen": 8, "tile_x": 157555, "tile_y": 134649, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 36.36886597, -4.90625506 ], [ 36.37023926, -4.90625506 ], [ 36.37023926, -4.90762331 ], [ 36.36886597, -4.90762331 ], [ 36.36886597, -4.90625506 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-157556-134649", "reference": 3, "category": "bad_no_imagery_2", "screen": 8, "tile_x": 157556, "tile_y": 134649, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 36.37023926, -4.90625506 ], [ 36.37161255, -4.90625506 ], [ 36.37161255, -4.90762331 ], [ 36.37023926, -4.90762331 ], [ 36.37023926, -4.90625506 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-157555-134647", "reference": 3, "category": "bad_no_imagery_2", "screen": 8, "tile_x": 157555, "tile_y": 134647, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 36.36886597, -4.90351853 ], [ 36.37023926, -4.90351853 ], [ 36.37023926, -4.90488679 ], [ 36.36886597, -4.90488679 ], [ 36.36886597, -4.90351853 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-157555-134648", "reference": 3, "category": "bad_no_imagery_2", "screen": 8, "tile_x": 157555, "tile_y": 134648, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 36.36886597, -4.90488679 ], [ 36.37023926, -4.90488679 ], [ 36.37023926, -4.90625506 ], [ 36.36886597, -4.90625506 ], [ 36.36886597, -4.90488679 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-157556-134647", "reference": 3, "category": "bad_no_imagery_2", "screen": 8, "tile_x": 157556, "tile_y": 134647, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 36.37023926, -4.90351853 ], [ 36.37161255, -4.90351853 ], [ 36.37161255, -4.90488679 ], [ 36.37023926, -4.90488679 ], [ 36.37023926, -4.90351853 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-208684-119949", "reference": 1, "category": "building_easy_2", "screen": 2, "tile_x": 208684, "tile_y": 119949, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 106.5838623, 15.09731698 ], [ 106.5852356, 15.09731698 ], [ 106.5852356, 15.09599108 ], [ 106.5838623, 15.09599108 ], [ 106.5838623, 15.09731698 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-208683-119949", "reference": 1, "category": "building_easy_2", "screen": 2, "tile_x": 208683, "tile_y": 119949, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 106.58248901, 15.09731698 ], [ 106.5838623, 15.09731698 ], [ 106.5838623, 15.09599108 ], [ 106.58248901, 15.09599108 ], [ 106.58248901, 15.09731698 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-208683-119950", "reference": 1, "category": "building_easy_2", "screen": 2, "tile_x": 208683, "tile_y": 119950, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 106.58248901, 15.09599108 ], [ 106.5838623, 15.09599108 ], [ 106.5838623, 15.09466518 ], [ 106.58248901, 15.09466518 ], [ 106.58248901, 15.09599108 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-208684-119950", "reference": 0, "category": "building_easy_2", "screen": 2, "tile_x": 208684, "tile_y": 119950, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 106.5838623, 15.09599108 ], [ 106.5852356, 15.09599108 ], [ 106.5852356, 15.09466518 ], [ 106.5838623, 15.09466518 ], [ 106.5838623, 15.09599108 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-208683-119948", "reference": 0, "category": "building_easy_2", "screen": 2, "tile_x": 208683, "tile_y": 119948, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 106.58248901, 15.09864287 ], [ 106.5838623, 15.09864287 ], [ 106.5838623, 15.09731698 ], [ 106.58248901, 15.09731698 ], [ 106.58248901, 15.09864287 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-208684-119948", "reference": 1, "category": "building_easy_2", "screen": 2, "tile_x": 208684, "tile_y": 119948, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 106.5838623, 15.09864287 ], [ 106.5852356, 15.09864287 ], [ 106.5852356, 15.09731698 ], [ 106.5838623, 15.09731698 ], [ 106.5838623, 15.09864287 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-65041-120546", "reference": 0, "category": "building_easy_1", "screen": 1, "tile_x": 65041, "tile_y": 120546, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ -90.67977905, 14.30430808 ], [ -90.67840576, 14.30430808 ], [ -90.67840576, 14.30297736 ], [ -90.67977905, 14.30297736 ], [ -90.67977905, 14.30430808 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-65041-120547", "reference": 0, "category": "building_easy_1", "screen": 1, "tile_x": 65041, "tile_y": 120547, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ -90.67977905, 14.30297736 ], [ -90.67840576, 14.30297736 ], [ -90.67840576, 14.30164664 ], [ -90.67977905, 14.30164664 ], [ -90.67977905, 14.30297736 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-65040-120547", "reference": 0, "category": "building_easy_1", "screen": 1, "tile_x": 65040, "tile_y": 120547, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ -90.68115234, 14.30297736 ], [ -90.67977905, 14.30297736 ], [ -90.67977905, 14.30164664 ], [ -90.68115234, 14.30164664 ], [ -90.68115234, 14.30297736 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-65041-120545", "reference": 1, "category": "building_easy_1", "screen": 1, "tile_x": 65041, "tile_y": 120545, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ -90.67977905, 14.30563879 ], [ -90.67840576, 14.30563879 ], [ -90.67840576, 14.30430808 ], [ -90.67977905, 14.30430808 ], [ -90.67977905, 14.30563879 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-65040-120545", "reference": 0, "category": "building_easy_1", "screen": 1, "tile_x": 65040, "tile_y": 120545, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ -90.68115234, 14.30563879 ], [ -90.67977905, 14.30563879 ], [ -90.67977905, 14.30430808 ], [ -90.68115234, 14.30430808 ], [ -90.68115234, 14.30563879 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-65040-120546", "reference": 1, "category": "building_easy_1", "screen": 1, "tile_x": 65040, "tile_y": 120546, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ -90.68115234, 14.30430808 ], [ -90.67977905, 14.30430808 ], [ -90.67977905, 14.30297736 ], [ -90.68115234, 14.30297736 ], [ -90.68115234, 14.30430808 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-154989-139653", "reference": 0, "category": "no_building_easy_1", "screen": 4, "tile_x": 154989, "tile_y": 139653, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 32.84500122, -11.70199656 ], [ 32.84637451, -11.70199656 ], [ 32.84637451, -11.7033413 ], [ 32.84500122, -11.7033413 ], [ 32.84500122, -11.70199656 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-154988-139651", "reference": 0, "category": "no_building_easy_1", "screen": 4, "tile_x": 154988, "tile_y": 139651, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 32.84362793, -11.69930705 ], [ 32.84500122, -11.69930705 ], [ 32.84500122, -11.7006518 ], [ 32.84362793, -11.7006518 ], [ 32.84362793, -11.69930705 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-154989-139651", "reference": 0, "category": "no_building_easy_1", "screen": 4, "tile_x": 154989, "tile_y": 139651, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 32.84500122, -11.69930705 ], [ 32.84637451, -11.69930705 ], [ 32.84637451, -11.7006518 ], [ 32.84500122, -11.7006518 ], [ 32.84500122, -11.69930705 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-154989-139652", "reference": 0, "category": "no_building_easy_1", "screen": 4, "tile_x": 154989, "tile_y": 139652, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 32.84500122, -11.7006518 ], [ 32.84637451, -11.7006518 ], [ 32.84637451, -11.70199656 ], [ 32.84500122, -11.70199656 ], [ 32.84500122, -11.7006518 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-154988-139652", "reference": 0, "category": "no_building_easy_1", "screen": 4, "tile_x": 154988, "tile_y": 139652, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 32.84362793, -11.7006518 ], [ 32.84500122, -11.7006518 ], [ 32.84500122, -11.70199656 ], [ 32.84362793, -11.70199656 ], [ 32.84362793, -11.7006518 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-154988-139653", "reference": 0, "category": "no_building_easy_1", "screen": 4, "tile_x": 154988, "tile_y": 139653, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 32.84362793, -11.70199656 ], [ 32.84500122, -11.70199656 ], [ 32.84500122, -11.7033413 ], [ 32.84362793, -11.7033413 ], [ 32.84362793, -11.70199656 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-208633-119337", "reference": 0, "category": "no_building_easy_2", "screen": 5, "tile_x": 208633, "tile_y": 119337, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 106.51382446, 15.90718754 ], [ 106.51519775, 15.90718754 ], [ 106.51519775, 15.90586683 ], [ 106.51382446, 15.90586683 ], [ 106.51382446, 15.90718754 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-208632-119337", "reference": 0, "category": "no_building_easy_2", "screen": 5, "tile_x": 208632, "tile_y": 119337, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 106.51245117, 15.90718754 ], [ 106.51382446, 15.90718754 ], [ 106.51382446, 15.90586683 ], [ 106.51245117, 15.90586683 ], [ 106.51245117, 15.90718754 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-208632-119338", "reference": 0, "category": "no_building_easy_2", "screen": 5, "tile_x": 208632, "tile_y": 119338, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 106.51245117, 15.90586683 ], [ 106.51382446, 15.90586683 ], [ 106.51382446, 15.90454611 ], [ 106.51245117, 15.90454611 ], [ 106.51245117, 15.90586683 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-208633-119338", "reference": 0, "category": "no_building_easy_2", "screen": 5, "tile_x": 208633, "tile_y": 119338, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 106.51382446, 15.90586683 ], [ 106.51519775, 15.90586683 ], [ 106.51519775, 15.90454611 ], [ 106.51382446, 15.90454611 ], [ 106.51382446, 15.90586683 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-208632-119336", "reference": 0, "category": "no_building_easy_2", "screen": 5, "tile_x": 208632, "tile_y": 119336, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 106.51245117, 15.90850824 ], [ 106.51382446, 15.90850824 ], [ 106.51382446, 15.90718754 ], [ 106.51245117, 15.90718754 ], [ 106.51245117, 15.90850824 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-208633-119336", "reference": 0, "category": "no_building_easy_2", "screen": 5, "tile_x": 208633, "tile_y": 119336, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 106.51382446, 15.90850824 ], [ 106.51519775, 15.90850824 ], [ 106.51519775, 15.90718754 ], [ 106.51382446, 15.90718754 ], [ 106.51382446, 15.90850824 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-141659-121580", "reference": 0, "category": "maybe_1", "screen": 3, "tile_x": 141659, "tile_y": 121580, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 14.53903198, 12.92426063 ], [ 14.54040527, 12.92426063 ], [ 14.54040527, 12.92292213 ], [ 14.53903198, 12.92292213 ], [ 14.53903198, 12.92426063 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-141659-121581", "reference": 0, "category": "maybe_1", "screen": 3, "tile_x": 141659, "tile_y": 121581, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 14.53903198, 12.92292213 ], [ 14.54040527, 12.92292213 ], [ 14.54040527, 12.92158362 ], [ 14.53903198, 12.92158362 ], [ 14.53903198, 12.92292213 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-141659-121582", "reference": 2, "category": "maybe_1", "screen": 3, "tile_x": 141659, "tile_y": 121582, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 14.53903198, 12.92158362 ], [ 14.54040527, 12.92158362 ], [ 14.54040527, 12.9202451 ], [ 14.53903198, 12.9202451 ], [ 14.53903198, 12.92158362 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-141660-121580", "reference": 0, "category": "maybe_1", "screen": 3, "tile_x": 141660, "tile_y": 121580, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 14.54040527, 12.92426063 ], [ 14.54177856, 12.92426063 ], [ 14.54177856, 12.92292213 ], [ 14.54040527, 12.92292213 ], [ 14.54040527, 12.92426063 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-141660-121581", "reference": 0, "category": "maybe_1", "screen": 3, "tile_x": 141660, "tile_y": 121581, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 14.54040527, 12.92292213 ], [ 14.54177856, 12.92292213 ], [ 14.54177856, 12.92158362 ], [ 14.54040527, 12.92158362 ], [ 14.54040527, 12.92292213 ] ] ] ] } }, -{ "type": "Feature", "properties": { "taskid": "18-141660-121582", "reference": 0, "category": "maybe_1", "screen": 3, "tile_x": 141660, "tile_y": 121582, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 14.54040527, 12.92158362 ], [ 14.54177856, 12.92158362 ], [ 14.54177856, 12.9202451 ], [ 14.54040527, 12.9202451 ], [ 14.54040527, 12.92158362 ] ] ] ] } } +{ "type": "Feature", "properties": { "taskid": "18-208640-120212", "reference": 0, "category": "bad_no_imagery_1", "screen": 7, "tile_x": 208640, "tile_y": 120212, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 106.5234375, 14.74832321, 0.0 ], [ 106.52481079, 14.74832321, 0.0 ], [ 106.52481079, 14.74699516, 0.0 ], [ 106.5234375, 14.74699516, 0.0 ], [ 106.5234375, 14.74832321, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-208640-120213", "reference": 0, "category": "bad_no_imagery_1", "screen": 7, "tile_x": 208640, "tile_y": 120213, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 106.5234375, 14.74699516, 0.0 ], [ 106.52481079, 14.74699516, 0.0 ], [ 106.52481079, 14.74566711, 0.0 ], [ 106.5234375, 14.74566711, 0.0 ], [ 106.5234375, 14.74699516, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-208640-120214", "reference": 0, "category": "bad_no_imagery_1", "screen": 7, "tile_x": 208640, "tile_y": 120214, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 106.5234375, 14.74566711, 0.0 ], [ 106.52481079, 14.74566711, 0.0 ], [ 106.52481079, 14.74433904, 0.0 ], [ 106.5234375, 14.74433904, 0.0 ], [ 106.5234375, 14.74566711, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-208639-120213", "reference": 3, "category": "bad_no_imagery_1", "screen": 7, "tile_x": 208639, "tile_y": 120213, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 106.52206421, 14.74699516, 0.0 ], [ 106.5234375, 14.74699516, 0.0 ], [ 106.5234375, 14.74566711, 0.0 ], [ 106.52206421, 14.74566711, 0.0 ], [ 106.52206421, 14.74699516, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-208639-120214", "reference": 3, "category": "bad_no_imagery_1", "screen": 7, "tile_x": 208639, "tile_y": 120214, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 106.52206421, 14.74566711, 0.0 ], [ 106.5234375, 14.74566711, 0.0 ], [ 106.5234375, 14.74433904, 0.0 ], [ 106.52206421, 14.74433904, 0.0 ], [ 106.52206421, 14.74566711, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-208639-120212", "reference": 3, "category": "bad_no_imagery_1", "screen": 7, "tile_x": 208639, "tile_y": 120212, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 106.52206421, 14.74832321, 0.0 ], [ 106.5234375, 14.74832321, 0.0 ], [ 106.5234375, 14.74699516, 0.0 ], [ 106.52206421, 14.74699516, 0.0 ], [ 106.52206421, 14.74832321, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-157556-134648", "reference": 3, "category": "bad_no_imagery_2", "screen": 8, "tile_x": 157556, "tile_y": 134648, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 36.37023926, -4.90488679, 0.0 ], [ 36.37161255, -4.90488679, 0.0 ], [ 36.37161255, -4.90625506, 0.0 ], [ 36.37023926, -4.90625506, 0.0 ], [ 36.37023926, -4.90488679, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-157555-134649", "reference": 3, "category": "bad_no_imagery_2", "screen": 8, "tile_x": 157555, "tile_y": 134649, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 36.36886597, -4.90625506, 0.0 ], [ 36.37023926, -4.90625506, 0.0 ], [ 36.37023926, -4.90762331, 0.0 ], [ 36.36886597, -4.90762331, 0.0 ], [ 36.36886597, -4.90625506, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-157556-134649", "reference": 3, "category": "bad_no_imagery_2", "screen": 8, "tile_x": 157556, "tile_y": 134649, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 36.37023926, -4.90625506, 0.0 ], [ 36.37161255, -4.90625506, 0.0 ], [ 36.37161255, -4.90762331, 0.0 ], [ 36.37023926, -4.90762331, 0.0 ], [ 36.37023926, -4.90625506, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-157555-134647", "reference": 3, "category": "bad_no_imagery_2", "screen": 8, "tile_x": 157555, "tile_y": 134647, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 36.36886597, -4.90351853, 0.0 ], [ 36.37023926, -4.90351853, 0.0 ], [ 36.37023926, -4.90488679, 0.0 ], [ 36.36886597, -4.90488679, 0.0 ], [ 36.36886597, -4.90351853, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-157555-134648", "reference": 3, "category": "bad_no_imagery_2", "screen": 8, "tile_x": 157555, "tile_y": 134648, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 36.36886597, -4.90488679, 0.0 ], [ 36.37023926, -4.90488679, 0.0 ], [ 36.37023926, -4.90625506, 0.0 ], [ 36.36886597, -4.90625506, 0.0 ], [ 36.36886597, -4.90488679, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-157556-134647", "reference": 3, "category": "bad_no_imagery_2", "screen": 8, "tile_x": 157556, "tile_y": 134647, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 36.37023926, -4.90351853, 0.0 ], [ 36.37161255, -4.90351853, 0.0 ], [ 36.37161255, -4.90488679, 0.0 ], [ 36.37023926, -4.90488679, 0.0 ], [ 36.37023926, -4.90351853, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-208684-119949", "reference": 1, "category": "building_easy_2", "screen": 2, "tile_x": 208684, "tile_y": 119949, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 106.5838623, 15.09731698, 0.0 ], [ 106.5852356, 15.09731698, 0.0 ], [ 106.5852356, 15.09599108, 0.0 ], [ 106.5838623, 15.09599108, 0.0 ], [ 106.5838623, 15.09731698, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-208683-119949", "reference": 1, "category": "building_easy_2", "screen": 2, "tile_x": 208683, "tile_y": 119949, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 106.58248901, 15.09731698, 0.0 ], [ 106.5838623, 15.09731698, 0.0 ], [ 106.5838623, 15.09599108, 0.0 ], [ 106.58248901, 15.09599108, 0.0 ], [ 106.58248901, 15.09731698, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-208683-119950", "reference": 1, "category": "building_easy_2", "screen": 2, "tile_x": 208683, "tile_y": 119950, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 106.58248901, 15.09599108, 0.0 ], [ 106.5838623, 15.09599108, 0.0 ], [ 106.5838623, 15.09466518, 0.0 ], [ 106.58248901, 15.09466518, 0.0 ], [ 106.58248901, 15.09599108, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-208684-119950", "reference": 0, "category": "building_easy_2", "screen": 2, "tile_x": 208684, "tile_y": 119950, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 106.5838623, 15.09599108, 0.0 ], [ 106.5852356, 15.09599108, 0.0 ], [ 106.5852356, 15.09466518, 0.0 ], [ 106.5838623, 15.09466518, 0.0 ], [ 106.5838623, 15.09599108, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-208683-119948", "reference": 0, "category": "building_easy_2", "screen": 2, "tile_x": 208683, "tile_y": 119948, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 106.58248901, 15.09864287, 0.0 ], [ 106.5838623, 15.09864287, 0.0 ], [ 106.5838623, 15.09731698, 0.0 ], [ 106.58248901, 15.09731698, 0.0 ], [ 106.58248901, 15.09864287, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-208684-119948", "reference": 1, "category": "building_easy_2", "screen": 2, "tile_x": 208684, "tile_y": 119948, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 106.5838623, 15.09864287, 0.0 ], [ 106.5852356, 15.09864287, 0.0 ], [ 106.5852356, 15.09731698, 0.0 ], [ 106.5838623, 15.09731698, 0.0 ], [ 106.5838623, 15.09864287, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-65041-120546", "reference": 0, "category": "building_easy_1", "screen": 1, "tile_x": 65041, "tile_y": 120546, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ -90.67977905, 14.30430808, 0.0 ], [ -90.67840576, 14.30430808, 0.0 ], [ -90.67840576, 14.30297736, 0.0 ], [ -90.67977905, 14.30297736, 0.0 ], [ -90.67977905, 14.30430808, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-65041-120547", "reference": 0, "category": "building_easy_1", "screen": 1, "tile_x": 65041, "tile_y": 120547, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ -90.67977905, 14.30297736, 0.0 ], [ -90.67840576, 14.30297736, 0.0 ], [ -90.67840576, 14.30164664, 0.0 ], [ -90.67977905, 14.30164664, 0.0 ], [ -90.67977905, 14.30297736, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-65040-120547", "reference": 0, "category": "building_easy_1", "screen": 1, "tile_x": 65040, "tile_y": 120547, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ -90.68115234, 14.30297736, 0.0 ], [ -90.67977905, 14.30297736, 0.0 ], [ -90.67977905, 14.30164664, 0.0 ], [ -90.68115234, 14.30164664, 0.0 ], [ -90.68115234, 14.30297736, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-65041-120545", "reference": 1, "category": "building_easy_1", "screen": 1, "tile_x": 65041, "tile_y": 120545, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ -90.67977905, 14.30563879, 0.0 ], [ -90.67840576, 14.30563879, 0.0 ], [ -90.67840576, 14.30430808, 0.0 ], [ -90.67977905, 14.30430808, 0.0 ], [ -90.67977905, 14.30563879, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-65040-120545", "reference": 0, "category": "building_easy_1", "screen": 1, "tile_x": 65040, "tile_y": 120545, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ -90.68115234, 14.30563879, 0.0 ], [ -90.67977905, 14.30563879, 0.0 ], [ -90.67977905, 14.30430808, 0.0 ], [ -90.68115234, 14.30430808, 0.0 ], [ -90.68115234, 14.30563879, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-65040-120546", "reference": 1, "category": "building_easy_1", "screen": 1, "tile_x": 65040, "tile_y": 120546, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ -90.68115234, 14.30430808, 0.0 ], [ -90.67977905, 14.30430808, 0.0 ], [ -90.67977905, 14.30297736, 0.0 ], [ -90.68115234, 14.30297736, 0.0 ], [ -90.68115234, 14.30430808, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-154989-139653", "reference": 0, "category": "no_building_easy_1", "screen": 4, "tile_x": 154989, "tile_y": 139653, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 32.84500122, -11.70199656, 0.0 ], [ 32.84637451, -11.70199656, 0.0 ], [ 32.84637451, -11.7033413, 0.0 ], [ 32.84500122, -11.7033413, 0.0 ], [ 32.84500122, -11.70199656, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-154988-139651", "reference": 0, "category": "no_building_easy_1", "screen": 4, "tile_x": 154988, "tile_y": 139651, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 32.84362793, -11.69930705, 0.0 ], [ 32.84500122, -11.69930705, 0.0 ], [ 32.84500122, -11.7006518, 0.0 ], [ 32.84362793, -11.7006518, 0.0 ], [ 32.84362793, -11.69930705, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-154989-139651", "reference": 0, "category": "no_building_easy_1", "screen": 4, "tile_x": 154989, "tile_y": 139651, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 32.84500122, -11.69930705, 0.0 ], [ 32.84637451, -11.69930705, 0.0 ], [ 32.84637451, -11.7006518, 0.0 ], [ 32.84500122, -11.7006518, 0.0 ], [ 32.84500122, -11.69930705, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-154989-139652", "reference": 0, "category": "no_building_easy_1", "screen": 4, "tile_x": 154989, "tile_y": 139652, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 32.84500122, -11.7006518, 0.0 ], [ 32.84637451, -11.7006518, 0.0 ], [ 32.84637451, -11.70199656, 0.0 ], [ 32.84500122, -11.70199656, 0.0 ], [ 32.84500122, -11.7006518, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-154988-139652", "reference": 0, "category": "no_building_easy_1", "screen": 4, "tile_x": 154988, "tile_y": 139652, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 32.84362793, -11.7006518, 0.0 ], [ 32.84500122, -11.7006518, 0.0 ], [ 32.84500122, -11.70199656, 0.0 ], [ 32.84362793, -11.70199656, 0.0 ], [ 32.84362793, -11.7006518, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-154988-139653", "reference": 0, "category": "no_building_easy_1", "screen": 4, "tile_x": 154988, "tile_y": 139653, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 32.84362793, -11.70199656, 0.0 ], [ 32.84500122, -11.70199656, 0.0 ], [ 32.84500122, -11.7033413, 0.0 ], [ 32.84362793, -11.7033413, 0.0 ], [ 32.84362793, -11.70199656, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-208633-119337", "reference": 0, "category": "no_building_easy_2", "screen": 5, "tile_x": 208633, "tile_y": 119337, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 106.51382446, 15.90718754, 0.0 ], [ 106.51519775, 15.90718754, 0.0 ], [ 106.51519775, 15.90586683, 0.0 ], [ 106.51382446, 15.90586683, 0.0 ], [ 106.51382446, 15.90718754, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-208632-119337", "reference": 0, "category": "no_building_easy_2", "screen": 5, "tile_x": 208632, "tile_y": 119337, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 106.51245117, 15.90718754, 0.0 ], [ 106.51382446, 15.90718754, 0.0 ], [ 106.51382446, 15.90586683, 0.0 ], [ 106.51245117, 15.90586683, 0.0 ], [ 106.51245117, 15.90718754, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-208632-119338", "reference": 0, "category": "no_building_easy_2", "screen": 5, "tile_x": 208632, "tile_y": 119338, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 106.51245117, 15.90586683, 0.0 ], [ 106.51382446, 15.90586683, 0.0 ], [ 106.51382446, 15.90454611, 0.0 ], [ 106.51245117, 15.90454611, 0.0 ], [ 106.51245117, 15.90586683, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-208633-119338", "reference": 0, "category": "no_building_easy_2", "screen": 5, "tile_x": 208633, "tile_y": 119338, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 106.51382446, 15.90586683, 0.0 ], [ 106.51519775, 15.90586683, 0.0 ], [ 106.51519775, 15.90454611, 0.0 ], [ 106.51382446, 15.90454611, 0.0 ], [ 106.51382446, 15.90586683, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-208632-119336", "reference": 0, "category": "no_building_easy_2", "screen": 5, "tile_x": 208632, "tile_y": 119336, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 106.51245117, 15.90850824, 0.0 ], [ 106.51382446, 15.90850824, 0.0 ], [ 106.51382446, 15.90718754, 0.0 ], [ 106.51245117, 15.90718754, 0.0 ], [ 106.51245117, 15.90850824, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-208633-119336", "reference": 0, "category": "no_building_easy_2", "screen": 5, "tile_x": 208633, "tile_y": 119336, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 106.51382446, 15.90850824, 0.0 ], [ 106.51519775, 15.90850824, 0.0 ], [ 106.51519775, 15.90718754, 0.0 ], [ 106.51382446, 15.90718754, 0.0 ], [ 106.51382446, 15.90850824, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-141659-121580", "reference": 0, "category": "maybe_1", "screen": 3, "tile_x": 141659, "tile_y": 121580, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 14.53903198, 12.92426063, 0.0 ], [ 14.54040527, 12.92426063, 0.0 ], [ 14.54040527, 12.92292213, 0.0 ], [ 14.53903198, 12.92292213, 0.0 ], [ 14.53903198, 12.92426063, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-141659-121581", "reference": 0, "category": "maybe_1", "screen": 3, "tile_x": 141659, "tile_y": 121581, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 14.53903198, 12.92292213, 0.0 ], [ 14.54040527, 12.92292213, 0.0 ], [ 14.54040527, 12.92158362, 0.0 ], [ 14.53903198, 12.92158362, 0.0 ], [ 14.53903198, 12.92292213, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-141659-121582", "reference": 2, "category": "maybe_1", "screen": 3, "tile_x": 141659, "tile_y": 121582, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 14.53903198, 12.92158362, 0.0 ], [ 14.54040527, 12.92158362, 0.0 ], [ 14.54040527, 12.9202451, 0.0 ], [ 14.53903198, 12.9202451, 0.0 ], [ 14.53903198, 12.92158362, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-141660-121580", "reference": 0, "category": "maybe_1", "screen": 3, "tile_x": 141660, "tile_y": 121580, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 14.54040527, 12.92426063, 0.0 ], [ 14.54177856, 12.92426063, 0.0 ], [ 14.54177856, 12.92292213, 0.0 ], [ 14.54040527, 12.92292213, 0.0 ], [ 14.54040527, 12.92426063, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-141660-121581", "reference": 0, "category": "maybe_1", "screen": 3, "tile_x": 141660, "tile_y": 121581, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 14.54040527, 12.92292213, 0.0 ], [ 14.54177856, 12.92292213, 0.0 ], [ 14.54177856, 12.92158362, 0.0 ], [ 14.54040527, 12.92158362, 0.0 ], [ 14.54040527, 12.92292213, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-141660-121582", "reference": 0, "category": "maybe_1", "screen": 3, "tile_x": 141660, "tile_y": 121582, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 14.54040527, 12.92158362, 0.0 ], [ 14.54177856, 12.92158362, 0.0 ], [ 14.54177856, 12.9202451, 0.0 ], [ 14.54040527, 12.9202451, 0.0 ], [ 14.54040527, 12.92158362, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-65471-120428", "reference": 0, "category": "bad_clouds_1", "screen": 6, "tile_x": 65471, "tile_y": 120428, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ -90.089263916015597, 14.461277417004201, 0.0 ], [ -90.087890625, 14.461277417004201, 0.0 ], [ -90.087890625, 14.4599476324892, 0.0 ], [ -90.089263916015597, 14.4599476324892, 0.0 ], [ -90.089263916015597, 14.461277417004201, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-65472-120428", "reference": 3, "category": "bad_clouds_1", "screen": 6, "tile_x": 65472, "tile_y": 120428, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ -90.087890625, 14.461277417004201, 0.0 ], [ -90.086517333984403, 14.461277417004201, 0.0 ], [ -90.086517333984403, 14.4599476324892, 0.0 ], [ -90.087890625, 14.4599476324892, 0.0 ], [ -90.087890625, 14.461277417004201, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-65471-120429", "reference": 0, "category": "bad_clouds_1", "screen": 6, "tile_x": 65471, "tile_y": 120429, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ -90.089263916015597, 14.4599476324892, 0.0 ], [ -90.087890625, 14.4599476324892, 0.0 ], [ -90.087890625, 14.4586178400154, 0.0 ], [ -90.089263916015597, 14.4586178400154, 0.0 ], [ -90.089263916015597, 14.4599476324892, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-65471-120430", "reference": 0, "category": "bad_clouds_1", "screen": 6, "tile_x": 65471, "tile_y": 120430, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ -90.089263916015597, 14.4586178400154, 0.0 ], [ -90.087890625, 14.4586178400154, 0.0 ], [ -90.087890625, 14.4572880395834, 0.0 ], [ -90.089263916015597, 14.4572880395834, 0.0 ], [ -90.089263916015597, 14.4586178400154, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-65472-120429", "reference": 3, "category": "bad_clouds_1", "screen": 6, "tile_x": 65472, "tile_y": 120429, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ -90.087890625, 14.4599476324892, 0.0 ], [ -90.086517333984403, 14.4599476324892, 0.0 ], [ -90.086517333984403, 14.4586178400154, 0.0 ], [ -90.087890625, 14.4586178400154, 0.0 ], [ -90.087890625, 14.4599476324892, 0.0 ] ] ] ] } }, +{ "type": "Feature", "properties": { "taskid": "18-65472-120430", "reference": 3, "category": "bad_clouds_1", "screen": 6, "tile_x": 65472, "tile_y": 120430, "tile_z": 18 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ -90.087890625, 14.4586178400154, 0.0 ], [ -90.086517333984403, 14.4586178400154, 0.0 ], [ -90.086517333984403, 14.4572880395834, 0.0 ], [ -90.087890625, 14.4572880395834, 0.0 ], [ -90.087890625, 14.4586178400154, 0.0 ] ] ] ] } } ] } From 66f273443b215821bf64df0c6fc250f56e61c740 Mon Sep 17 00:00:00 2001 From: Jochen Date: Tue, 9 Nov 2021 20:31:20 +0100 Subject: [PATCH 03/62] typo --- mapswipe_workers/mapswipe_workers/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapswipe_workers/mapswipe_workers/auth.py b/mapswipe_workers/mapswipe_workers/auth.py index f3e57d552..06d94492f 100644 --- a/mapswipe_workers/mapswipe_workers/auth.py +++ b/mapswipe_workers/mapswipe_workers/auth.py @@ -45,7 +45,7 @@ def firebaseDB() -> object: class postgresDB(object): - """Helper calss for Postgres interactions""" + """Helper class for Postgres interactions""" _db_connection = None _db_cur = None From bf643f713fd5d5c27d573bc07478de747e0275b6 Mon Sep 17 00:00:00 2001 From: Jochen Date: Wed, 10 Nov 2021 12:49:43 +0100 Subject: [PATCH 04/62] typo --- api/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/index.html b/api/index.html index 6fac0e244..1d22e77dc 100644 --- a/api/index.html +++ b/api/index.html @@ -8,7 +8,7 @@

Welcome to Mapswipe API

Read the Documentation -

Directory Tree

+

Directory Tree

stats.json
projects.json
users.json
From 02701f12702483faa961fca5211d5f46270647c7 Mon Sep 17 00:00:00 2001 From: Jochen Date: Wed, 10 Nov 2021 16:10:36 +0100 Subject: [PATCH 05/62] standartized formating --- .../manager_dashboard/create.html | 568 +++++++++--------- 1 file changed, 280 insertions(+), 288 deletions(-) diff --git a/manager_dashboard/manager_dashboard/create.html b/manager_dashboard/manager_dashboard/create.html index 5c07d7b02..2c1654dbd 100644 --- a/manager_dashboard/manager_dashboard/create.html +++ b/manager_dashboard/manager_dashboard/create.html @@ -3,50 +3,50 @@ - - Project Management | MapSwipe - - - - - - + + Project Management | MapSwipe + + + + + + - - - + + + - - - + + + - - - + + + - - + + - - - + + + - - + + - - - - + + + + - - - + @@ -91,257 +91,249 @@

Please log in first.

diff --git a/manager_dashboard/manager_dashboard/css/mapswipe.css b/manager_dashboard/manager_dashboard/css/mapswipe.css index ca7371e71..74849ca5f 100644 --- a/manager_dashboard/manager_dashboard/css/mapswipe.css +++ b/manager_dashboard/manager_dashboard/css/mapswipe.css @@ -64,7 +64,6 @@ padding: 2px 5px 2px 5px; color: #B9B9B9; font-size: 20px; - overflow: hidden; font-family: Arial, Helvetica, sans-serif; } .form-style-7 input[type="text"], diff --git a/manager_dashboard/manager_dashboard/js/forms.js b/manager_dashboard/manager_dashboard/js/forms.js index 245a6816c..021cd31c6 100644 --- a/manager_dashboard/manager_dashboard/js/forms.js +++ b/manager_dashboard/manager_dashboard/js/forms.js @@ -163,7 +163,7 @@ function clear_fields() { console.log('clear fields.') document.getElementById('projectNumber').value = 1 document.getElementById('inputAoi').value = null - document.getElementById('geometryInfo').innerHTML = '' + $(".geometryInfo").each(()=>{$(this).text('')}) document.getElementById('geometryContent').innerHTML = '' aoiLayer.clearLayers() displayProjectTypeForm("build_area") @@ -180,9 +180,19 @@ function displayImportForm() { function openFile(event) { var input = event.target; + let maxFilesize, maxFeatures; + + if(input.id=="inputAoi"){ + maxFilesize = 1 * 1024 * 1024 + maxFeatures = 10 + } + else if (input.id=="inputTaskGeometriesFile"){ + maxFilesize = 10 * 1024 * 1024 + maxFeatures = 5000 + } // clear info field - var info_output = document.getElementById("geometryInfo"); + var info_output = $(input).siblings(".geometryInfo")[0]; info_output.innerHTML = ''; info_output.style.display = 'block' @@ -191,8 +201,8 @@ function openFile(event) { // Check file size before loading var filesize = input.files[0].size; - if (filesize > 1 * 1024 * 1024) { - var err='filesize is too big (max 1MB): ' + filesize/(1000*1000) + if (filesize > maxFilesize) { + var err=`filesize is too big (max ${maxFilesize}MB): ${filesize/(1024*1024)}` info_output.innerHTML = 'Error reading GeoJSON file
' + err; info_output.style.display = 'block' } else { @@ -209,7 +219,7 @@ function openFile(event) { numberOfFeatures = geojsonData['features'].length console.log('number of features: ' + numberOfFeatures) - if (numberOfFeatures > 10) { + if (numberOfFeatures > maxFeatures) { throw 'too many features: ' + numberOfFeatures } info_output.innerHTML += 'Number of Features: ' + numberOfFeatures + '
'; @@ -225,9 +235,10 @@ function openFile(event) { if (type !== 'Polygon' & type !== 'MultiPolygon') { throw 'GeoJson contains one or more wrong geometry type(s): ' + type } - - info_output.innerHTML += 'Feature Type: ' + type + '
'; - info_output.style.display = 'block' + if (input.id=="inputAoi"){ + info_output.innerHTML += 'Feature Type: ' + type + '
'; + info_output.style.display = 'block' + } sumArea += turf.area(feature)/1000000 // area in square kilometers } @@ -293,3 +304,28 @@ function closeModal() { var modalSuccess = document.getElementById("modalSuccess"); modalSuccess.style.display = "none"; } + + +function show_input(select){ + let link_div = $("#inputTaskGeometries_Link"); + let file_div = $("#inputTaskGeometries_File"); + let id_div = $("#inputTaskGeometries_ProjectId"); + + switch(select.value){ + case "link": + link_div.css("display", "block"); + file_div.css("display", "none"); + id_div.css("display", "none"); + break; + case "file": + link_div.css("display", "none"); + file_div.css("display", "block"); + id_div.css("display", "none"); + break; + case "id": + link_div.css("display", "none"); + file_div.css("display", "none"); + id_div.css("display", "block"); + break; + } +} \ No newline at end of file diff --git a/manager_dashboard/manager_dashboard/js/uploadProjects.js b/manager_dashboard/manager_dashboard/js/uploadProjects.js index 90d05de46..3dded9f6c 100644 --- a/manager_dashboard/manager_dashboard/js/uploadProjects.js +++ b/manager_dashboard/manager_dashboard/js/uploadProjects.js @@ -1,6 +1,13 @@ var database = firebase.database(); +async function fetchProjectId(project_id){ + const response = await fetch(`https://tasking-manager-tm4-production-api.hotosm.org/api/v2/projects/${project_id}/?as_file=false&abbreviated=false`); + const answer = await response.json(); + return answer; + } + + function getFormInput() { var form_data = { projectRegion: document.getElementById("projectRegion").value, @@ -15,6 +22,7 @@ function getFormInput() { createdBy: currentUid, tutorialId: document.getElementById("tutorial").value } + form_data.name = form_data.projectTopic + ' - ' + form_data.projectRegion + ' (' + form_data.projectNumber + ')\n' + @@ -46,7 +54,19 @@ function getFormInput() { break; case "footprint": form_data.projectType = 2; - form_data.inputGeometries = document.getElementById("inputTaskGeometries").value; + switch($("#geometryInputOption").val()){ + case "link": + form_data.inputGeometries = $("#inputTaskGeometries").val(); + break; + case "file": + form_data.inputGeometries = "file"; + form_data.geometry = JSON.parse(projectAoiGeometry) + break; + case "id": + form_data.inputGeometries = $("#inputTaskGeometriesId").val(); + form_data.is_projectId = true + break; + } form_data.tileServer = { name: document.getElementById("tileServerAName").value, url: document.getElementById("tileServerAUrl").value, @@ -177,7 +197,16 @@ function upload_to_firebase() { else { mapswipe_import = getFormInput() // upload projectDraft to firebase once image has been uploaded - upload_project_image(mapswipe_import) + // if inputGeometries is a projectId, check if it is a valid project, only upload if it is + if (mapswipe_import.is_projectId == null){ + upload_project_image(mapswipe_import) + } + else { + fetchProjectId(mapswipe_import.inputGeometries).then(answer=>{ + answer.Error != null ? alert(`Invalid ProjectId: ${answer.Error}`) : upload_project_image(mapswipe_import)}) + } } + + } } diff --git a/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/project.py b/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/project.py index f70d634e1..645d9ee7b 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/project.py +++ b/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/project.py @@ -1,3 +1,4 @@ +import json import os import urllib.request @@ -16,6 +17,7 @@ def __init__(self, project_draft: dict) -> None: # set group size self.groupSize = project_draft["groupSize"] + self.geometry = project_draft["geometry"] self.inputGeometries = project_draft["inputGeometries"] self.tileServer = vars(BaseTileServer(project_draft["tileServer"])) @@ -30,9 +32,17 @@ def validate_geometries(self): if not os.path.isdir("{}/input_geometries".format(DATA_PATH)): os.mkdir("{}/input_geometries".format(DATA_PATH)) - # download file from given url - url = self.inputGeometries - urllib.request.urlretrieve(url, raw_input_file) + + # self.inputGeometries can be "file", an url or an integer (project_id) + if self.inputGeometries == "file": + # write string to geom file + with open(raw_input_file, "w") as geom_file: + json.dump(self.geometry, geom_file) + else: + try: + project_id = int(self.inputGeometries) + except: + urllib.request.urlretrieve(self.inputGeometries, raw_input_file) logger.info( f"{self.projectId}" f" - __init__ - " diff --git a/mapswipe_workers/mapswipe_workers/project_types/base/project.py b/mapswipe_workers/mapswipe_workers/project_types/base/project.py index 220efaa37..5a847918f 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/base/project.py +++ b/mapswipe_workers/mapswipe_workers/project_types/base/project.py @@ -107,7 +107,7 @@ def save_project(self): """ logger.info(f"{self.projectId}" f" - start creating a project") - # Convert object attributes to dictonaries + # Convert object attributes to dictionaries # for saving it to firebase and postgres project = vars(self) groups = dict() From 198acc3041bc1206609071c3ae4c8ab0ca0fedaf Mon Sep 17 00:00:00 2001 From: Jochen Date: Thu, 25 Nov 2021 23:56:38 +0100 Subject: [PATCH 07/62] second batch --- .../manager_dashboard/create.html | 49 +++++-- .../manager_dashboard/css/mapswipe.css | 27 +++- .../manager_dashboard/js/forms.js | 104 +++++++------- .../manager_dashboard/js/upload-tutorial.js | 13 +- .../manager_dashboard/js/uploadProjects.js | 79 +++++++---- .../manager_dashboard/js/utils.js | 134 ++++++++++++++++++ .../manager_dashboard/tutorial.html | 2 +- .../mapswipe_workers/definitions.py | 1 + .../mapswipe_workers/mapswipe_workers.py | 1 + .../arbitrary_geometry/grouping_functions.py | 6 + .../arbitrary_geometry/project.py | 66 +++++++-- .../project_types/base/project.py | 2 + .../mapswipe_workers/utils/api_calls.py | 37 +++++ .../mapswipe_workers/utils/slack_helper.py | 2 +- 14 files changed, 410 insertions(+), 113 deletions(-) create mode 100644 manager_dashboard/manager_dashboard/js/utils.js create mode 100644 mapswipe_workers/mapswipe_workers/utils/api_calls.py diff --git a/manager_dashboard/manager_dashboard/create.html b/manager_dashboard/manager_dashboard/create.html index bbd224051..3fcdcaecd 100644 --- a/manager_dashboard/manager_dashboard/create.html +++ b/manager_dashboard/manager_dashboard/create.html @@ -93,13 +93,17 @@

Please log in first.

From 592539835903f885cc43bfb4d1335bb33d513138 Mon Sep 17 00:00:00 2001 From: Jochen Date: Fri, 26 Nov 2021 21:48:12 +0100 Subject: [PATCH 14/62] changed test --- .../footprint/projectDrafts/footprint.json | 17 ------- .../projectDrafts/footprint_TMId.json | 20 +++++++++ .../projectDrafts/footprint_aoiFile.json | 25 +++++++++++ .../projectDrafts/footprint_link.json | 16 +++++++ mapswipe_workers/tests/integration/set_up.py | 10 ++--- .../test_create_footprint_project.py | 44 ++++++++++--------- 6 files changed, 90 insertions(+), 42 deletions(-) delete mode 100644 mapswipe_workers/tests/integration/fixtures/footprint/projectDrafts/footprint.json create mode 100644 mapswipe_workers/tests/integration/fixtures/footprint/projectDrafts/footprint_TMId.json create mode 100644 mapswipe_workers/tests/integration/fixtures/footprint/projectDrafts/footprint_aoiFile.json create mode 100644 mapswipe_workers/tests/integration/fixtures/footprint/projectDrafts/footprint_link.json diff --git a/mapswipe_workers/tests/integration/fixtures/footprint/projectDrafts/footprint.json b/mapswipe_workers/tests/integration/fixtures/footprint/projectDrafts/footprint.json deleted file mode 100644 index c40f0d2f2..000000000 --- a/mapswipe_workers/tests/integration/fixtures/footprint/projectDrafts/footprint.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "createdBy": "Sample Admin", - "inputGeometries": "https://firebasestorage.googleapis.com/v0/b/dev-mapswipe.appspot.com/o/buildings_heidelberg.geojson?alt=media&token=1631f7d4-5fb2-4e1b-a133-2e14582d116c", - "image": "http://www.fragosus.com/test/Javita.jpg", - "lookFor": "Buildings", - "name": "Footprint Sample Project", - "projectDetails": "This is a template for a building by building project. We use Bing as the tile server.", - "verificationNumber": 3, - "groupSize": 35, - "projectType": 2, - "tileServer": { - "name": "bing", - "url": "", - "apiKey": "", - "credits": "© 2019 Microsoft Corporation, Earthstar Geographics SIO" - } -} \ No newline at end of file diff --git a/mapswipe_workers/tests/integration/fixtures/footprint/projectDrafts/footprint_TMId.json b/mapswipe_workers/tests/integration/fixtures/footprint/projectDrafts/footprint_TMId.json new file mode 100644 index 000000000..1463273fe --- /dev/null +++ b/mapswipe_workers/tests/integration/fixtures/footprint/projectDrafts/footprint_TMId.json @@ -0,0 +1,20 @@ +{ + "TMId" : "122", + "createdBy" : "Sample Admin", + "filter" : "building=* and geometry:polygon", + "geometry" : { + "coordinates" : [ [ [ [ 29.1465238959428, -0.982870599863187 ], [ 29.2766207959247, -0.979139499863717 ], [ 29.2761807959247, -1.25351369982555 ], [ 29.1295774959451, -1.25402109982547 ], [ 29.1286471959453, -1.04515049985454 ], [ 28.9781401959662, -1.03921789985535 ], [ 28.9778474959663, -0.915405199872588 ], [ 29.1459010959429, -0.914885499872663 ], [ 29.1465238959428, -0.982870599863187 ] ] ] ], + "type" : "MultiPolygon" + }, + "groupSize" : 25, + "inputType" : "TMId", + "name" : "Test Footprint TMId", + "projectTopic" : "Test Footprint TMId", + "projectType" : 2, + "tileServer" : { + "credits" : "© 2019 Microsoft Corporation, Earthstar Geographics SIO", + "name" : "bing", + "url" : "", + "wmtsLayerName" : "" + } +} \ No newline at end of file diff --git a/mapswipe_workers/tests/integration/fixtures/footprint/projectDrafts/footprint_aoiFile.json b/mapswipe_workers/tests/integration/fixtures/footprint/projectDrafts/footprint_aoiFile.json new file mode 100644 index 000000000..27f43a7ef --- /dev/null +++ b/mapswipe_workers/tests/integration/fixtures/footprint/projectDrafts/footprint_aoiFile.json @@ -0,0 +1,25 @@ +{ + "createdBy" : "Sample Admin", + "filter" : "building=* and geometry:polygon", + "geometry" : { + "features" : [ { + "geometry" : { + "coordinates" : [ [ [ 9.14886474609375, 48.788092642076286 ], [ 9.129638671875, 48.77316143513268 ], [ 9.171180725097656, 48.76614675936893 ], [ 9.193840026855469, 48.77451900112444 ], [ 9.192466735839844, 48.7874140472094 ], [ 9.14886474609375, 48.788092642076286 ] ] ], + "type" : "Polygon" + }, + "type" : "Feature" + } ], + "type" : "FeatureCollection" + }, + "groupSize" : 25, + "inputType" : "aoi_file", + "name" : "Test Footprint GeoJSON AOI", + "projectTopic" : "Test Footprint GeoJSON AOI", + "projectType" : 2, + "tileServer" : { + "credits" : "© 2019 Microsoft Corporation, Earthstar Geographics SIO", + "name" : "bing", + "url" : "", + "wmtsLayerName" : "" + } +} diff --git a/mapswipe_workers/tests/integration/fixtures/footprint/projectDrafts/footprint_link.json b/mapswipe_workers/tests/integration/fixtures/footprint/projectDrafts/footprint_link.json new file mode 100644 index 000000000..4b1b71738 --- /dev/null +++ b/mapswipe_workers/tests/integration/fixtures/footprint/projectDrafts/footprint_link.json @@ -0,0 +1,16 @@ +{ + "createdBy" : "Sample Admin", + "geometry" : "https://firebasestorage.googleapis.com/v0/b/dev-mapswipe.appspot.com/o/buildings_heidelberg.geojson?alt=media&token=1631f7d4-5fb2-4e1b-a133-2e14582d116c", + "groupSize" : 25, + "inputType" : "link", + "name" : "Test Footprint Link", + "projectDetails": "This is a template for a building by building project. We use Bing as the tile server.", + "projectTopic" : "Test Footprint Link", + "projectType" : 2, + "tileServer" : { + "credits" : "© 2019 Microsoft Corporation, Earthstar Geographics SIO", + "name" : "bing", + "url" : "", + "wmtsLayerName" : "" + } +} diff --git a/mapswipe_workers/tests/integration/set_up.py b/mapswipe_workers/tests/integration/set_up.py index 168237907..9e8d2dceb 100644 --- a/mapswipe_workers/tests/integration/set_up.py +++ b/mapswipe_workers/tests/integration/set_up.py @@ -82,16 +82,16 @@ def create_test_user(project_type: str, user_id: str = None) -> str: def create_test_project_draft( - project_type: str, fixture_name: str = "user", user_id: str = "" + project_type: str, fixture_name: str = "user", identifier: str = "" ) -> str: """ Create test project drafts in Firebase and return project ids. Project drafts in Firebase are create by project manager using the dashboard. """ - if not user_id: - user_id = f"test_{project_type}" - set_firebase_test_data(project_type, "projectDrafts", fixture_name, user_id) - return user_id + if not identifier: + identifier = f"test_{fixture_name}" + set_firebase_test_data(project_type, "projectDrafts", fixture_name, identifier) + return identifier if __name__ == "__main__": diff --git a/mapswipe_workers/tests/integration/test_create_footprint_project.py b/mapswipe_workers/tests/integration/test_create_footprint_project.py index 9b7f4d900..8d417dc52 100644 --- a/mapswipe_workers/tests/integration/test_create_footprint_project.py +++ b/mapswipe_workers/tests/integration/test_create_footprint_project.py @@ -9,7 +9,11 @@ class TestCreateProject(unittest.TestCase): def setUp(self): - self.project_id = set_up.create_test_project_draft("footprint", "footprint") + self.project_id = [ + set_up.create_test_project_draft("footprint", "footprint_aoiFile"), + set_up.create_test_project_draft("footprint", "footprint_link"), + set_up.create_test_project_draft("footprint", "footprint_TMId"), + ] create_directories() def tearDown(self): @@ -17,25 +21,25 @@ def tearDown(self): def test_create_footprint_project(self): mapswipe_workers.run_create_projects() - - pg_db = auth.postgresDB() - query = "SELECT project_id FROM projects WHERE project_id = %s" - result = pg_db.retr_query(query, [self.project_id])[0][0] - self.assertEqual(result, self.project_id) - - fb_db = auth.firebaseDB() - ref = fb_db.reference(f"/v2/projects/{self.project_id}") - result = ref.get(shallow=True) - self.assertIsNotNone(result) - - ref = fb_db.reference(f"/v2/groups/{self.project_id}") - result = ref.get(shallow=True) - self.assertIsNotNone(result) - - # Footprint projects have tasks in Firebase - ref = fb_db.reference(f"/v2/tasks/{self.project_id}") - result = ref.get(shallow=True) - self.assertIsNotNone(result) + for element in self.project_id: + pg_db = auth.postgresDB() + query = "SELECT project_id FROM projects WHERE project_id = %s" + result = pg_db.retr_query(query, [element])[0][0] + self.assertEqual(result, element) + + fb_db = auth.firebaseDB() + ref = fb_db.reference(f"/v2/projects/{element}") + result = ref.get(shallow=True) + self.assertIsNotNone(result) + + ref = fb_db.reference(f"/v2/groups/{element}") + result = ref.get(shallow=True) + self.assertIsNotNone(result) + + # Footprint projects have tasks in Firebase + ref = fb_db.reference(f"/v2/tasks/{element}") + result = ref.get(shallow=True) + self.assertIsNotNone(result) if __name__ == "__main__": From 32209e55c00e4b1308714228fa716f333c6f1e20 Mon Sep 17 00:00:00 2001 From: Jochen Date: Fri, 26 Nov 2021 22:07:45 +0100 Subject: [PATCH 15/62] now it should work --- .../tests/integration/test_create_footprint_project.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mapswipe_workers/tests/integration/test_create_footprint_project.py b/mapswipe_workers/tests/integration/test_create_footprint_project.py index 8d417dc52..524f7933c 100644 --- a/mapswipe_workers/tests/integration/test_create_footprint_project.py +++ b/mapswipe_workers/tests/integration/test_create_footprint_project.py @@ -17,7 +17,8 @@ def setUp(self): create_directories() def tearDown(self): - tear_down.delete_test_data(self.project_id) + for element in self.project_id: + tear_down.delete_test_data(element) def test_create_footprint_project(self): mapswipe_workers.run_create_projects() From b59cec295d6489c3f6825b4f033164cc306e3a16 Mon Sep 17 00:00:00 2001 From: Jochen Date: Fri, 26 Nov 2021 22:44:09 +0100 Subject: [PATCH 16/62] update test_files --- .../fixtures/footprint/projectDrafts/footprint_TMId.json | 4 ++++ .../fixtures/footprint/projectDrafts/footprint_aoiFile.json | 4 ++++ .../fixtures/footprint/projectDrafts/footprint_link.json | 5 ++++- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/mapswipe_workers/tests/integration/fixtures/footprint/projectDrafts/footprint_TMId.json b/mapswipe_workers/tests/integration/fixtures/footprint/projectDrafts/footprint_TMId.json index 1463273fe..1d9ee1247 100644 --- a/mapswipe_workers/tests/integration/fixtures/footprint/projectDrafts/footprint_TMId.json +++ b/mapswipe_workers/tests/integration/fixtures/footprint/projectDrafts/footprint_TMId.json @@ -7,10 +7,14 @@ "type" : "MultiPolygon" }, "groupSize" : 25, + "lookFor": "Buildings", + "image": "http://www.fragosus.com/test/Javita.jpg", + "projectDetails": "This is a template for a Footprint TMId project. We use Bing as the tile server.", "inputType" : "TMId", "name" : "Test Footprint TMId", "projectTopic" : "Test Footprint TMId", "projectType" : 2, + "verificationNumber": 3, "tileServer" : { "credits" : "© 2019 Microsoft Corporation, Earthstar Geographics SIO", "name" : "bing", diff --git a/mapswipe_workers/tests/integration/fixtures/footprint/projectDrafts/footprint_aoiFile.json b/mapswipe_workers/tests/integration/fixtures/footprint/projectDrafts/footprint_aoiFile.json index 27f43a7ef..07ba96d64 100644 --- a/mapswipe_workers/tests/integration/fixtures/footprint/projectDrafts/footprint_aoiFile.json +++ b/mapswipe_workers/tests/integration/fixtures/footprint/projectDrafts/footprint_aoiFile.json @@ -12,10 +12,14 @@ "type" : "FeatureCollection" }, "groupSize" : 25, + "lookFor": "Buildings", + "image": "http://www.fragosus.com/test/Javita.jpg", + "projectDetails": "This is a template for a GeoJSON AOI project. We use Bing as the tile server.", "inputType" : "aoi_file", "name" : "Test Footprint GeoJSON AOI", "projectTopic" : "Test Footprint GeoJSON AOI", "projectType" : 2, + "verificationNumber": 3, "tileServer" : { "credits" : "© 2019 Microsoft Corporation, Earthstar Geographics SIO", "name" : "bing", diff --git a/mapswipe_workers/tests/integration/fixtures/footprint/projectDrafts/footprint_link.json b/mapswipe_workers/tests/integration/fixtures/footprint/projectDrafts/footprint_link.json index 4b1b71738..6a8220bc7 100644 --- a/mapswipe_workers/tests/integration/fixtures/footprint/projectDrafts/footprint_link.json +++ b/mapswipe_workers/tests/integration/fixtures/footprint/projectDrafts/footprint_link.json @@ -2,11 +2,14 @@ "createdBy" : "Sample Admin", "geometry" : "https://firebasestorage.googleapis.com/v0/b/dev-mapswipe.appspot.com/o/buildings_heidelberg.geojson?alt=media&token=1631f7d4-5fb2-4e1b-a133-2e14582d116c", "groupSize" : 25, + "lookFor": "Buildings", + "image": "http://www.fragosus.com/test/Javita.jpg", "inputType" : "link", "name" : "Test Footprint Link", - "projectDetails": "This is a template for a building by building project. We use Bing as the tile server.", + "projectDetails": "This is a template for a Footprint Link project. We use Bing as the tile server.", "projectTopic" : "Test Footprint Link", "projectType" : 2, + "verificationNumber": 3, "tileServer" : { "credits" : "© 2019 Microsoft Corporation, Earthstar Geographics SIO", "name" : "bing", From dcdc437ce542d04bec527a371db9c897824c6b9a Mon Sep 17 00:00:00 2001 From: Jochen Date: Fri, 26 Nov 2021 23:08:41 +0100 Subject: [PATCH 17/62] adjusted log --- .../project_types/arbitrary_geometry/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/project.py b/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/project.py index 71fcb5ef8..607d1f84f 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/project.py +++ b/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/project.py @@ -42,7 +42,7 @@ def handle_input_type(self, raw_input_file: str): self.geometry = json.dumps(self.geometry) if self.inputType == "aoi_file": - logger.info("file detected") + logger.info("aoi file detected") # write string to geom file ohsome_request = {"endpoint": "elements/geometry", "filter": self.filter} From 0e6c4776d37c08df43e96dd32a5fef81ecc485ca Mon Sep 17 00:00:00 2001 From: Jochen Date: Fri, 26 Nov 2021 23:58:55 +0100 Subject: [PATCH 18/62] try --- .../tests/integration/test_create_footprint_project.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/mapswipe_workers/tests/integration/test_create_footprint_project.py b/mapswipe_workers/tests/integration/test_create_footprint_project.py index 524f7933c..360f07e17 100644 --- a/mapswipe_workers/tests/integration/test_create_footprint_project.py +++ b/mapswipe_workers/tests/integration/test_create_footprint_project.py @@ -4,6 +4,7 @@ import tear_down from mapswipe_workers import auth, mapswipe_workers +from mapswipe_workers.definitions import logger from mapswipe_workers.utils.create_directories import create_directories @@ -21,9 +22,12 @@ def tearDown(self): tear_down.delete_test_data(element) def test_create_footprint_project(self): - mapswipe_workers.run_create_projects() + try: + mapswipe_workers.run_create_projects() + except BaseException as e: + logger.error(e, exc_info=True) + pg_db = auth.postgresDB() for element in self.project_id: - pg_db = auth.postgresDB() query = "SELECT project_id FROM projects WHERE project_id = %s" result = pg_db.retr_query(query, [element])[0][0] self.assertEqual(result, element) From 827cec821fbc0780a7a183338ef70b26be27dc8d Mon Sep 17 00:00:00 2001 From: Jochen Date: Sat, 27 Nov 2021 00:10:28 +0100 Subject: [PATCH 19/62] try --- .../tests/integration/test_create_footprint_project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapswipe_workers/tests/integration/test_create_footprint_project.py b/mapswipe_workers/tests/integration/test_create_footprint_project.py index 360f07e17..b716c1011 100644 --- a/mapswipe_workers/tests/integration/test_create_footprint_project.py +++ b/mapswipe_workers/tests/integration/test_create_footprint_project.py @@ -31,7 +31,7 @@ def test_create_footprint_project(self): query = "SELECT project_id FROM projects WHERE project_id = %s" result = pg_db.retr_query(query, [element])[0][0] self.assertEqual(result, element) - + # testchange fb_db = auth.firebaseDB() ref = fb_db.reference(f"/v2/projects/{element}") result = ref.get(shallow=True) From 37c60980ab8b86c425c5df2745a561531966ed20 Mon Sep 17 00:00:00 2001 From: Jochen Date: Sat, 27 Nov 2021 00:20:50 +0100 Subject: [PATCH 20/62] revert test_changes --- .../tests/integration/test_create_footprint_project.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/mapswipe_workers/tests/integration/test_create_footprint_project.py b/mapswipe_workers/tests/integration/test_create_footprint_project.py index b716c1011..5770f99d0 100644 --- a/mapswipe_workers/tests/integration/test_create_footprint_project.py +++ b/mapswipe_workers/tests/integration/test_create_footprint_project.py @@ -4,7 +4,6 @@ import tear_down from mapswipe_workers import auth, mapswipe_workers -from mapswipe_workers.definitions import logger from mapswipe_workers.utils.create_directories import create_directories @@ -22,16 +21,12 @@ def tearDown(self): tear_down.delete_test_data(element) def test_create_footprint_project(self): - try: - mapswipe_workers.run_create_projects() - except BaseException as e: - logger.error(e, exc_info=True) + mapswipe_workers.run_create_projects() pg_db = auth.postgresDB() for element in self.project_id: query = "SELECT project_id FROM projects WHERE project_id = %s" result = pg_db.retr_query(query, [element])[0][0] self.assertEqual(result, element) - # testchange fb_db = auth.firebaseDB() ref = fb_db.reference(f"/v2/projects/{element}") result = ref.get(shallow=True) From d730ce1a348e15a61360ce8324ec7f6d4a3cf6c0 Mon Sep 17 00:00:00 2001 From: Hagellach37 Date: Mon, 29 Nov 2021 13:20:37 +0100 Subject: [PATCH 21/62] properly call run_create_projects function using CliRunner --- .../tests/integration/test_create_footprint_project.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mapswipe_workers/tests/integration/test_create_footprint_project.py b/mapswipe_workers/tests/integration/test_create_footprint_project.py index 5770f99d0..9652a9494 100644 --- a/mapswipe_workers/tests/integration/test_create_footprint_project.py +++ b/mapswipe_workers/tests/integration/test_create_footprint_project.py @@ -2,6 +2,7 @@ import set_up import tear_down +from click.testing import CliRunner from mapswipe_workers import auth, mapswipe_workers from mapswipe_workers.utils.create_directories import create_directories @@ -21,7 +22,9 @@ def tearDown(self): tear_down.delete_test_data(element) def test_create_footprint_project(self): - mapswipe_workers.run_create_projects() + runner = CliRunner() + runner.invoke(mapswipe_workers.run_create_projects) + pg_db = auth.postgresDB() for element in self.project_id: query = "SELECT project_id FROM projects WHERE project_id = %s" From 06213e0058f4df964a4b66bf1691d8e2fd172202 Mon Sep 17 00:00:00 2001 From: Hagellach37 Date: Mon, 29 Nov 2021 13:27:19 +0100 Subject: [PATCH 22/62] small fix in importing sub module --- mapswipe_workers/mapswipe_workers/utils/api_calls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/api_calls.py b/mapswipe_workers/mapswipe_workers/utils/api_calls.py index 2a5cec5d6..85ab3237e 100644 --- a/mapswipe_workers/mapswipe_workers/utils/api_calls.py +++ b/mapswipe_workers/mapswipe_workers/utils/api_calls.py @@ -1,6 +1,6 @@ import requests -from ..definitions import OHSOME_API_LINK, CustomError, logger +from mapswipe_workers.definitions import OHSOME_API_LINK, CustomError, logger def geojsonToFeatureCollection(geojson): From fc0ffad5155db3008f6f7a6c1923aa7af2af992f Mon Sep 17 00:00:00 2001 From: Jochen Stier <44268882+ElJocho@users.noreply.github.com> Date: Fri, 3 Dec 2021 15:16:47 +0100 Subject: [PATCH 23/62] bugfix check if type is footprint before trying to access project specific variable --- manager_dashboard/manager_dashboard/js/uploadProjects.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/manager_dashboard/manager_dashboard/js/uploadProjects.js b/manager_dashboard/manager_dashboard/js/uploadProjects.js index f7b6fe3b7..dd6ee043d 100644 --- a/manager_dashboard/manager_dashboard/js/uploadProjects.js +++ b/manager_dashboard/manager_dashboard/js/uploadProjects.js @@ -197,7 +197,12 @@ async function uploadToFirebase() { // get form data mapswipe_import = getFormInput() // upload projectDraft to firebase once image has been uploaded - // if inputGeometries is a TMId, check if it is a valid project, only upload if it is + + if (document.getElementById("projectType").value != "footprint"){ + uploadProjectImage(mapswipe_import); + return; + } + var modal = document.getElementById("uploadModal"); modal.style.display = "block"; var modalValidating = document.getElementById("modalValidating"); From 1529ffdd6db07c91a47b2c6e4a9c410d4b888ffe Mon Sep 17 00:00:00 2001 From: Jochen Date: Mon, 6 Dec 2021 16:07:32 +0100 Subject: [PATCH 24/62] changebundle --- .../mapswipe_workers/definitions.py | 2 + .../project_types/arbitrary_geometry/group.py | 18 +--- .../arbitrary_geometry/grouping_functions.py | 52 +++-------- .../arbitrary_geometry/project.py | 12 +-- .../project_types/arbitrary_geometry/task.py | 35 +++----- .../project_types/base/project.py | 18 +++- .../project_types/base/task.py | 2 +- .../mapswipe_workers/utils/api_calls.py | 87 +++++++++++++++++-- .../test_create_footprint_project.py | 7 +- 9 files changed, 130 insertions(+), 103 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/definitions.py b/mapswipe_workers/mapswipe_workers/definitions.py index 5a3fbb9d9..62daebdd5 100644 --- a/mapswipe_workers/mapswipe_workers/definitions.py +++ b/mapswipe_workers/mapswipe_workers/definitions.py @@ -13,6 +13,8 @@ LOGGING_FILE_PATH = os.path.join(DATA_PATH, "mapswipe_workers.log") OHSOME_API_LINK = "https://api.ohsome.org/v1/" +OSM_API_LINK = "https://www.openstreetmap.org/api/0.6/" + # number of geometries for project geometries MAX_INPUT_GEOMETRIES = 10 diff --git a/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/group.py b/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/group.py index 7d2f1347e..9438932fd 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/group.py +++ b/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/group.py @@ -8,14 +8,7 @@ class Group(BaseGroup): def __init__(self, project: object, groupId: int) -> None: super().__init__(project, groupId) - def create_tasks( - self, - feature_ids: List, - feature_geometries: List, - center_points: List, - reference: List, - screen: List, - ) -> None: + def create_tasks(self, feature_ids: List, feature: List,) -> None: """Create tasks for a group feature_geometries is a list of geometries or feature in geojson format. @@ -24,13 +17,6 @@ def create_tasks( Every coordinate pair is a vertex. """ for i in range(0, len(feature_ids)): - task = Task( - self, - feature_ids[i], - feature_geometries[i], - center_points[i], - reference[i], - screen[i], - ) + task = Task(self, feature_ids[i], feature[i],) self.tasks.append(task) self.numberOfTasks = len(self.tasks) diff --git a/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/grouping_functions.py b/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/grouping_functions.py index a1425de65..97e408606 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/grouping_functions.py +++ b/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/grouping_functions.py @@ -1,8 +1,6 @@ import argparse import json -from osgeo import ogr - ######################################################################################## parser = argparse.ArgumentParser(description="Process some integers.") parser.add_argument( @@ -43,17 +41,15 @@ def group_input_geometries(input_geometries_file, group_size): "feature_geometries" per group with given group id key """ - driver = ogr.GetDriverByName("GeoJSON") - datasource = driver.Open(input_geometries_file, 0) - layer = datasource.GetLayer() - + with open(input_geometries_file, "r") as infile: + layer = json.load(infile) groups = {} - # we will simply, we will start with min group id = 100 + # we will simply start with min group id = 100 group_id = 100 group_id_string = f"g{group_id}" feature_count = 0 - for feature in layer: + for feature in layer["features"]: feature_count += 1 # feature count starts at 1 # assuming group size would be 10 @@ -65,50 +61,22 @@ def group_input_geometries(input_geometries_file, group_size): try: groups[group_id_string] except KeyError: - # todo: change this and call feature_geoms -> geojson - groups[group_id_string] = { - "feature_ids": [], - "feature_geometries": [], - "center_points": [], - "reference": [], - "screen": [], - } + groups[group_id_string] = {"feature_ids": [], "feature": []} # we use a new id here based on the count # since we are not sure that GetFID returns unique values groups[group_id_string]["feature_ids"].append(feature_count) - groups[group_id_string]["feature_geometries"].append( - json.loads(feature.GetGeometryRef().ExportToJson()) - ) - - # from gdal documentation GetFieldAsDouble(): - # OFTInteger fields will be cast to double. - # Other field types, or errors will result in a return value of zero. - center_x = feature.GetFieldAsDouble("center_x") - center_y = feature.GetFieldAsDouble("center_y") - - # check if center attribute has been provided in geojson - # normal tasks will never have a center of 0.0, 0.0 - # this is just in the middle of the ocean - if (center_x == 0.0) and (center_y == 0.0): - groups[group_id_string]["center_points"].append(None) - else: - groups[group_id_string]["center_points"].append([center_x, center_y]) # this is relevant for the tutorial - reference = feature.GetFieldAsDouble("reference") - screen = feature.GetFieldAsDouble("screen") - groups[group_id_string]["reference"].append(reference) - groups[group_id_string]["screen"].append(screen) + if "screen" in feature.keys(): + feature["properties"]["reference"] = int(feature["reference"]) + feature["properties"]["screen"] = int(feature["screen"]) + + groups[group_id_string]["feature"].append(feature) return groups -# todo: groups should have this output formatting -# reference is correct answer for tutorial, -# if not tutorial reference = None -# {"g100":[{"task_id":1, "screen": 1, "reference":0, -# "geojson":{"valid geojson FeatureCollection Object"}}]} ######################################################################################## if __name__ == "__main__": diff --git a/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/project.py b/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/project.py index 607d1f84f..65212b2a3 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/project.py +++ b/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/project.py @@ -46,14 +46,14 @@ def handle_input_type(self, raw_input_file: str): # write string to geom file ohsome_request = {"endpoint": "elements/geometry", "filter": self.filter} - result = ohsome(ohsome_request, self.geometry) + result = ohsome(ohsome_request, self.geometry, properties="tags, metadata") with open(raw_input_file, "w") as geom_file: json.dump(result, geom_file) elif self.inputType == "TMId": logger.info("TMId detected") hot_tm_project_id = int(self.TMId) ohsome_request = {"endpoint": "elements/geometry", "filter": self.filter} - result = ohsome(ohsome_request, self.geometry) + result = ohsome(ohsome_request, self.geometry, properties="tags, metadata") result["properties"] = {} result["properties"]["hot_tm_project_id"] = hot_tm_project_id with open(raw_input_file, "w") as geom_file: @@ -194,13 +194,7 @@ def create_groups(self): for group_id, item in raw_groups.items(): group = Group(self, group_id) - group.create_tasks( - item["feature_ids"], - item["feature_geometries"], - item["center_points"], - item["reference"], - item["screen"], - ) + group.create_tasks(item["feature_ids"], item["feature"]) # only append valid groups if group.is_valid(): diff --git a/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/task.py b/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/task.py index 136209921..b1867310f 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/task.py +++ b/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/task.py @@ -1,4 +1,4 @@ -from typing import Dict, List, Optional, Union +from typing import Dict, Union from osgeo import ogr @@ -7,35 +7,22 @@ class Task(BaseTask): def __init__( - self, - group: object, - featureId: Union[int, str], - featureGeometry: Dict, - center: Optional[List[float]], - reference: Optional[int], - screen: Optional[int], + self, group: object, featureId: Union[int, str], feature: Dict, ): """ Parameters ---------- - feature_geometry: dict - The geometries or feature in geojson format. - It consist of two keys: Coordinates and type. - Coordinates of four two pair coordinates. - Every coordinate pair is a vertex. + feature: dict + a Feature in geojson format. """ task_id = f"t{featureId}" super().__init__(group, taskId=task_id) - self.geojson = featureGeometry - - # only tasks that use Google tile map service need this - if center: - self.center = center - + self.geojson = feature["geometry"] + self.properties = feature["properties"] # only tasks that are part of a tutorial need this - if screen: - self.screen = screen - self.reference = reference + if "screen" in feature["properties"].keys(): + self.screen = feature["properties"]["screen"] + self.reference = feature["properties"]["screen"] # Remove projectId and groupId for tasks of Footprint project type del self.projectId @@ -43,7 +30,7 @@ def __init__( # create wkt geometry from geojson # this geometry will be stored in postgres - # it will be remove before storing the data in firebase - poly = ogr.CreateGeometryFromJson(str(featureGeometry)) + # it will be removed before storing the data in firebase + poly = ogr.CreateGeometryFromJson(str(self.geojson)) wkt_geometry = poly.ExportToWkt() self.geometry = wkt_geometry diff --git a/mapswipe_workers/mapswipe_workers/project_types/base/project.py b/mapswipe_workers/mapswipe_workers/project_types/base/project.py index 2b56a90f8..a9e5c7eac 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/base/project.py +++ b/mapswipe_workers/mapswipe_workers/project_types/base/project.py @@ -121,8 +121,13 @@ def save_project(self): del group["tasks"] groups[group["groupId"]] = group del project["groups"] + project.pop("inputGeometries", None) project.pop("validInputGeometries", None) + project.pop("inputType", None) + project.pop("filter", None) + project.pop("TMId", None) + # Convert Date object to ISO Datetime: # https://www.w3.org/TR/NOTE-datetime project["created"] = self.created.strftime("%Y-%m-%dT%H:%M:%S.%fZ") @@ -230,7 +235,8 @@ def save_to_firebase(self, project, groups, groupsOfTasks): # since the tasks hold geometries their storage size # can get quite big otherwise if self.projectType in [ProjectType.FOOTPRINT.value]: - # todo: remove extra information from geojson + # removing properties from each task + tasks_list = [task.pop("properties", None) for task in tasks_list] compressed_tasks = gzip_str.compress_tasks(tasks_list) task_upload_dict[ @@ -587,14 +593,20 @@ def create_tasks_txt_file(self, groupsOfTasks): # these common attributes don't need to be written # to the project_type_specifics since they are # already stored in separate columns - common_attributes = ["projectId", "groupId", "taskId", "geometry"] + common_attributes = [ + "projectId", + "groupId", + "taskId", + "geometry", + "geojson", + ] for key in task.keys(): if key not in common_attributes: output_dict["project_type_specifics"][key] = task[key] output_dict["project_type_specifics"] = json.dumps( output_dict["project_type_specifics"] - ) + ).replace("'", "") w.writerow(output_dict) tasks_txt_file.close() diff --git a/mapswipe_workers/mapswipe_workers/project_types/base/task.py b/mapswipe_workers/mapswipe_workers/project_types/base/task.py index 5fe4eb6d6..1585869d3 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/base/task.py +++ b/mapswipe_workers/mapswipe_workers/project_types/base/task.py @@ -10,7 +10,7 @@ class BaseTask(object): def __init__(self, group, taskId): """ - The Constructor Method for a task instance + The Constructor Method for a task instance Parameters ---------- diff --git a/mapswipe_workers/mapswipe_workers/utils/api_calls.py b/mapswipe_workers/mapswipe_workers/utils/api_calls.py index 85ab3237e..db7c4c3b0 100644 --- a/mapswipe_workers/mapswipe_workers/utils/api_calls.py +++ b/mapswipe_workers/mapswipe_workers/utils/api_calls.py @@ -1,9 +1,14 @@ import requests -from mapswipe_workers.definitions import OHSOME_API_LINK, CustomError, logger +from mapswipe_workers.definitions import ( + OHSOME_API_LINK, + OSM_API_LINK, + CustomError, + logger, +) -def geojsonToFeatureCollection(geojson): +def geojsonToFeatureCollection(geojson: dict) -> dict: if geojson["type"] != "FeatureCollection": collection = { "type": "FeatureCollection", @@ -13,7 +18,74 @@ def geojsonToFeatureCollection(geojson): return geojson -def ohsome(request: dict, area: str, properties=None): +def query_osm(changeset_id: int): + """Get data from changesetId.""" + url = OSM_API_LINK + f"changeset/{changeset_id}.json" + valid_response = False + times_timed_out = 0 + while not valid_response: + try: + response = requests.get(url, timeout=3) + valid_response = True + except requests.exceptions.Timeout: + if times_timed_out > 4: + raise requests.exceptions.Timeout("timed out 5 times") + logger.warn(url) + logger.warn("connection timed out") + times_timed_out += 1 + + if response.status_code != 200: + err = f"osm request failed: {response.status_code}" + logger.warning(f"{err}") + logger.warning(response.json()) + raise CustomError(err) + response = response.json()["elements"][0] + return response + + +def remove_noise_and_add_user_info(json: dict) -> dict: + """Delete unwanted information from properties.""" + logger.info("starting filtering and adding extra info") + + wanted_rows = ["@changesetId", "@lastEdit", "@osmId", "@version", "source"] + changeset_results = {} + missing_rows = { + "@changesetId": 0, + "@lastEdit": 0, + "@osmId": 0, + "@version": 0, + "source": 0, + "comment": 0, + } + + for feature in json["features"]: + new_properties = {} + for attribute in wanted_rows: + try: + if attribute != "comment": + new_properties[attribute.replace("@", "")] = feature["properties"][ + attribute + ] + else: + new_properties[attribute.replace("@", "")] = feature["properties"][ + "tags" + ][attribute] + except KeyError: + missing_rows[attribute] += 1 + changeset_id = new_properties["changesetId"] + if changeset_id not in changeset_results.keys(): + changeset_results[changeset_id] = query_osm(changeset_id) + new_properties["username"] = changeset_results[changeset_id]["user"] + new_properties["userid"] = changeset_results[changeset_id]["uid"] + feature["properties"] = new_properties + logger.info("finished filtering and adding extra info") + if any(x > 0 for x in missing_rows.values()): + logger.warning(f"features missing values:\n{missing_rows}") + + return json + + +def ohsome(request: dict, area: str, properties=None) -> dict: """ Request data from Ohsome API. """ @@ -25,7 +97,7 @@ def ohsome(request: dict, area: str, properties=None): logger.info("Filter: " + request["filter"]) response = requests.post(url, data=data) if response.status_code != 200: - err = f"ohsome Request failed: {response.status_code}" + err = f"ohsome request failed: {response.status_code}" logger.warning( f"{err} - check for errors in filter or geometries - {request['filter']}" ) @@ -33,4 +105,9 @@ def ohsome(request: dict, area: str, properties=None): raise CustomError(err) else: logger.info("Query succesfull.") - return response.json() + + response = response.json() + + if properties: + remove_noise_and_add_user_info(response) + return response diff --git a/mapswipe_workers/tests/integration/test_create_footprint_project.py b/mapswipe_workers/tests/integration/test_create_footprint_project.py index 9652a9494..6bb4590fc 100644 --- a/mapswipe_workers/tests/integration/test_create_footprint_project.py +++ b/mapswipe_workers/tests/integration/test_create_footprint_project.py @@ -11,9 +11,9 @@ class TestCreateProject(unittest.TestCase): def setUp(self): self.project_id = [ + set_up.create_test_project_draft("footprint", "footprint_TMId"), set_up.create_test_project_draft("footprint", "footprint_aoiFile"), set_up.create_test_project_draft("footprint", "footprint_link"), - set_up.create_test_project_draft("footprint", "footprint_TMId"), ] create_directories() @@ -23,8 +23,9 @@ def tearDown(self): def test_create_footprint_project(self): runner = CliRunner() - runner.invoke(mapswipe_workers.run_create_projects) - + result = runner.invoke(mapswipe_workers.run_create_projects) + if result.exit_code != 0: + raise result.exception pg_db = auth.postgresDB() for element in self.project_id: query = "SELECT project_id FROM projects WHERE project_id = %s" From f2b331733b83a083a26b9183b544191b9f2334ff Mon Sep 17 00:00:00 2001 From: Jochen Date: Mon, 6 Dec 2021 16:30:01 +0100 Subject: [PATCH 25/62] smaller aoi_file --- .../projectDrafts/footprint_aoiFile.json | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/mapswipe_workers/tests/integration/fixtures/footprint/projectDrafts/footprint_aoiFile.json b/mapswipe_workers/tests/integration/fixtures/footprint/projectDrafts/footprint_aoiFile.json index 07ba96d64..07bdd001f 100644 --- a/mapswipe_workers/tests/integration/fixtures/footprint/projectDrafts/footprint_aoiFile.json +++ b/mapswipe_workers/tests/integration/fixtures/footprint/projectDrafts/footprint_aoiFile.json @@ -2,14 +2,16 @@ "createdBy" : "Sample Admin", "filter" : "building=* and geometry:polygon", "geometry" : { - "features" : [ { - "geometry" : { - "coordinates" : [ [ [ 9.14886474609375, 48.788092642076286 ], [ 9.129638671875, 48.77316143513268 ], [ 9.171180725097656, 48.76614675936893 ], [ 9.193840026855469, 48.77451900112444 ], [ 9.192466735839844, 48.7874140472094 ], [ 9.14886474609375, 48.788092642076286 ] ] ], - "type" : "Polygon" - }, - "type" : "Feature" - } ], - "type" : "FeatureCollection" + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": {}, + "geometry": { + "type": "Polygon", + "coordinates": [[[9.18032169342041, 48.790552471542284],[9.187102317810059,48.790552471542284],[9.187102317810059,48.79407236257656],[9.18032169342041,48.79407236257656],[9.18032169342041,48.790552471542284]]]} + } + ] }, "groupSize" : 25, "lookFor": "Buildings", From 04fad6b8e0fe35def3c9c243f287f1cbc223f325 Mon Sep 17 00:00:00 2001 From: Jochen Date: Mon, 6 Dec 2021 17:18:56 +0100 Subject: [PATCH 26/62] added feedback --- .../arbitrary_geometry/grouping_functions.py | 39 +------------- .../project_types/arbitrary_geometry/task.py | 2 +- .../project_types/base/project.py | 7 ++- .../mapswipe_workers/utils/api_calls.py | 51 ++++++++++--------- 4 files changed, 35 insertions(+), 64 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/grouping_functions.py b/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/grouping_functions.py index 97e408606..739222640 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/grouping_functions.py +++ b/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/grouping_functions.py @@ -1,26 +1,5 @@ -import argparse import json -######################################################################################## -parser = argparse.ArgumentParser(description="Process some integers.") -parser.add_argument( - "-i", - "--input_file", - required=False, - default=None, - type=str, - help="the input file containning the geometries as geojson", -) -parser.add_argument( - "-g", - "--group_size", - required=False, - default=50, - type=int, - help="the size of each group", -) -######################################################################################## - def group_input_geometries(input_geometries_file, group_size): """ @@ -66,22 +45,6 @@ def group_input_geometries(input_geometries_file, group_size): # we use a new id here based on the count # since we are not sure that GetFID returns unique values groups[group_id_string]["feature_ids"].append(feature_count) - - # this is relevant for the tutorial - if "screen" in feature.keys(): - feature["properties"]["reference"] = int(feature["reference"]) - feature["properties"]["screen"] = int(feature["screen"]) - - groups[group_id_string]["feature"].append(feature) + groups[group_id_string]["features"].append(feature) return groups - - -######################################################################################## -if __name__ == "__main__": - - args = parser.parse_args() - - groups = group_input_geometries(args.input_file, args.group_size) - - print(groups) diff --git a/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/task.py b/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/task.py index b1867310f..193589863 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/task.py +++ b/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/task.py @@ -22,7 +22,7 @@ def __init__( # only tasks that are part of a tutorial need this if "screen" in feature["properties"].keys(): self.screen = feature["properties"]["screen"] - self.reference = feature["properties"]["screen"] + self.reference = feature["properties"]["reference"] # Remove projectId and groupId for tasks of Footprint project type del self.projectId diff --git a/mapswipe_workers/mapswipe_workers/project_types/base/project.py b/mapswipe_workers/mapswipe_workers/project_types/base/project.py index a9e5c7eac..f718665fd 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/base/project.py +++ b/mapswipe_workers/mapswipe_workers/project_types/base/project.py @@ -236,7 +236,8 @@ def save_to_firebase(self, project, groups, groupsOfTasks): # can get quite big otherwise if self.projectType in [ProjectType.FOOTPRINT.value]: # removing properties from each task - tasks_list = [task.pop("properties", None) for task in tasks_list] + for task in tasks_list: + task.pop("properties", None) compressed_tasks = gzip_str.compress_tasks(tasks_list) task_upload_dict[ @@ -606,7 +607,9 @@ def create_tasks_txt_file(self, groupsOfTasks): output_dict["project_type_specifics"][key] = task[key] output_dict["project_type_specifics"] = json.dumps( output_dict["project_type_specifics"] - ).replace("'", "") + ).replace( + "'", "" + ) # to prevent error: invalid token "'" w.writerow(output_dict) tasks_txt_file.close() diff --git a/mapswipe_workers/mapswipe_workers/utils/api_calls.py b/mapswipe_workers/mapswipe_workers/utils/api_calls.py index db7c4c3b0..dd348d870 100644 --- a/mapswipe_workers/mapswipe_workers/utils/api_calls.py +++ b/mapswipe_workers/mapswipe_workers/utils/api_calls.py @@ -1,4 +1,6 @@ import requests +from requests.adapters import HTTPAdapter +from requests.packages.urllib3.util.retry import Retry from mapswipe_workers.definitions import ( OHSOME_API_LINK, @@ -8,6 +10,13 @@ ) +def retry_get(url, retries=3, timeout=4): + retry = Retry(total=retries) + session = requests.Session() + session.mount("https://", HTTPAdapter(max_retries=retry)) + return session.get(url, timeout=timeout) + + def geojsonToFeatureCollection(geojson: dict) -> dict: if geojson["type"] != "FeatureCollection": collection = { @@ -21,18 +30,8 @@ def geojsonToFeatureCollection(geojson: dict) -> dict: def query_osm(changeset_id: int): """Get data from changesetId.""" url = OSM_API_LINK + f"changeset/{changeset_id}.json" - valid_response = False - times_timed_out = 0 - while not valid_response: - try: - response = requests.get(url, timeout=3) - valid_response = True - except requests.exceptions.Timeout: - if times_timed_out > 4: - raise requests.exceptions.Timeout("timed out 5 times") - logger.warn(url) - logger.warn("connection timed out") - times_timed_out += 1 + + response = retry_get(url) if response.status_code != 200: err = f"osm request failed: {response.status_code}" @@ -43,11 +42,22 @@ def query_osm(changeset_id: int): return response +def add_to_properties(attribute: str, feature: dict, new_properties: dict): + """Adds attribute to new geojson properties if it is needed.""" + if attribute != "comment": + new_properties[attribute.replace("@", "")] = feature["properties"][attribute] + else: + new_properties[attribute.replace("@", "")] = feature["properties"]["tags"][ + attribute + ] + return new_properties + + def remove_noise_and_add_user_info(json: dict) -> dict: """Delete unwanted information from properties.""" logger.info("starting filtering and adding extra info") - wanted_rows = ["@changesetId", "@lastEdit", "@osmId", "@version", "source"] + wanted_attributes = ["@changesetId", "@lastEdit", "@osmId", "@version", "source"] changeset_results = {} missing_rows = { "@changesetId": 0, @@ -60,19 +70,14 @@ def remove_noise_and_add_user_info(json: dict) -> dict: for feature in json["features"]: new_properties = {} - for attribute in wanted_rows: + for attribute in wanted_attributes: try: - if attribute != "comment": - new_properties[attribute.replace("@", "")] = feature["properties"][ - attribute - ] - else: - new_properties[attribute.replace("@", "")] = feature["properties"][ - "tags" - ][attribute] + new_properties = add_to_properties(attribute, feature, new_properties) except KeyError: missing_rows[attribute] += 1 changeset_id = new_properties["changesetId"] + + # if changeset_id already queried, use stored result if changeset_id not in changeset_results.keys(): changeset_results[changeset_id] = query_osm(changeset_id) new_properties["username"] = changeset_results[changeset_id]["user"] @@ -109,5 +114,5 @@ def ohsome(request: dict, area: str, properties=None) -> dict: response = response.json() if properties: - remove_noise_and_add_user_info(response) + response = remove_noise_and_add_user_info(response) return response From 3dfe5e2513945a96dd6609d410a7bf5aded19613 Mon Sep 17 00:00:00 2001 From: Jochen Date: Mon, 6 Dec 2021 17:30:27 +0100 Subject: [PATCH 27/62] enumertare --- .../project_types/arbitrary_geometry/group.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/group.py b/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/group.py index 9438932fd..c11a55614 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/group.py +++ b/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/group.py @@ -8,7 +8,7 @@ class Group(BaseGroup): def __init__(self, project: object, groupId: int) -> None: super().__init__(project, groupId) - def create_tasks(self, feature_ids: List, feature: List,) -> None: + def create_tasks(self, feature_ids: List, features: List,) -> None: """Create tasks for a group feature_geometries is a list of geometries or feature in geojson format. @@ -16,7 +16,7 @@ def create_tasks(self, feature_ids: List, feature: List,) -> None: Coordinates of four two pair coordinates. Every coordinate pair is a vertex. """ - for i in range(0, len(feature_ids)): - task = Task(self, feature_ids[i], feature[i],) + for i, feature_id in enumerate(feature_ids): + task = Task(self, feature_id, features[i],) self.tasks.append(task) self.numberOfTasks = len(self.tasks) From b4ca27356dae0cb6d9d50925456cd284403c49b3 Mon Sep 17 00:00:00 2001 From: Jochen Date: Mon, 6 Dec 2021 17:36:30 +0100 Subject: [PATCH 28/62] check if we enriched properties --- .../integration/test_create_footprint_project.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/mapswipe_workers/tests/integration/test_create_footprint_project.py b/mapswipe_workers/tests/integration/test_create_footprint_project.py index 6bb4590fc..bdfe10c7f 100644 --- a/mapswipe_workers/tests/integration/test_create_footprint_project.py +++ b/mapswipe_workers/tests/integration/test_create_footprint_project.py @@ -31,6 +31,18 @@ def test_create_footprint_project(self): query = "SELECT project_id FROM projects WHERE project_id = %s" result = pg_db.retr_query(query, [element])[0][0] self.assertEqual(result, element) + + # check if usernames made it to postgres + if element != "footprint_link": + query = """ + SELECT count(*) + FROM tasks + WHERE project_id = %s + and project_type_specifics->username is not null + """ + result = pg_db.retr_query(query, [element])[0][0] + self.assertGreater(result, 0) + fb_db = auth.firebaseDB() ref = fb_db.reference(f"/v2/projects/{element}") result = ref.get(shallow=True) From eda27f591ceae0e1e22129299bb21f68452f0a8f Mon Sep 17 00:00:00 2001 From: Jochen Date: Tue, 7 Dec 2021 11:40:08 +0100 Subject: [PATCH 29/62] added comment --- mapswipe_workers/mapswipe_workers/utils/api_calls.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/api_calls.py b/mapswipe_workers/mapswipe_workers/utils/api_calls.py index dd348d870..00031fdb3 100644 --- a/mapswipe_workers/mapswipe_workers/utils/api_calls.py +++ b/mapswipe_workers/mapswipe_workers/utils/api_calls.py @@ -57,7 +57,14 @@ def remove_noise_and_add_user_info(json: dict) -> dict: """Delete unwanted information from properties.""" logger.info("starting filtering and adding extra info") - wanted_attributes = ["@changesetId", "@lastEdit", "@osmId", "@version", "source"] + wanted_attributes = [ + "@changesetId", + "@lastEdit", + "@osmId", + "@version", + "source", + "comment", + ] changeset_results = {} missing_rows = { "@changesetId": 0, From bd1df540225c763ec7d04211e6eac961345ae4dc Mon Sep 17 00:00:00 2001 From: Jochen Date: Tue, 7 Dec 2021 11:42:44 +0100 Subject: [PATCH 30/62] keep more features --- .../mapswipe_workers/project_types/base/project.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/project_types/base/project.py b/mapswipe_workers/mapswipe_workers/project_types/base/project.py index f718665fd..9e2c18914 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/base/project.py +++ b/mapswipe_workers/mapswipe_workers/project_types/base/project.py @@ -124,9 +124,6 @@ def save_project(self): project.pop("inputGeometries", None) project.pop("validInputGeometries", None) - project.pop("inputType", None) - project.pop("filter", None) - project.pop("TMId", None) # Convert Date object to ISO Datetime: # https://www.w3.org/TR/NOTE-datetime From 5432a00597f130b86e52ab5c999d3c7f22869429 Mon Sep 17 00:00:00 2001 From: Jochen Date: Tue, 7 Dec 2021 12:24:28 +0100 Subject: [PATCH 31/62] bug fixes --- .../project_types/arbitrary_geometry/grouping_functions.py | 2 +- .../project_types/arbitrary_geometry/project.py | 5 +++-- .../tests/integration/test_create_footprint_project.py | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/grouping_functions.py b/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/grouping_functions.py index 739222640..2987f3648 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/grouping_functions.py +++ b/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/grouping_functions.py @@ -40,7 +40,7 @@ def group_input_geometries(input_geometries_file, group_size): try: groups[group_id_string] except KeyError: - groups[group_id_string] = {"feature_ids": [], "feature": []} + groups[group_id_string] = {"feature_ids": [], "features": []} # we use a new id here based on the count # since we are not sure that GetFID returns unique values diff --git a/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/project.py b/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/project.py index 65212b2a3..2abdf78a9 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/project.py +++ b/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/project.py @@ -139,7 +139,8 @@ def validate_geometries(self): for feature in layer: feat_geom = feature.GetGeometryRef() geom_name = feat_geom.GetGeometryName() - fid = feature.GetFID + fid = feature.GetFID() + if not feat_geom.IsValid(): layer.DeleteFeature(fid) logger.warning( @@ -194,7 +195,7 @@ def create_groups(self): for group_id, item in raw_groups.items(): group = Group(self, group_id) - group.create_tasks(item["feature_ids"], item["feature"]) + group.create_tasks(item["feature_ids"], item["features"]) # only append valid groups if group.is_valid(): diff --git a/mapswipe_workers/tests/integration/test_create_footprint_project.py b/mapswipe_workers/tests/integration/test_create_footprint_project.py index bdfe10c7f..9afaa3b55 100644 --- a/mapswipe_workers/tests/integration/test_create_footprint_project.py +++ b/mapswipe_workers/tests/integration/test_create_footprint_project.py @@ -33,12 +33,12 @@ def test_create_footprint_project(self): self.assertEqual(result, element) # check if usernames made it to postgres - if element != "footprint_link": + if element != "test_footprint_link": query = """ SELECT count(*) FROM tasks WHERE project_id = %s - and project_type_specifics->username is not null + and project_type_specifics->'properties'->'username' is not null """ result = pg_db.retr_query(query, [element])[0][0] self.assertGreater(result, 0) From e8eac123ff7a5eb256b555ea700064a9e6460eb8 Mon Sep 17 00:00:00 2001 From: Jochen Date: Tue, 7 Dec 2021 13:04:02 +0100 Subject: [PATCH 32/62] remove trailing , --- .../mapswipe_workers/project_types/arbitrary_geometry/group.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/group.py b/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/group.py index c11a55614..af21f8f6e 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/group.py +++ b/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/group.py @@ -17,6 +17,6 @@ def create_tasks(self, feature_ids: List, features: List,) -> None: Every coordinate pair is a vertex. """ for i, feature_id in enumerate(feature_ids): - task = Task(self, feature_id, features[i],) + task = Task(self, feature_id, features[i]) self.tasks.append(task) self.numberOfTasks = len(self.tasks) From e29f2d412ce87d6fdd433c630f9d8c08bd01185a Mon Sep 17 00:00:00 2001 From: Jochen Date: Wed, 8 Dec 2021 12:22:30 +0100 Subject: [PATCH 33/62] catch missing image and fixed a type --- manager_dashboard/manager_dashboard/create.html | 2 +- manager_dashboard/manager_dashboard/js/uploadProjects.js | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/manager_dashboard/manager_dashboard/create.html b/manager_dashboard/manager_dashboard/create.html index fc2cbcd68..fe64e9669 100644 --- a/manager_dashboard/manager_dashboard/create.html +++ b/manager_dashboard/manager_dashboard/create.html @@ -180,7 +180,7 @@

Basic Project Information

  • - + Make sure you have the rights to use this image. It should end with .jpg or .png.

    diff --git a/manager_dashboard/manager_dashboard/js/uploadProjects.js b/manager_dashboard/manager_dashboard/js/uploadProjects.js index dd6ee043d..cae5110a9 100644 --- a/manager_dashboard/manager_dashboard/js/uploadProjects.js +++ b/manager_dashboard/manager_dashboard/js/uploadProjects.js @@ -114,6 +114,12 @@ function uploadProjectImage(mapswipe_import) { var file = document.getElementById('image').files[0] console.log(file) + if (file === undefined){ + alert("Please upload an image, before retrying to upload the project.")ä + modalOngoing.style.display = "none"; + modal.style.display = "none"; + return + } var filename = file.name console.log(filename) // Create a reference to the image From bb221f543d552ac970af9e384306b529784616fd Mon Sep 17 00:00:00 2001 From: Jochen Date: Wed, 8 Dec 2021 12:24:28 +0100 Subject: [PATCH 34/62] =?UTF-8?q?remove=20=C3=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- manager_dashboard/manager_dashboard/js/uploadProjects.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manager_dashboard/manager_dashboard/js/uploadProjects.js b/manager_dashboard/manager_dashboard/js/uploadProjects.js index cae5110a9..01abaefaa 100644 --- a/manager_dashboard/manager_dashboard/js/uploadProjects.js +++ b/manager_dashboard/manager_dashboard/js/uploadProjects.js @@ -115,7 +115,7 @@ function uploadProjectImage(mapswipe_import) { var file = document.getElementById('image').files[0] console.log(file) if (file === undefined){ - alert("Please upload an image, before retrying to upload the project.")ä + alert("Please upload an image, before retrying to upload the project.") modalOngoing.style.display = "none"; modal.style.display = "none"; return From 080baf8fb83e576f7a2f94cea2ad45f6856984f1 Mon Sep 17 00:00:00 2001 From: Jochen Date: Wed, 8 Dec 2021 13:39:13 +0100 Subject: [PATCH 35/62] now working --- .../mapswipe_workers/utils/api_calls.py | 37 +++++++------------ 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/api_calls.py b/mapswipe_workers/mapswipe_workers/utils/api_calls.py index 00031fdb3..e42c0df1a 100644 --- a/mapswipe_workers/mapswipe_workers/utils/api_calls.py +++ b/mapswipe_workers/mapswipe_workers/utils/api_calls.py @@ -42,29 +42,10 @@ def query_osm(changeset_id: int): return response -def add_to_properties(attribute: str, feature: dict, new_properties: dict): - """Adds attribute to new geojson properties if it is needed.""" - if attribute != "comment": - new_properties[attribute.replace("@", "")] = feature["properties"][attribute] - else: - new_properties[attribute.replace("@", "")] = feature["properties"]["tags"][ - attribute - ] - return new_properties - - def remove_noise_and_add_user_info(json: dict) -> dict: """Delete unwanted information from properties.""" logger.info("starting filtering and adding extra info") - wanted_attributes = [ - "@changesetId", - "@lastEdit", - "@osmId", - "@version", - "source", - "comment", - ] changeset_results = {} missing_rows = { "@changesetId": 0, @@ -72,16 +53,19 @@ def remove_noise_and_add_user_info(json: dict) -> dict: "@osmId": 0, "@version": 0, "source": 0, - "comment": 0, + "hashtags": 0, } for feature in json["features"]: new_properties = {} - for attribute in wanted_attributes: + for attribute in missing_rows.keys(): try: - new_properties = add_to_properties(attribute, feature, new_properties) + new_properties[attribute.replace("@", "")] = feature["properties"][ + attribute + ] except KeyError: - missing_rows[attribute] += 1 + if attribute != "hashtags": + missing_rows[attribute] += 1 changeset_id = new_properties["changesetId"] # if changeset_id already queried, use stored result @@ -89,6 +73,13 @@ def remove_noise_and_add_user_info(json: dict) -> dict: changeset_results[changeset_id] = query_osm(changeset_id) new_properties["username"] = changeset_results[changeset_id]["user"] new_properties["userid"] = changeset_results[changeset_id]["uid"] + try: + new_properties["hashtags"] = changeset_results[changeset_id]["tags"][ + "hashtags" + ] + except KeyError: + missing_rows["hashtags"] += 1 + feature["properties"] = new_properties logger.info("finished filtering and adding extra info") if any(x > 0 for x in missing_rows.values()): From f283126cdb39abe8be159a721af4d3cb9e78dce3 Mon Sep 17 00:00:00 2001 From: Hagellach37 Date: Wed, 8 Dec 2021 18:44:41 +0100 Subject: [PATCH 36/62] query osm editor instead of source --- mapswipe_workers/mapswipe_workers/utils/api_calls.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/api_calls.py b/mapswipe_workers/mapswipe_workers/utils/api_calls.py index e42c0df1a..a601eb65d 100644 --- a/mapswipe_workers/mapswipe_workers/utils/api_calls.py +++ b/mapswipe_workers/mapswipe_workers/utils/api_calls.py @@ -52,7 +52,7 @@ def remove_noise_and_add_user_info(json: dict) -> dict: "@lastEdit": 0, "@osmId": 0, "@version": 0, - "source": 0, + "created_by": 0, "hashtags": 0, } @@ -80,6 +80,13 @@ def remove_noise_and_add_user_info(json: dict) -> dict: except KeyError: missing_rows["hashtags"] += 1 + try: + new_properties["created_by"] = changeset_results[changeset_id]["tags"][ + "created_by" + ] + except KeyError: + missing_rows["created_by"] += 1 + feature["properties"] = new_properties logger.info("finished filtering and adding extra info") if any(x > 0 for x in missing_rows.values()): From 7347648cd0406311cd3bb48feec63c0747164121 Mon Sep 17 00:00:00 2001 From: Jochen Date: Thu, 9 Dec 2021 09:49:44 +0100 Subject: [PATCH 37/62] not working properly --- .../mapswipe_workers/utils/api_calls.py | 125 +++++++++++++++--- 1 file changed, 103 insertions(+), 22 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/api_calls.py b/mapswipe_workers/mapswipe_workers/utils/api_calls.py index 00031fdb3..52ae950a3 100644 --- a/mapswipe_workers/mapswipe_workers/utils/api_calls.py +++ b/mapswipe_workers/mapswipe_workers/utils/api_calls.py @@ -1,3 +1,7 @@ +import queue +import threading +from xml.etree import ElementTree + import requests from requests.adapters import HTTPAdapter from requests.packages.urllib3.util.retry import Retry @@ -27,21 +31,6 @@ def geojsonToFeatureCollection(geojson: dict) -> dict: return geojson -def query_osm(changeset_id: int): - """Get data from changesetId.""" - url = OSM_API_LINK + f"changeset/{changeset_id}.json" - - response = retry_get(url) - - if response.status_code != 200: - err = f"osm request failed: {response.status_code}" - logger.warning(f"{err}") - logger.warning(response.json()) - raise CustomError(err) - response = response.json()["elements"][0] - return response - - def add_to_properties(attribute: str, feature: dict, new_properties: dict): """Adds attribute to new geojson properties if it is needed.""" if attribute != "comment": @@ -53,6 +42,78 @@ def add_to_properties(attribute: str, feature: dict, new_properties: dict): return new_properties +# startblock for osm changesets threading +q = queue.Queue() +osm_results = {} + + +def chunks(_list, n_objects): + return [ + _list[i * n_objects : (i + 1) * n_objects] + for i in range((len(_list) + n_objects - 1) // n_objects) + ] + + +def query_osm(changeset_ids: list): + """Get data from changesetId.""" + id_string = "" + for id in changeset_ids: + id_string += f"{id}," + + id_string = id_string[:-1] + + url = OSM_API_LINK + f"changesets?changesets={id_string}" + response = retry_get(url) + + if response.status_code != 200: + err = f"osm request failed: {response.status_code}" + logger.warning(f"{err}") + logger.warning(response.json()) + raise CustomError(err) + tree = ElementTree.fromstring(response.content) + for changeset in tree: + id = changeset.attrib["id"] + username = changeset.attrib["user"] + userid = changeset.attrib["uid"] + hashtags = None + for tag in changeset: + if tag.attrib["k"] == "hashtags": + hashtags = tag.attrib["v"] + osm_results[id] = {"username": username, "userid": userid, "hashtags": hashtags} + return + + +def worker(): + while True: + item = q.get() + if item is None: + break + query_osm(item) + logger.info("worker finished") + q.task_done() + + +def start_workers(worker_pool=2): + threads = [] + for i in range(worker_pool): + t = threading.Thread(target=worker) + t.daemon = True + t.start() + threads.append(t) + return threads + + +def stop_workers(threads): + # stop workers + for i in threads: + q.put(None) + for t in threads: + t.join() + + +# endblock for osm changesets threading + + def remove_noise_and_add_user_info(json: dict) -> dict: """Delete unwanted information from properties.""" logger.info("starting filtering and adding extra info") @@ -65,7 +126,7 @@ def remove_noise_and_add_user_info(json: dict) -> dict: "source", "comment", ] - changeset_results = {} + changeset_ids = [] missing_rows = { "@changesetId": 0, "@lastEdit": 0, @@ -83,13 +144,33 @@ def remove_noise_and_add_user_info(json: dict) -> dict: except KeyError: missing_rows[attribute] += 1 changeset_id = new_properties["changesetId"] - - # if changeset_id already queried, use stored result - if changeset_id not in changeset_results.keys(): - changeset_results[changeset_id] = query_osm(changeset_id) - new_properties["username"] = changeset_results[changeset_id]["user"] - new_properties["userid"] = changeset_results[changeset_id]["uid"] + if changeset_id not in changeset_ids: + changeset_ids.append(changeset_id) feature["properties"] = new_properties + logger.info( + f"""{len(changeset_ids)} changesets will + be queried in roughly {int(len(changeset_ids)/99) +1} batches""" + ) + chunk_list = chunks(changeset_ids, 99) + for subset in chunk_list: + q.put(subset) + q.put([100, 2002]) + assert len(changeset_ids) == sum([len(arr) for arr in chunk_list]) + workers = start_workers(worker_pool=2) # more could lead to ban from osm api + + q.join() + stop_workers(workers) + assert len(changeset_ids) == len(osm_results.keys()) - 2 + + for feature in json["features"]: + try: + changeset = osm_results[feature["properties"]["changesetId"]] + feature["properties"]["username"] = changeset["username"] + feature["properties"]["userid"] = changeset["userid"] + feature["properties"]["hashtags"] = changeset["hashtags"] + except KeyError: + logger.info(feature["properties"]["changesetId"]) + logger.info("finished filtering and adding extra info") if any(x > 0 for x in missing_rows.values()): logger.warning(f"features missing values:\n{missing_rows}") From b955be7fb957eae84759e8874470317e5c0af0c6 Mon Sep 17 00:00:00 2001 From: Jochen Date: Thu, 9 Dec 2021 11:54:18 +0100 Subject: [PATCH 38/62] no threads no worries --- .../mapswipe_workers/utils/api_calls.py | 83 +++++-------------- 1 file changed, 23 insertions(+), 60 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/api_calls.py b/mapswipe_workers/mapswipe_workers/utils/api_calls.py index 52ae950a3..2f6d5d77b 100644 --- a/mapswipe_workers/mapswipe_workers/utils/api_calls.py +++ b/mapswipe_workers/mapswipe_workers/utils/api_calls.py @@ -1,5 +1,3 @@ -import queue -import threading from xml.etree import ElementTree import requests @@ -35,6 +33,10 @@ def add_to_properties(attribute: str, feature: dict, new_properties: dict): """Adds attribute to new geojson properties if it is needed.""" if attribute != "comment": new_properties[attribute.replace("@", "")] = feature["properties"][attribute] + elif attribute != "@changesetId": + new_properties[attribute.replace("@", "")] = int( + feature["properties"][attribute] + ) else: new_properties[attribute.replace("@", "")] = feature["properties"]["tags"][ attribute @@ -42,8 +44,6 @@ def add_to_properties(attribute: str, feature: dict, new_properties: dict): return new_properties -# startblock for osm changesets threading -q = queue.Queue() osm_results = {} @@ -63,6 +63,7 @@ def query_osm(changeset_ids: list): id_string = id_string[:-1] url = OSM_API_LINK + f"changesets?changesets={id_string}" + logger.info(url) response = retry_get(url) if response.status_code != 200: @@ -71,52 +72,27 @@ def query_osm(changeset_ids: list): logger.warning(response.json()) raise CustomError(err) tree = ElementTree.fromstring(response.content) - for changeset in tree: + + for changeset in tree.iter("changeset"): id = changeset.attrib["id"] username = changeset.attrib["user"] userid = changeset.attrib["uid"] hashtags = None - for tag in changeset: + for tag in changeset.iter("tag"): if tag.attrib["k"] == "hashtags": hashtags = tag.attrib["v"] - osm_results[id] = {"username": username, "userid": userid, "hashtags": hashtags} + osm_results[int(id)] = { + "username": username, + "userid": userid, + "hashtags": hashtags, + } return -def worker(): - while True: - item = q.get() - if item is None: - break - query_osm(item) - logger.info("worker finished") - q.task_done() - - -def start_workers(worker_pool=2): - threads = [] - for i in range(worker_pool): - t = threading.Thread(target=worker) - t.daemon = True - t.start() - threads.append(t) - return threads - - -def stop_workers(threads): - # stop workers - for i in threads: - q.put(None) - for t in threads: - t.join() - - -# endblock for osm changesets threading - - def remove_noise_and_add_user_info(json: dict) -> dict: """Delete unwanted information from properties.""" logger.info("starting filtering and adding extra info") + osm_results.clear() wanted_attributes = [ "@changesetId", @@ -126,7 +102,6 @@ def remove_noise_and_add_user_info(json: dict) -> dict: "source", "comment", ] - changeset_ids = [] missing_rows = { "@changesetId": 0, "@lastEdit": 0, @@ -143,33 +118,21 @@ def remove_noise_and_add_user_info(json: dict) -> dict: new_properties = add_to_properties(attribute, feature, new_properties) except KeyError: missing_rows[attribute] += 1 - changeset_id = new_properties["changesetId"] - if changeset_id not in changeset_ids: - changeset_ids.append(changeset_id) + osm_results[new_properties["changesetId"]] = None feature["properties"] = new_properties logger.info( - f"""{len(changeset_ids)} changesets will - be queried in roughly {int(len(changeset_ids)/99) +1} batches""" + f"""{len(osm_results.keys())} changesets will + be queried in roughly {int(len(osm_results.keys())/100) +1} batches""" ) - chunk_list = chunks(changeset_ids, 99) + chunk_list = chunks(list(osm_results.keys()), 100) for subset in chunk_list: - q.put(subset) - q.put([100, 2002]) - assert len(changeset_ids) == sum([len(arr) for arr in chunk_list]) - workers = start_workers(worker_pool=2) # more could lead to ban from osm api - - q.join() - stop_workers(workers) - assert len(changeset_ids) == len(osm_results.keys()) - 2 + query_osm(subset) for feature in json["features"]: - try: - changeset = osm_results[feature["properties"]["changesetId"]] - feature["properties"]["username"] = changeset["username"] - feature["properties"]["userid"] = changeset["userid"] - feature["properties"]["hashtags"] = changeset["hashtags"] - except KeyError: - logger.info(feature["properties"]["changesetId"]) + changeset = osm_results[feature["properties"]["changesetId"]] + feature["properties"]["username"] = changeset["username"] + feature["properties"]["userid"] = changeset["userid"] + feature["properties"]["hashtags"] = changeset["hashtags"] logger.info("finished filtering and adding extra info") if any(x > 0 for x in missing_rows.values()): From 58756d00a3a8b21f699137208af4f56e954d171f Mon Sep 17 00:00:00 2001 From: Jochen Date: Thu, 9 Dec 2021 15:08:01 +0100 Subject: [PATCH 39/62] merge --- mapswipe_workers/mapswipe_workers/utils/api_calls.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/api_calls.py b/mapswipe_workers/mapswipe_workers/utils/api_calls.py index a8fa1869e..1eea8fb0d 100644 --- a/mapswipe_workers/mapswipe_workers/utils/api_calls.py +++ b/mapswipe_workers/mapswipe_workers/utils/api_calls.py @@ -62,17 +62,17 @@ def query_osm(changeset_ids: list): id = changeset.attrib["id"] username = changeset.attrib["user"] userid = changeset.attrib["uid"] - hashtags = created_by = None + comment = created_by = None for tag in changeset.iter("tag"): - if tag.attrib["k"] == "hashtags": - hashtags = tag.attrib["v"] + if tag.attrib["k"] == "comment": + comment = tag.attrib["v"] if tag.attrib["k"] == "created_by": created_by = tag.attrib["v"] osm_results[int(id)] = { "username": username, "userid": userid, - "hashtags": hashtags, + "comment": comment, "created_by": created_by, } return @@ -125,7 +125,7 @@ def remove_noise_and_add_user_info(json: dict) -> dict: changeset = osm_results[feature["properties"]["changesetId"]] feature["properties"]["username"] = changeset["username"] feature["properties"]["userid"] = changeset["userid"] - feature["properties"]["hashtags"] = changeset["hashtags"] + feature["properties"]["comment"] = changeset["comment"] feature["properties"]["created_by"] = changeset["created_by"] logger.info("finished filtering and adding extra info") From 0617fce29463a054ff4fc530fb94062df99ad243 Mon Sep 17 00:00:00 2001 From: Jochen Date: Thu, 9 Dec 2021 15:08:48 +0100 Subject: [PATCH 40/62] remove url --- mapswipe_workers/mapswipe_workers/utils/api_calls.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/api_calls.py b/mapswipe_workers/mapswipe_workers/utils/api_calls.py index 1eea8fb0d..8a4465959 100644 --- a/mapswipe_workers/mapswipe_workers/utils/api_calls.py +++ b/mapswipe_workers/mapswipe_workers/utils/api_calls.py @@ -48,9 +48,7 @@ def query_osm(changeset_ids: list): id_string = id_string[:-1] url = OSM_API_LINK + f"changesets?changesets={id_string}" - logger.info(url) response = retry_get(url) - if response.status_code != 200: err = f"osm request failed: {response.status_code}" logger.warning(f"{err}") From 1b524344cccdb9d93f1394eec08d6f44bd59df11 Mon Sep 17 00:00:00 2001 From: Jochen Date: Thu, 9 Dec 2021 15:17:48 +0100 Subject: [PATCH 41/62] remove url --- mapswipe_workers/mapswipe_workers/utils/api_calls.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/api_calls.py b/mapswipe_workers/mapswipe_workers/utils/api_calls.py index 8a4465959..affe4c603 100644 --- a/mapswipe_workers/mapswipe_workers/utils/api_calls.py +++ b/mapswipe_workers/mapswipe_workers/utils/api_calls.py @@ -111,19 +111,23 @@ def remove_noise_and_add_user_info(json: dict) -> dict: osm_results[new_properties["changesetId"]] = None feature["properties"] = new_properties + len_osm = len(osm_results.keys()) + batches = int(len(osm_results.keys()) / 100) + 1 logger.info( - f"""{len(osm_results.keys())} changesets will - be queried in roughly {int(len(osm_results.keys()) / 100) + 1} batches""" + f"""{len_osm} changesets will be queried in roughly {batches} batches""" ) chunk_list = chunks(list(osm_results.keys()), 100) - for subset in chunk_list: + for i, subset in enumerate(chunk_list): query_osm(subset) + logger.info( + "finished query {i}/{len(chunk_list)},{round(i/len(chunk_list), 1)}%" + ) for feature in json["features"]: changeset = osm_results[feature["properties"]["changesetId"]] feature["properties"]["username"] = changeset["username"] feature["properties"]["userid"] = changeset["userid"] - feature["properties"]["comment"] = changeset["comment"] + feature["properties"]["comment"] = changeset["comment"].replace("\n", "") feature["properties"]["created_by"] = changeset["created_by"] logger.info("finished filtering and adding extra info") From 6e29d725efc927590cc3b5cdeb72da85e38b0f4c Mon Sep 17 00:00:00 2001 From: Jochen Date: Thu, 9 Dec 2021 15:18:03 +0100 Subject: [PATCH 42/62] remove url --- mapswipe_workers/mapswipe_workers/utils/api_calls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/api_calls.py b/mapswipe_workers/mapswipe_workers/utils/api_calls.py index affe4c603..6f2ab88ce 100644 --- a/mapswipe_workers/mapswipe_workers/utils/api_calls.py +++ b/mapswipe_workers/mapswipe_workers/utils/api_calls.py @@ -120,7 +120,7 @@ def remove_noise_and_add_user_info(json: dict) -> dict: for i, subset in enumerate(chunk_list): query_osm(subset) logger.info( - "finished query {i}/{len(chunk_list)},{round(i/len(chunk_list), 1)}%" + f"finished query {i}/{len(chunk_list)},{round(i/len(chunk_list), 1)}%" ) for feature in json["features"]: From e6bdcb504b8e900c634f511a86bc3f4299d64159 Mon Sep 17 00:00:00 2001 From: Jochen Date: Thu, 9 Dec 2021 15:21:19 +0100 Subject: [PATCH 43/62] bugfix --- mapswipe_workers/mapswipe_workers/utils/api_calls.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/api_calls.py b/mapswipe_workers/mapswipe_workers/utils/api_calls.py index 6f2ab88ce..596e8da07 100644 --- a/mapswipe_workers/mapswipe_workers/utils/api_calls.py +++ b/mapswipe_workers/mapswipe_workers/utils/api_calls.py @@ -63,7 +63,10 @@ def query_osm(changeset_ids: list): comment = created_by = None for tag in changeset.iter("tag"): if tag.attrib["k"] == "comment": - comment = tag.attrib["v"] + try: + comment = tag.attrib["v"].replace("\n", " ") + except AttributeError: + pass if tag.attrib["k"] == "created_by": created_by = tag.attrib["v"] @@ -127,7 +130,7 @@ def remove_noise_and_add_user_info(json: dict) -> dict: changeset = osm_results[feature["properties"]["changesetId"]] feature["properties"]["username"] = changeset["username"] feature["properties"]["userid"] = changeset["userid"] - feature["properties"]["comment"] = changeset["comment"].replace("\n", "") + feature["properties"]["comment"] = changeset["comment"] feature["properties"]["created_by"] = changeset["created_by"] logger.info("finished filtering and adding extra info") From 36ac891ea48dfda1e01edc463d4a7456cb8ed481 Mon Sep 17 00:00:00 2001 From: Jochen Date: Thu, 9 Dec 2021 15:24:03 +0100 Subject: [PATCH 44/62] better readability --- mapswipe_workers/mapswipe_workers/utils/api_calls.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/api_calls.py b/mapswipe_workers/mapswipe_workers/utils/api_calls.py index 596e8da07..70151cf0e 100644 --- a/mapswipe_workers/mapswipe_workers/utils/api_calls.py +++ b/mapswipe_workers/mapswipe_workers/utils/api_calls.py @@ -29,9 +29,6 @@ def geojsonToFeatureCollection(geojson: dict) -> dict: return geojson -osm_results = {} - - def chunks(_list, n_objects): return [ _list[i * n_objects : (i + 1) * n_objects] @@ -39,7 +36,7 @@ def chunks(_list, n_objects): ] -def query_osm(changeset_ids: list): +def query_osm(changeset_ids: list, osm_results): """Get data from changesetId.""" id_string = "" for id in changeset_ids: @@ -76,7 +73,7 @@ def query_osm(changeset_ids: list): "comment": comment, "created_by": created_by, } - return + return osm_results def add_to_properties(attribute: str, feature: dict, new_properties: dict): @@ -93,7 +90,7 @@ def add_to_properties(attribute: str, feature: dict, new_properties: dict): def remove_noise_and_add_user_info(json: dict) -> dict: """Delete unwanted information from properties.""" logger.info("starting filtering and adding extra info") - osm_results.clear() + osm_results = {} missing_rows = { "@changesetId": 0, @@ -121,7 +118,7 @@ def remove_noise_and_add_user_info(json: dict) -> dict: ) chunk_list = chunks(list(osm_results.keys()), 100) for i, subset in enumerate(chunk_list): - query_osm(subset) + osm_results = query_osm(subset, osm_results) logger.info( f"finished query {i}/{len(chunk_list)},{round(i/len(chunk_list), 1)}%" ) From edb5b9662daa7b1754be7bbe02a14416ddf902fe Mon Sep 17 00:00:00 2001 From: Jochen Date: Thu, 9 Dec 2021 15:24:29 +0100 Subject: [PATCH 45/62] _list to arr --- mapswipe_workers/mapswipe_workers/utils/api_calls.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/api_calls.py b/mapswipe_workers/mapswipe_workers/utils/api_calls.py index 70151cf0e..7d84e68be 100644 --- a/mapswipe_workers/mapswipe_workers/utils/api_calls.py +++ b/mapswipe_workers/mapswipe_workers/utils/api_calls.py @@ -29,10 +29,10 @@ def geojsonToFeatureCollection(geojson: dict) -> dict: return geojson -def chunks(_list, n_objects): +def chunks(arr, n_objects): return [ - _list[i * n_objects : (i + 1) * n_objects] - for i in range((len(_list) + n_objects - 1) // n_objects) + arr[i * n_objects : (i + 1) * n_objects] + for i in range((len(arr) + n_objects - 1) // n_objects) ] From 4241fa621d21cb545adc1eb7663db9d9735049c7 Mon Sep 17 00:00:00 2001 From: Jochen Date: Thu, 9 Dec 2021 16:01:21 +0100 Subject: [PATCH 46/62] osm_result to changeset_result --- .../mapswipe_workers/utils/api_calls.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/api_calls.py b/mapswipe_workers/mapswipe_workers/utils/api_calls.py index 7d84e68be..04ce4357a 100644 --- a/mapswipe_workers/mapswipe_workers/utils/api_calls.py +++ b/mapswipe_workers/mapswipe_workers/utils/api_calls.py @@ -36,7 +36,7 @@ def chunks(arr, n_objects): ] -def query_osm(changeset_ids: list, osm_results): +def query_osm(changeset_ids: list, changeset_results): """Get data from changesetId.""" id_string = "" for id in changeset_ids: @@ -67,13 +67,13 @@ def query_osm(changeset_ids: list, osm_results): if tag.attrib["k"] == "created_by": created_by = tag.attrib["v"] - osm_results[int(id)] = { + changeset_results[int(id)] = { "username": username, "userid": userid, "comment": comment, "created_by": created_by, } - return osm_results + return changeset_results def add_to_properties(attribute: str, feature: dict, new_properties: dict): @@ -90,7 +90,7 @@ def add_to_properties(attribute: str, feature: dict, new_properties: dict): def remove_noise_and_add_user_info(json: dict) -> dict: """Delete unwanted information from properties.""" logger.info("starting filtering and adding extra info") - osm_results = {} + changeset_results = {} missing_rows = { "@changesetId": 0, @@ -108,23 +108,23 @@ def remove_noise_and_add_user_info(json: dict) -> dict: ] except KeyError: missing_rows[attribute] += 1 - osm_results[new_properties["changesetId"]] = None + changeset_results[new_properties["changesetId"]] = None feature["properties"] = new_properties - len_osm = len(osm_results.keys()) - batches = int(len(osm_results.keys()) / 100) + 1 + len_osm = len(changeset_results.keys()) + batches = int(len(changeset_results.keys()) / 100) + 1 logger.info( f"""{len_osm} changesets will be queried in roughly {batches} batches""" ) - chunk_list = chunks(list(osm_results.keys()), 100) + chunk_list = chunks(list(changeset_results.keys()), 100) for i, subset in enumerate(chunk_list): - osm_results = query_osm(subset, osm_results) + changeset_results = query_osm(subset, changeset_results) logger.info( f"finished query {i}/{len(chunk_list)},{round(i/len(chunk_list), 1)}%" ) for feature in json["features"]: - changeset = osm_results[feature["properties"]["changesetId"]] + changeset = changeset_results[feature["properties"]["changesetId"]] feature["properties"]["username"] = changeset["username"] feature["properties"]["userid"] = changeset["userid"] feature["properties"]["comment"] = changeset["comment"] From 90ca09753d63e8ef1fe6cdd21e3ac3857f6d43b8 Mon Sep 17 00:00:00 2001 From: Hagellach37 Date: Thu, 9 Dec 2021 18:12:01 +0100 Subject: [PATCH 47/62] fix progress calculation. multiply by 100 to get percent --- mapswipe_workers/mapswipe_workers/utils/api_calls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/api_calls.py b/mapswipe_workers/mapswipe_workers/utils/api_calls.py index 04ce4357a..76fb6723d 100644 --- a/mapswipe_workers/mapswipe_workers/utils/api_calls.py +++ b/mapswipe_workers/mapswipe_workers/utils/api_calls.py @@ -120,7 +120,7 @@ def remove_noise_and_add_user_info(json: dict) -> dict: for i, subset in enumerate(chunk_list): changeset_results = query_osm(subset, changeset_results) logger.info( - f"finished query {i}/{len(chunk_list)},{round(i/len(chunk_list), 1)}%" + f"finished query {i}/{len(chunk_list)}, {100*round(i/len(chunk_list), 2)}%" ) for feature in json["features"]: From 4ac49048316e6ee25a2bbeca518c2c0277ab0072 Mon Sep 17 00:00:00 2001 From: Hagellach37 Date: Thu, 9 Dec 2021 18:18:37 +0100 Subject: [PATCH 48/62] make sure to remove quotation marks from osm changeset attribute values to avoid problemns in postgres --- mapswipe_workers/mapswipe_workers/utils/api_calls.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/api_calls.py b/mapswipe_workers/mapswipe_workers/utils/api_calls.py index 76fb6723d..15039b0f9 100644 --- a/mapswipe_workers/mapswipe_workers/utils/api_calls.py +++ b/mapswipe_workers/mapswipe_workers/utils/api_calls.py @@ -125,10 +125,11 @@ def remove_noise_and_add_user_info(json: dict) -> dict: for feature in json["features"]: changeset = changeset_results[feature["properties"]["changesetId"]] - feature["properties"]["username"] = changeset["username"] + # we need to replace " as this will cause problems when importing to postgres + feature["properties"]["username"] = changeset["username"].replace('"', "") feature["properties"]["userid"] = changeset["userid"] - feature["properties"]["comment"] = changeset["comment"] - feature["properties"]["created_by"] = changeset["created_by"] + feature["properties"]["comment"] = changeset["comment"].replace('"', "") + feature["properties"]["created_by"] = changeset["created_by"].replace('"', "") logger.info("finished filtering and adding extra info") if any(x > 0 for x in missing_rows.values()): From a4a6585e2bec98a9a65a6eddca08e5345233a5c8 Mon Sep 17 00:00:00 2001 From: Hagellach37 Date: Thu, 9 Dec 2021 19:47:47 +0100 Subject: [PATCH 49/62] move check for quotes in comment --- .../mapswipe_workers/utils/api_calls.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/api_calls.py b/mapswipe_workers/mapswipe_workers/utils/api_calls.py index 15039b0f9..f4f744443 100644 --- a/mapswipe_workers/mapswipe_workers/utils/api_calls.py +++ b/mapswipe_workers/mapswipe_workers/utils/api_calls.py @@ -125,11 +125,16 @@ def remove_noise_and_add_user_info(json: dict) -> dict: for feature in json["features"]: changeset = changeset_results[feature["properties"]["changesetId"]] - # we need to replace " as this will cause problems when importing to postgres - feature["properties"]["username"] = changeset["username"].replace('"', "") feature["properties"]["userid"] = changeset["userid"] - feature["properties"]["comment"] = changeset["comment"].replace('"', "") - feature["properties"]["created_by"] = changeset["created_by"].replace('"', "") + for attribute_name in ["username", "comment", "created_by"]: + # we need to replace " as this will cause problems + # when importing to postgres + try: + feature["properties"][attribute_name] = changeset[ + attribute_name + ].replace('"', "") + except AttributeError: + pass logger.info("finished filtering and adding extra info") if any(x > 0 for x in missing_rows.values()): From fb710fcf186081e30548db28c33125ca4fde18d9 Mon Sep 17 00:00:00 2001 From: Jochen Date: Thu, 9 Dec 2021 21:33:28 +0100 Subject: [PATCH 50/62] changes --- .../mapswipe_workers/utils/api_calls.py | 22 ++++++++----------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/api_calls.py b/mapswipe_workers/mapswipe_workers/utils/api_calls.py index 04ce4357a..f9cf14f09 100644 --- a/mapswipe_workers/mapswipe_workers/utils/api_calls.py +++ b/mapswipe_workers/mapswipe_workers/utils/api_calls.py @@ -12,6 +12,13 @@ ) +def remove_troublesome_chars(string: str): + troublesome_chars = {'"': "", "'": "", "\n": ""} + for k, v in troublesome_chars.items(): + string = string.replace(k, v) + return string + + def retry_get(url, retries=3, timeout=4): retry = Retry(total=retries) session = requests.Session() @@ -55,13 +62,13 @@ def query_osm(changeset_ids: list, changeset_results): for changeset in tree.iter("changeset"): id = changeset.attrib["id"] - username = changeset.attrib["user"] + username = remove_troublesome_chars(changeset.attrib["user"]) userid = changeset.attrib["uid"] comment = created_by = None for tag in changeset.iter("tag"): if tag.attrib["k"] == "comment": try: - comment = tag.attrib["v"].replace("\n", " ") + comment = remove_troublesome_chars(tag.attrib["v"]) except AttributeError: pass if tag.attrib["k"] == "created_by": @@ -76,17 +83,6 @@ def query_osm(changeset_ids: list, changeset_results): return changeset_results -def add_to_properties(attribute: str, feature: dict, new_properties: dict): - """Adds attribute to new geojson properties if it is needed.""" - if attribute != "comment": - new_properties[attribute.replace("@", "")] = feature["properties"][attribute] - else: - new_properties[attribute.replace("@", "")] = feature["properties"]["tags"][ - attribute - ] - return new_properties - - def remove_noise_and_add_user_info(json: dict) -> dict: """Delete unwanted information from properties.""" logger.info("starting filtering and adding extra info") From 365fa8342df2108c5c28b2ad71169a86b590df0e Mon Sep 17 00:00:00 2001 From: Jochen Date: Thu, 9 Dec 2021 21:57:11 +0100 Subject: [PATCH 51/62] fixed 3? bugs --- .../mapswipe_workers/utils/api_calls.py | 36 +++++++------------ 1 file changed, 13 insertions(+), 23 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/api_calls.py b/mapswipe_workers/mapswipe_workers/utils/api_calls.py index dc5f4adaf..830a6745b 100644 --- a/mapswipe_workers/mapswipe_workers/utils/api_calls.py +++ b/mapswipe_workers/mapswipe_workers/utils/api_calls.py @@ -13,6 +13,8 @@ def remove_troublesome_chars(string: str): + if type(string) is not str: + return string troublesome_chars = {'"': "", "'": "", "\n": ""} for k, v in troublesome_chars.items(): string = string.replace(k, v) @@ -21,9 +23,9 @@ def remove_troublesome_chars(string: str): def retry_get(url, retries=3, timeout=4): retry = Retry(total=retries) - session = requests.Session() - session.mount("https://", HTTPAdapter(max_retries=retry)) - return session.get(url, timeout=timeout) + with requests.Session() as session: + session.mount("https://", HTTPAdapter(max_retries=retry)) + return session.get(url, timeout=timeout) def geojsonToFeatureCollection(geojson: dict) -> dict: @@ -67,18 +69,15 @@ def query_osm(changeset_ids: list, changeset_results): comment = created_by = None for tag in changeset.iter("tag"): if tag.attrib["k"] == "comment": - try: - comment = remove_troublesome_chars(tag.attrib["v"]) - except AttributeError: - pass + comment = tag.attrib["v"] if tag.attrib["k"] == "created_by": created_by = tag.attrib["v"] changeset_results[int(id)] = { - "username": username, + "username": remove_troublesome_chars(username), "userid": userid, - "comment": comment, - "created_by": created_by, + "comment": remove_troublesome_chars(comment), + "created_by": remove_troublesome_chars(created_by), } return changeset_results @@ -115,22 +114,13 @@ def remove_noise_and_add_user_info(json: dict) -> dict: chunk_list = chunks(list(changeset_results.keys()), 100) for i, subset in enumerate(chunk_list): changeset_results = query_osm(subset, changeset_results) - logger.info( - f"finished query {i}/{len(chunk_list)}, {100*round(i/len(chunk_list), 2)}%" - ) + progress = round(100 * ((i + 1) / len(chunk_list)), 1) + logger.info(f"finished query {i+1}/{len(chunk_list)}, {progress}") for feature in json["features"]: changeset = changeset_results[feature["properties"]["changesetId"]] - feature["properties"]["userid"] = changeset["userid"] - for attribute_name in ["username", "comment", "created_by"]: - # we need to replace " as this will cause problems - # when importing to postgres - try: - feature["properties"][attribute_name] = changeset[ - attribute_name - ].replace('"', "") - except AttributeError: - pass + for attribute_name in ["username", "comment", "created_by", "userid"]: + feature["properties"][attribute_name] = changeset[attribute_name] logger.info("finished filtering and adding extra info") if any(x > 0 for x in missing_rows.values()): From 2303dcd81b4328dad5320c2063a05f20d3a6c3d4 Mon Sep 17 00:00:00 2001 From: Jochen Date: Thu, 9 Dec 2021 22:00:38 +0100 Subject: [PATCH 52/62] add docstrings --- mapswipe_workers/mapswipe_workers/utils/api_calls.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/api_calls.py b/mapswipe_workers/mapswipe_workers/utils/api_calls.py index 830a6745b..a4b644715 100644 --- a/mapswipe_workers/mapswipe_workers/utils/api_calls.py +++ b/mapswipe_workers/mapswipe_workers/utils/api_calls.py @@ -13,6 +13,7 @@ def remove_troublesome_chars(string: str): + """Remove chars that cause trouble when pushed into postgres.""" if type(string) is not str: return string troublesome_chars = {'"': "", "'": "", "\n": ""} @@ -22,6 +23,7 @@ def remove_troublesome_chars(string: str): def retry_get(url, retries=3, timeout=4): + """Retry a query for a variable amount of tries.""" retry = Retry(total=retries) with requests.Session() as session: session.mount("https://", HTTPAdapter(max_retries=retry)) @@ -29,6 +31,7 @@ def retry_get(url, retries=3, timeout=4): def geojsonToFeatureCollection(geojson: dict) -> dict: + """Take a GeoJson and wrap it in a FeatureCollection.""" if geojson["type"] != "FeatureCollection": collection = { "type": "FeatureCollection", @@ -39,6 +42,7 @@ def geojsonToFeatureCollection(geojson: dict) -> dict: def chunks(arr, n_objects): + """Return a list of list with n_objects in each sublist.""" return [ arr[i * n_objects : (i + 1) * n_objects] for i in range((len(arr) + n_objects - 1) // n_objects) @@ -130,9 +134,7 @@ def remove_noise_and_add_user_info(json: dict) -> dict: def ohsome(request: dict, area: str, properties=None) -> dict: - """ - Request data from Ohsome API. - """ + """Request data from Ohsome API.""" url = OHSOME_API_LINK + request["endpoint"] data = {"bpolys": area, "filter": request["filter"]} if properties: From b33e2c1cca3938f1a22fb05b0d4c3d4c6fd73af4 Mon Sep 17 00:00:00 2001 From: Jochen Date: Thu, 9 Dec 2021 22:07:34 +0100 Subject: [PATCH 53/62] just stylin --- mapswipe_workers/mapswipe_workers/utils/api_calls.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/api_calls.py b/mapswipe_workers/mapswipe_workers/utils/api_calls.py index a4b644715..555dce1a2 100644 --- a/mapswipe_workers/mapswipe_workers/utils/api_calls.py +++ b/mapswipe_workers/mapswipe_workers/utils/api_calls.py @@ -51,11 +51,7 @@ def chunks(arr, n_objects): def query_osm(changeset_ids: list, changeset_results): """Get data from changesetId.""" - id_string = "" - for id in changeset_ids: - id_string += f"{id}," - - id_string = id_string[:-1] + id_string = ",".join(map(str, changeset_ids)) url = OSM_API_LINK + f"changesets?changesets={id_string}" response = retry_get(url) From adb49598cf8cb703a022369a83055ef8894feac7 Mon Sep 17 00:00:00 2001 From: Hagellach37 Date: Fri, 10 Dec 2021 11:47:10 +0100 Subject: [PATCH 54/62] replace \n with space --- mapswipe_workers/mapswipe_workers/utils/api_calls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/api_calls.py b/mapswipe_workers/mapswipe_workers/utils/api_calls.py index 555dce1a2..e3100d149 100644 --- a/mapswipe_workers/mapswipe_workers/utils/api_calls.py +++ b/mapswipe_workers/mapswipe_workers/utils/api_calls.py @@ -16,7 +16,7 @@ def remove_troublesome_chars(string: str): """Remove chars that cause trouble when pushed into postgres.""" if type(string) is not str: return string - troublesome_chars = {'"': "", "'": "", "\n": ""} + troublesome_chars = {'"': "", "'": "", "\n": " "} for k, v in troublesome_chars.items(): string = string.replace(k, v) return string From 88ab2f15be1f13db4dbfd30a3f7a8aa46cb11f48 Mon Sep 17 00:00:00 2001 From: Jochen Date: Tue, 21 Dec 2021 18:12:08 +0100 Subject: [PATCH 55/62] notworking --- .../mapswipe_workers/definitions.py | 3 +- .../mapswipe_workers/utils/api_calls.py | 53 +++++++++---------- 2 files changed, 26 insertions(+), 30 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/definitions.py b/mapswipe_workers/mapswipe_workers/definitions.py index 62daebdd5..526e85365 100644 --- a/mapswipe_workers/mapswipe_workers/definitions.py +++ b/mapswipe_workers/mapswipe_workers/definitions.py @@ -13,7 +13,8 @@ LOGGING_FILE_PATH = os.path.join(DATA_PATH, "mapswipe_workers.log") OHSOME_API_LINK = "https://api.ohsome.org/v1/" -OSM_API_LINK = "https://www.openstreetmap.org/api/0.6/" +OSMCHA_API_LINK = "https://osmcha.org/api/v1/" +OSMCHA_API_KEY = os.environ["OSMCHA_API_KEY"] # number of geometries for project geometries MAX_INPUT_GEOMETRIES = 10 diff --git a/mapswipe_workers/mapswipe_workers/utils/api_calls.py b/mapswipe_workers/mapswipe_workers/utils/api_calls.py index e3100d149..345e4215e 100644 --- a/mapswipe_workers/mapswipe_workers/utils/api_calls.py +++ b/mapswipe_workers/mapswipe_workers/utils/api_calls.py @@ -1,12 +1,11 @@ -from xml.etree import ElementTree - import requests from requests.adapters import HTTPAdapter from requests.packages.urllib3.util.retry import Retry from mapswipe_workers.definitions import ( OHSOME_API_LINK, - OSM_API_LINK, + OSMCHA_API_KEY, + OSMCHA_API_LINK, CustomError, logger, ) @@ -27,7 +26,8 @@ def retry_get(url, retries=3, timeout=4): retry = Retry(total=retries) with requests.Session() as session: session.mount("https://", HTTPAdapter(max_retries=retry)) - return session.get(url, timeout=timeout) + headers = {"Authorization": OSMCHA_API_KEY} + return session.get(url, timeout=timeout, headers=headers) def geojsonToFeatureCollection(geojson: dict) -> dict: @@ -49,36 +49,30 @@ def chunks(arr, n_objects): ] -def query_osm(changeset_ids: list, changeset_results): +def query_osmcha(changeset_ids: list, changeset_results): """Get data from changesetId.""" id_string = ",".join(map(str, changeset_ids)) - url = OSM_API_LINK + f"changesets?changesets={id_string}" + url = OSMCHA_API_LINK + f"changesets/?ids={id_string}" + logger.info(url) + logger.info(len(changeset_ids)) response = retry_get(url) if response.status_code != 200: - err = f"osm request failed: {response.status_code}" + err = f"osmcha request failed: {response.status_code}" logger.warning(f"{err}") logger.warning(response.json()) raise CustomError(err) - tree = ElementTree.fromstring(response.content) - - for changeset in tree.iter("changeset"): - id = changeset.attrib["id"] - username = remove_troublesome_chars(changeset.attrib["user"]) - userid = changeset.attrib["uid"] - comment = created_by = None - for tag in changeset.iter("tag"): - if tag.attrib["k"] == "comment": - comment = tag.attrib["v"] - if tag.attrib["k"] == "created_by": - created_by = tag.attrib["v"] - - changeset_results[int(id)] = { - "username": remove_troublesome_chars(username), - "userid": userid, - "comment": remove_troublesome_chars(comment), - "created_by": remove_troublesome_chars(created_by), + response = response.json() + logger.info(response) + for feature in response["features"]: + logger.info(feature) + changeset_results[int(feature["id"])] = { + "username": remove_troublesome_chars(feature["properties"]["user"]), + "userid": feature["properties"]["uid"], + "comment": remove_troublesome_chars(feature["properties"]["comment"]), + "editor": remove_troublesome_chars(feature["properties"]["editor"]), } + return changeset_results @@ -111,15 +105,16 @@ def remove_noise_and_add_user_info(json: dict) -> dict: logger.info( f"""{len_osm} changesets will be queried in roughly {batches} batches""" ) - chunk_list = chunks(list(changeset_results.keys()), 100) + chunk_list = chunks(list(changeset_results.keys()), 50) for i, subset in enumerate(chunk_list): - changeset_results = query_osm(subset, changeset_results) + changeset_results = query_osmcha(subset, changeset_results) progress = round(100 * ((i + 1) / len(chunk_list)), 1) logger.info(f"finished query {i+1}/{len(chunk_list)}, {progress}") for feature in json["features"]: - changeset = changeset_results[feature["properties"]["changesetId"]] - for attribute_name in ["username", "comment", "created_by", "userid"]: + changeset = changeset_results[int(feature["properties"]["changesetId"])] + logger.warn(changeset) + for attribute_name in ["username", "comment", "editor", "userid"]: feature["properties"][attribute_name] = changeset[attribute_name] logger.info("finished filtering and adding extra info") From f31ef4a49ef8a2b38030e16e5d5fbffdaad42e3b Mon Sep 17 00:00:00 2001 From: Hagellach37 Date: Wed, 22 Dec 2021 11:03:18 +0100 Subject: [PATCH 56/62] fix bug in tutorial create tasks --- .../project_types/arbitrary_geometry/tutorial.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/tutorial.py b/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/tutorial.py index 35ed74ce0..483aad20c 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/tutorial.py +++ b/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/tutorial.py @@ -57,13 +57,7 @@ def create_tutorial_tasks(self): ) for group_id, item in raw_groups.items(): group = Group(self, groupId=101) - group.create_tasks( - item["feature_ids"], - item["feature_geometries"], - item["center_points"], - item["reference"], - item["screen"], - ) + group.create_tasks(item["feature_ids"], item["features"]) for task in group.tasks: logger.info(task) From cff45787cd7cce44f5550cb703bae8a4793fd8ed Mon Sep 17 00:00:00 2001 From: Hagellach37 Date: Wed, 22 Dec 2021 19:13:14 +0100 Subject: [PATCH 57/62] fix order or tutorial screens --- .../arbitrary_geometry/grouping_functions.py | 14 ++++++++++++-- .../arbitrary_geometry/tutorial.py | 18 ++++++++++++++++-- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/grouping_functions.py b/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/grouping_functions.py index 2987f3648..9ad219ab2 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/grouping_functions.py +++ b/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/grouping_functions.py @@ -1,7 +1,7 @@ import json -def group_input_geometries(input_geometries_file, group_size): +def group_input_geometries(input_geometries_file, group_size, tutorial=False): """ The function to create groups of input geometries using the given size (number of features) per group @@ -12,6 +12,8 @@ def group_input_geometries(input_geometries_file, group_size): the path to the GeoJSON file containing the input geometries group_size : int the maximum number of features per group + tutorial: boolean + if this function is called to create the grouping within a tutorial Returns ------- @@ -44,7 +46,15 @@ def group_input_geometries(input_geometries_file, group_size): # we use a new id here based on the count # since we are not sure that GetFID returns unique values - groups[group_id_string]["feature_ids"].append(feature_count) + if not tutorial: + groups[group_id_string]["feature_ids"].append(feature_count) + else: + # In the tutorial the feature id is defined by the "screen" attribute. + # We do this so that we can sort by the feature id later and + # get the screens displayed in the right order on the app. + groups[group_id_string]["feature_ids"].append( + feature["properties"]["screen"] + ) groups[group_id_string]["features"].append(feature) return groups diff --git a/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/tutorial.py b/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/tutorial.py index 483aad20c..0cc6b891f 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/tutorial.py +++ b/mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/tutorial.py @@ -1,5 +1,7 @@ import json +import numpy as np + from mapswipe_workers.definitions import DATA_PATH, logger from mapswipe_workers.project_types.arbitrary_geometry import grouping_functions as g from mapswipe_workers.project_types.arbitrary_geometry.group import Group @@ -53,11 +55,23 @@ def create_tutorial_tasks(self): """Create the tasks dict based on provided examples in geojson file.""" raw_groups = g.group_input_geometries( - self.inputGeometries, len(self.tutorial_tasks) + 1 + self.inputGeometries, + len(self.tutorial_tasks["features"]) + 1, + tutorial=True, ) + for group_id, item in raw_groups.items(): group = Group(self, groupId=101) - group.create_tasks(item["feature_ids"], item["features"]) + + # Make sure that we sort the tasks. + # For the tutorial the feature_id represents the number of the screen. + # The group_input_geometries functions doesn't return + # the screens in the right order. + sorted_idx = np.array(item["feature_ids"]).argsort() + sorted_feature_ids = [item["feature_ids"][x] for x in sorted_idx] + sorted_features = [item["features"][x] for x in sorted_idx] + + group.create_tasks(sorted_feature_ids, sorted_features) for task in group.tasks: logger.info(task) From 94d1d069b49c23335ac415774b1d81fffb6e29df Mon Sep 17 00:00:00 2001 From: Hagellach37 Date: Thu, 23 Dec 2021 12:23:56 +0100 Subject: [PATCH 58/62] change g parameter in bing imagery --- mapswipe_workers/mapswipe_workers/definitions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapswipe_workers/mapswipe_workers/definitions.py b/mapswipe_workers/mapswipe_workers/definitions.py index 62daebdd5..af46f5202 100644 --- a/mapswipe_workers/mapswipe_workers/definitions.py +++ b/mapswipe_workers/mapswipe_workers/definitions.py @@ -68,7 +68,7 @@ IMAGE_URLS = { "bing": ( "https://ecn.t0.tiles.virtualearth.net" - + "/tiles/a{quad_key}.jpeg?g=7505&mkt=en-US&token={key}" + + "/tiles/a{quad_key}.jpeg?g=1&token={key}" ), "mapbox": ( "https://d.tiles.mapbox.com" From d4f21cda057d523ce7534a6eadd5ffec4a89edb6 Mon Sep 17 00:00:00 2001 From: Jochen Date: Wed, 29 Dec 2021 18:16:16 +0100 Subject: [PATCH 59/62] query from osm if not available in osmcha --- .../mapswipe_workers/definitions.py | 1 + .../mapswipe_workers/utils/api_calls.py | 74 ++++++++++++++++--- 2 files changed, 63 insertions(+), 12 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/definitions.py b/mapswipe_workers/mapswipe_workers/definitions.py index 526e85365..a620296ed 100644 --- a/mapswipe_workers/mapswipe_workers/definitions.py +++ b/mapswipe_workers/mapswipe_workers/definitions.py @@ -13,6 +13,7 @@ LOGGING_FILE_PATH = os.path.join(DATA_PATH, "mapswipe_workers.log") OHSOME_API_LINK = "https://api.ohsome.org/v1/" +OSM_API_LINK = "https://www.openstreetmap.org/api/0.6/" OSMCHA_API_LINK = "https://osmcha.org/api/v1/" OSMCHA_API_KEY = os.environ["OSMCHA_API_KEY"] diff --git a/mapswipe_workers/mapswipe_workers/utils/api_calls.py b/mapswipe_workers/mapswipe_workers/utils/api_calls.py index 345e4215e..5ca01722b 100644 --- a/mapswipe_workers/mapswipe_workers/utils/api_calls.py +++ b/mapswipe_workers/mapswipe_workers/utils/api_calls.py @@ -1,9 +1,12 @@ +from xml.etree import ElementTree + import requests from requests.adapters import HTTPAdapter from requests.packages.urllib3.util.retry import Retry from mapswipe_workers.definitions import ( OHSOME_API_LINK, + OSM_API_LINK, OSMCHA_API_KEY, OSMCHA_API_LINK, CustomError, @@ -21,13 +24,16 @@ def remove_troublesome_chars(string: str): return string -def retry_get(url, retries=3, timeout=4): +def retry_get(url, retries=3, timeout=4, to_osmcha: bool = False): """Retry a query for a variable amount of tries.""" retry = Retry(total=retries) with requests.Session() as session: session.mount("https://", HTTPAdapter(max_retries=retry)) - headers = {"Authorization": OSMCHA_API_KEY} - return session.get(url, timeout=timeout, headers=headers) + if to_osmcha: + headers = {"Authorization": OSMCHA_API_KEY} + return session.get(url, timeout=timeout, headers=headers) + else: + return session.get(url, timeout=timeout) def geojsonToFeatureCollection(geojson: dict) -> dict: @@ -54,18 +60,14 @@ def query_osmcha(changeset_ids: list, changeset_results): id_string = ",".join(map(str, changeset_ids)) url = OSMCHA_API_LINK + f"changesets/?ids={id_string}" - logger.info(url) - logger.info(len(changeset_ids)) - response = retry_get(url) + response = retry_get(url, to_osmcha=True) if response.status_code != 200: err = f"osmcha request failed: {response.status_code}" logger.warning(f"{err}") logger.warning(response.json()) raise CustomError(err) response = response.json() - logger.info(response) for feature in response["features"]: - logger.info(feature) changeset_results[int(feature["id"])] = { "username": remove_troublesome_chars(feature["properties"]["user"]), "userid": feature["properties"]["uid"], @@ -76,9 +78,45 @@ def query_osmcha(changeset_ids: list, changeset_results): return changeset_results +def query_osm(changeset_ids: list, changeset_results): + """Get data from changesetId.""" + id_string = ",".join(map(str, changeset_ids)) + + url = OSM_API_LINK + f"changesets?changesets={id_string}" + response = retry_get(url) + if response.status_code != 200: + err = f"osm request failed: {response.status_code}" + logger.warning(f"{err}") + logger.warning(response.json()) + raise CustomError(err) + tree = ElementTree.fromstring(response.content) + + for changeset in tree.iter("changeset"): + id = changeset.attrib["id"] + username = remove_troublesome_chars(changeset.attrib["user"]) + userid = changeset.attrib["uid"] + comment = created_by = None + for tag in changeset.iter("tag"): + if tag.attrib["k"] == "comment": + comment = tag.attrib["v"] + if tag.attrib["k"] == "created_by": + created_by = tag.attrib["v"] + + changeset_results[int(id)] = { + "username": remove_troublesome_chars(username), + "userid": userid, + "comment": remove_troublesome_chars(comment), + "editor": remove_troublesome_chars(created_by), + } + return changeset_results + + def remove_noise_and_add_user_info(json: dict) -> dict: """Delete unwanted information from properties.""" logger.info("starting filtering and adding extra info") + batch_size = 100 + + # remove noise changeset_results = {} missing_rows = { @@ -100,20 +138,32 @@ def remove_noise_and_add_user_info(json: dict) -> dict: changeset_results[new_properties["changesetId"]] = None feature["properties"] = new_properties + # add info len_osm = len(changeset_results.keys()) - batches = int(len(changeset_results.keys()) / 100) + 1 + batches = int(len(changeset_results.keys()) / batch_size) + 1 logger.info( - f"""{len_osm} changesets will be queried in roughly {batches} batches""" + f"""{len_osm} changesets will be queried in roughly {batches} batches from osmCHA""" # noqa E501 ) - chunk_list = chunks(list(changeset_results.keys()), 50) + + chunk_list = chunks(list(changeset_results.keys()), batch_size) for i, subset in enumerate(chunk_list): changeset_results = query_osmcha(subset, changeset_results) progress = round(100 * ((i + 1) / len(chunk_list)), 1) logger.info(f"finished query {i+1}/{len(chunk_list)}, {progress}") + missing_ids = [i for i, v in changeset_results.items() if v is None] + chunk_list = chunks(missing_ids, batch_size) + batches = int(len(missing_ids) / batch_size) + 1 + logger.info( + f"""{len(missing_ids)} changesets where missing from osmCHA and are now queried via osmAPI in {batches} batches""" # noqa E501 + ) + for i, subset in enumerate(chunk_list): + changeset_results = query_osm(subset, changeset_results) + progress = round(100 * ((i + 1) / len(chunk_list)), 1) + logger.info(f"finished query {i+1}/{len(chunk_list)}, {progress}") + for feature in json["features"]: changeset = changeset_results[int(feature["properties"]["changesetId"])] - logger.warn(changeset) for attribute_name in ["username", "comment", "editor", "userid"]: feature["properties"][attribute_name] = changeset[attribute_name] From 64ac66f0e8ded3b8312490a9e9829c9ce25757fa Mon Sep 17 00:00:00 2001 From: Jochen Stier <44268882+ElJocho@users.noreply.github.com> Date: Wed, 5 Jan 2022 12:59:54 +0100 Subject: [PATCH 60/62] Update api_calls.py --- mapswipe_workers/mapswipe_workers/utils/api_calls.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/api_calls.py b/mapswipe_workers/mapswipe_workers/utils/api_calls.py index 5ca01722b..7f20ea33f 100644 --- a/mapswipe_workers/mapswipe_workers/utils/api_calls.py +++ b/mapswipe_workers/mapswipe_workers/utils/api_calls.py @@ -165,7 +165,10 @@ def remove_noise_and_add_user_info(json: dict) -> dict: for feature in json["features"]: changeset = changeset_results[int(feature["properties"]["changesetId"])] for attribute_name in ["username", "comment", "editor", "userid"]: - feature["properties"][attribute_name] = changeset[attribute_name] + if attribute_name == "userid": + feature["properties"][attribute_name] = int(changeset[attribute_name]) + else: + feature["properties"][attribute_name] = changeset[attribute_name] logger.info("finished filtering and adding extra info") if any(x > 0 for x in missing_rows.values()): From e9c634799e9cc65db8792a16a3b932b55c3b223f Mon Sep 17 00:00:00 2001 From: Hagellach37 Date: Fri, 7 Jan 2022 11:55:44 +0100 Subject: [PATCH 61/62] add osmcha api key to environment and github actions workflow --- .github/workflows/actions.yml | 1 + docker-compose.yaml | 3 +++ example.env | 3 +++ 3 files changed, 7 insertions(+) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 2c4757e79..c1a967740 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -59,6 +59,7 @@ jobs: POSTGRES_PASSWORD: test POSTGRES_USER: test POSTGRES_DB: test + OSMCHA_API_KEY: ${{ secrets.OSMCHA_API_KEY }} run: | docker-compose run mapswipe_workers_creation python -m unittest discover --verbose --start-directory tests/unittests/ docker-compose run mapswipe_workers_creation python -m unittest discover --verbose --start-directory tests/integration/ \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index 96f9e1338..3e0b59f26 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -73,6 +73,7 @@ services: SLACK_TOKEN: '${SLACK_TOKEN}' SLACK_CHANNEL: '${SLACK_CHANNEL}' SENTRY_DSN: '${SENTRY_DSN}' + OSMCHA_API_KEY: '${OSMCHA_API_KEY}' depends_on: - postgres command: mapswipe_workers --verbose run --analysis_type=creation --schedule --time_interval=5 @@ -107,6 +108,7 @@ services: SLACK_TOKEN: '${SLACK_TOKEN}' SLACK_CHANNEL: '${SLACK_CHANNEL}' SENTRY_DSN: '${SENTRY_DSN}' + OSMCHA_API_KEY: '${OSMCHA_API_KEY}' depends_on: - postgres command: mapswipe_workers --verbose run --analysis_type=generate-stats --schedule --time_interval=60 @@ -141,6 +143,7 @@ services: SLACK_TOKEN: '${SLACK_TOKEN}' SLACK_CHANNEL: '${SLACK_CHANNEL}' SENTRY_DSN: '${SENTRY_DSN}' + OSMCHA_API_KEY: '${OSMCHA_API_KEY}' depends_on: - postgres command: mapswipe_workers --verbose run --analysis_type=firebase-to-postgres --schedule --time_interval=2 diff --git a/example.env b/example.env index 48ab420d8..a81591918 100644 --- a/example.env +++ b/example.env @@ -28,3 +28,6 @@ SLACK_CHANNEL= # sentry configuration SENTRY_DSN= + +# osmcha configuration +OSMCHA_API_KEY= From 6b4d0fcf8ad80b47349074fd76444f2f225e7f52 Mon Sep 17 00:00:00 2001 From: Hagellach37 Date: Fri, 7 Jan 2022 12:24:35 +0100 Subject: [PATCH 62/62] small fix in header. --- mapswipe_workers/mapswipe_workers/utils/api_calls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/api_calls.py b/mapswipe_workers/mapswipe_workers/utils/api_calls.py index 7f20ea33f..9fbdf67fe 100644 --- a/mapswipe_workers/mapswipe_workers/utils/api_calls.py +++ b/mapswipe_workers/mapswipe_workers/utils/api_calls.py @@ -30,7 +30,7 @@ def retry_get(url, retries=3, timeout=4, to_osmcha: bool = False): with requests.Session() as session: session.mount("https://", HTTPAdapter(max_retries=retry)) if to_osmcha: - headers = {"Authorization": OSMCHA_API_KEY} + headers = {"Authorization": f"Token {OSMCHA_API_KEY}"} return session.get(url, timeout=timeout, headers=headers) else: return session.get(url, timeout=timeout)