Fares v2

pom.xml

@@ -272,7 +272,7 @@
             <groupId>com.github.ibi-group</groupId>
             <artifactId>gtfs-lib</artifactId>
             <!-- Latest dev build on jitpack.io -->
-            <version>e9de250</version>
+            <version>518fcc2f494e89ee378d5c5a5f691208d53f7f2d</version>
             <!-- Exclusions added in order to silence SLF4J warnings about multiple bindings:
                 http://www.slf4j.org/codes.html#multiple_bindings
             -->

src/main/java/com/conveyal/datatools/manager/DataManager.java

@@ -204,19 +204,29 @@ static void registerRoutes() throws IOException {
             SnapshotController.register(EDITOR_API_PREFIX);
             EditorLockController.register(EDITOR_API_PREFIX);
 
+            new EditorControllerImpl(EDITOR_API_PREFIX, Table.AREAS, DataManager.GTFS_DATA_SOURCE);
             new EditorControllerImpl(EDITOR_API_PREFIX, Table.AGENCY, DataManager.GTFS_DATA_SOURCE);
             new EditorControllerImpl(EDITOR_API_PREFIX, Table.ATTRIBUTIONS, DataManager.GTFS_DATA_SOURCE);
             new EditorControllerImpl(EDITOR_API_PREFIX, Table.CALENDAR, DataManager.GTFS_DATA_SOURCE);
             // NOTE: fare_attributes controller handles updates to nested table fare_rules.
             new EditorControllerImpl(EDITOR_API_PREFIX, Table.FARE_ATTRIBUTES, DataManager.GTFS_DATA_SOURCE);
+            new EditorControllerImpl(EDITOR_API_PREFIX, Table.FARE_LEG_RULES, DataManager.GTFS_DATA_SOURCE);
+            new EditorControllerImpl(EDITOR_API_PREFIX, Table.FARE_MEDIAS, DataManager.GTFS_DATA_SOURCE);
+            new EditorControllerImpl(EDITOR_API_PREFIX, Table.FARE_PRODUCTS, DataManager.GTFS_DATA_SOURCE);
+            new EditorControllerImpl(EDITOR_API_PREFIX, Table.FARE_TRANSFER_RULES, DataManager.GTFS_DATA_SOURCE);
             new EditorControllerImpl(EDITOR_API_PREFIX, Table.FEED_INFO, DataManager.GTFS_DATA_SOURCE);
+            new EditorControllerImpl(EDITOR_API_PREFIX, Table.NETWORKS, DataManager.GTFS_DATA_SOURCE);
             new EditorControllerImpl(EDITOR_API_PREFIX, Table.ROUTES, DataManager.GTFS_DATA_SOURCE);
+            new EditorControllerImpl(EDITOR_API_PREFIX, Table.ROUTE_NETWORKS, DataManager.GTFS_DATA_SOURCE);
             // NOTE: Patterns controller handles updates to nested tables shapes, pattern stops, and frequencies.
             new EditorControllerImpl(EDITOR_API_PREFIX, Table.PATTERNS, DataManager.GTFS_DATA_SOURCE);
             new EditorControllerImpl(EDITOR_API_PREFIX, Table.SCHEDULE_EXCEPTIONS, DataManager.GTFS_DATA_SOURCE);
             new EditorControllerImpl(EDITOR_API_PREFIX, Table.STOPS, DataManager.GTFS_DATA_SOURCE);
+            new EditorControllerImpl(EDITOR_API_PREFIX, Table.STOP_AREAS, DataManager.GTFS_DATA_SOURCE);
+            new EditorControllerImpl(EDITOR_API_PREFIX, Table.TIME_FRAMES, DataManager.GTFS_DATA_SOURCE);
             new EditorControllerImpl(EDITOR_API_PREFIX, Table.TRANSLATIONS, DataManager.GTFS_DATA_SOURCE);
             new EditorControllerImpl(EDITOR_API_PREFIX, Table.TRIPS, DataManager.GTFS_DATA_SOURCE);
+
             // TODO: Add transfers.txt controller?
         }
 

src/test/java/com/conveyal/datatools/editor/controllers/api/EditorControllerTest.java

@@ -14,6 +14,7 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.restassured.http.Method;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
@@ -27,10 +28,13 @@
 import java.io.IOException;
 import java.sql.SQLException;
 import java.util.Date;
+import java.util.Objects;
 import java.util.stream.Stream;
 
 import static com.conveyal.datatools.TestUtils.assertThatSqlCountQueryYieldsExpectedCount;
+import static com.conveyal.datatools.TestUtils.createFeedVersion;
 import static com.conveyal.datatools.TestUtils.createFeedVersionFromGtfsZip;
+import static com.conveyal.datatools.TestUtils.zipFolderFiles;
 import static com.conveyal.datatools.manager.auth.Auth0Users.USERS_API_PATH;
 import static com.conveyal.datatools.manager.controllers.api.UserController.TEST_AUTH0_DOMAIN;
 import static io.restassured.RestAssured.given;
@@ -45,8 +49,10 @@ public class EditorControllerTest extends UnitTest {
     private static Project project;
     private static FeedSource feedSource;
     private static FeedSource feedSourceCascadeDelete;
+    private static FeedSource faresV2FeedSource;
     private static FeedVersion feedVersion;
     private static FeedVersion feedVersionCascadeDelete;
+    private static FeedVersion faresV2Version;
     private static final ObjectMapper mapper = new ObjectMapper();
 
     /**
@@ -72,12 +78,18 @@ public static void setUp() throws Exception {
         feedSourceCascadeDelete.projectId = project.id;
         Persistence.feedSources.create(feedSourceCascadeDelete);
 
+        faresV2FeedSource = new FeedSource("FaresV2");
+        faresV2FeedSource.projectId = project.id;
+        Persistence.feedSources.create(faresV2FeedSource);
+
         feedVersion = createFeedVersionFromGtfsZip(feedSource, "bart_old.zip");
         feedVersionCascadeDelete = createFeedVersionFromGtfsZip(feedSourceCascadeDelete, "bart_old.zip");
+        faresV2Version = createFeedVersion(faresV2FeedSource, zipFolderFiles("fake-agency-with-fares-v2"));
 
         // Create and run snapshot jobs
-        crateAndRunSnapshotJob(feedVersion.name, feedSource.id, feedVersion.namespace);
-        crateAndRunSnapshotJob(feedVersionCascadeDelete.name, feedSourceCascadeDelete.id, feedVersionCascadeDelete.namespace);
+        createAndRunSnapshotJob(feedVersion.name, feedSource.id, feedVersion.namespace);
+        createAndRunSnapshotJob(feedVersionCascadeDelete.name, feedSourceCascadeDelete.id, feedVersionCascadeDelete.namespace);
+        createAndRunSnapshotJob(faresV2Version.name, faresV2FeedSource.id, faresV2Version.namespace);
         LOG.info("{} setup completed in {} ms", EditorControllerTest.class.getSimpleName(), System.currentTimeMillis() - startTime);
     }
 
@@ -86,15 +98,22 @@ public static void tearDown() {
         project.delete();
         feedSource.delete();
         feedSourceCascadeDelete.delete();
+        faresV2FeedSource.delete();
     }
 
     /**
      * Create and run a snapshot job in the current thread (so tests do not run until this is complete).
      */
-    private static void crateAndRunSnapshotJob(String feedVersionName, String feedSourceId, String namespace) {
+    private static void createAndRunSnapshotJob(String feedVersionName, String feedSourceId, String namespace) {
         Snapshot snapshot = new Snapshot("Snapshot of " + feedVersionName, feedSourceId, namespace);
         CreateSnapshotJob createSnapshotJob =
-            new CreateSnapshotJob(Auth0UserProfile.createTestAdminUser(), snapshot, true, false, false);
+            new CreateSnapshotJob(
+                Auth0UserProfile.createTestAdminUser(),
+                snapshot,
+                true,
+                false,
+                false
+            );
         createSnapshotJob.run();
     }
 
@@ -111,14 +130,7 @@ private static Stream<Arguments> createPatchTableTests() {
      */
     @ParameterizedTest
     @MethodSource("createPatchTableTests")
-    public void canPatchTableTests(
-        String field,
-        String entity,
-        int expectedCount,
-        String graphQLQueryFile,
-        String table
-    ) throws IOException {
-
+    void canPatchTableTests(String field, String entity, int expectedCount, String graphQLQueryFile, String table) throws IOException {
         LOG.info("Making patch {} request", table);
         String value = "NEW";
         ObjectNode jsonBody = mapper.createObjectNode();
@@ -138,7 +150,7 @@ public void canPatchTableTests(
      * Make sure the patch table endpoint can patch stops conditionally with query.
      */
     @Test
-    public void canPatchStopsConditionally() throws IOException {
+    void canPatchStopsConditionally() throws IOException {
         LOG.info("Making conditional patch stops request");
         ObjectNode jsonBody = mapper.createObjectNode();
         String field = "stop_desc";
@@ -187,13 +199,7 @@ void canCascadeDeleteStop() throws IOException, SQLException {
             stopId,
             feedVersionCascadeDelete.feedSourceId
         );
-        String response = given()
-            .port(DataManager.PORT)
-            .delete(path)
-            .then()
-            .extract()
-            .response()
-            .asString();
+        String response = makeRequest(Method.DELETE, path, "");
         JsonNode json = mapper.readTree(response);
         assertEquals(OK_200, json.get("code").asInt());
 
@@ -204,20 +210,63 @@ void canCascadeDeleteStop() throws IOException, SQLException {
     }
 
     /**
-     * Perform patch table request on the feed source ID with the requested query and patch JSON. A null query will
-     * apply the patch JSON to the entire table.
+     * Confirm that a fare product can be created, updated and deleted.
      */
-    private static int patchTableRequest(String entity, String feedId, String query, JsonNode oatchJSON) throws IOException {
-        String path = String.format("/api/editor/secure/%s?feedId=%s", entity, feedId);
-        if (query != null) path += "&" + query;
-        String response = given()
+    @Test
+    void canCreateUpdateAndDeleteFareProduct() throws IOException {
+        String fareProductId = "AERIAL_TRAM_ROUND_TRIP";
+        String fareProductIdUpdated = "AERIAL_TRAM_ROUND_TRIP_UPDATED";
+        String urlPrefix = "/api/editor/secure/fareproduct";
+        String urlSuffix = String.format("?feedId=%s&sessionId=test", faresV2Version.feedSourceId);
+        String payload =
+            "{" +
+                "\"fare_product_id\":\"" + fareProductId + "\"," +
+                "\"fare_product_name\":\"Portland Aerial Tram Single Round Trip\"," +
+                "\"fare_media_id\":\"1\"," +
+                "\"amount\":\"13.5\"," +
+                "\"currency\":\"USD\"" +
+            "}";
+
+        // Create.
+        String response = makeRequest(Method.POST, String.format("%s%s", urlPrefix, urlSuffix), payload);
+        JsonNode json = mapper.readTree(response);
+        String id = json.get("id").asText();
+        assertEquals(fareProductId, json.get("fare_product_id").asText());
+
+        // Update.
+        payload = payload.replace(fareProductId, fareProductIdUpdated);
+        response = makeRequest(Method.PUT, String.format("%s/%s%s", urlPrefix, id, urlSuffix), payload);
+        json = mapper.readTree(response);
+        assertEquals(fareProductIdUpdated, json.get("fare_product_id").asText());
+
+        // Delete.
+        response = makeRequest(Method.DELETE, String.format("%s/%s%s", urlPrefix, id, urlSuffix), "");
+        json = mapper.readTree(response);
+        assertEquals(200, json.get("code").asInt());
+    }
+
+    /**
+     * Make request and return the response.
+     */
+    private static String makeRequest(Method method, String path, Object payload) {
+        return given()
             .port(DataManager.PORT)
-            .body(oatchJSON)
-            .patch(path)
+            .body(payload)
+            .request(method, path)
             .then()
             .extract()
             .response()
             .asString();
+    }
+
+    /**
+     * Perform patch table request on the feed source ID with the requested query and patch JSON. A null query will
+     * apply the patch JSON to the entire table.
+     */
+    private static int patchTableRequest(String entity, String feedId, String query, JsonNode patchJSON) throws IOException {
+        String path = String.format("/api/editor/secure/%s?feedId=%s", entity, feedId);
+        if (query != null) path += "&" + query;
+        String response = makeRequest(Method.PATCH, path, patchJSON);
         JsonNode json = mapper.readTree(response);
         return json.get("count").asInt();
     }
@@ -230,16 +279,9 @@ private static JsonNode graphqlQuery (String namespace, String graphQLQueryFile)
         ObjectNode variables = mapper.createObjectNode();
         variables.put("namespace", namespace);
         graphQLBody.set("variables", variables);
-        String query = IOUtils.toString(EditorControllerTest.class.getClassLoader().getResourceAsStream(graphQLQueryFile));
+        String query = IOUtils.toString(Objects.requireNonNull(EditorControllerTest.class.getClassLoader().getResourceAsStream(graphQLQueryFile)));
         graphQLBody.put("query", query);
-        String graphQLString = given()
-            .port(DataManager.PORT)
-            .body(graphQLBody)
-            .post("api/manager/secure/gtfs/graphql")
-            .then()
-            .extract()
-            .response()
-            .asString();
+        String graphQLString = makeRequest(Method.POST, "api/manager/secure/gtfs/graphql", graphQLBody);
         return mapper.readTree(graphQLString);
     }
 

src/test/java/com/conveyal/datatools/manager/jobs/validation/FaresV2ValidationTest.java

@@ -0,0 +1,98 @@
+package com.conveyal.datatools.manager.jobs.validation;
+
+import com.conveyal.datatools.DatatoolsTest;
+import com.conveyal.datatools.manager.auth.Auth0Connection;
+import com.conveyal.datatools.manager.models.FeedSource;
+import com.conveyal.datatools.manager.models.FeedVersion;
+import com.conveyal.datatools.manager.models.Project;
+import com.conveyal.datatools.manager.persistence.Persistence;
+import org.bson.Document;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Stream;
+
+import static com.conveyal.datatools.TestUtils.createFeedVersion;
+import static com.conveyal.datatools.TestUtils.zipFolderFiles;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class FaresV2ValidationTest {
+
+    private static Project project;
+    private static FeedVersion faresV2Version;
+
+    @BeforeAll
+    public static void setUp() throws IOException {
+        // Start server if it isn't already running.
+        DatatoolsTest.setUp();
+        Auth0Connection.setAuthDisabled(true);
+        // Create a project, feed sources, and feed versions to merge.
+        project = new Project();
+        project.name = String.format("Test %s", new Date());
+        Persistence.projects.create(project);
+
+        FeedSource faresV2FeedSource = new FeedSource("fares-v2");
+        faresV2FeedSource.projectId = project.id;
+        Persistence.feedSources.create(faresV2FeedSource);
+        faresV2Version = createFeedVersion(
+            faresV2FeedSource,
+            zipFolderFiles("fake-agency-with-fares-v2")
+        );
+        Persistence.feedVersions.replace(faresV2Version.id, faresV2Version);
+    }
+
+    @AfterAll
+    static void tearDown() {
+        Auth0Connection.setAuthDisabled(Auth0Connection.getDefaultAuthDisabled());
+        project.delete();
+    }
+
+    @Test
+    void canValidateFareV2Files() {
+        Document summary = (Document) faresV2Version.mobilityDataResult.get("summary");
+        List<String> files = summary.getList("files", String.class);
+        assertTrue(files.contains("areas.txt"));
+        assertTrue(files.contains("fare_leg_rules.txt"));
+        assertTrue(files.contains("fare_media.txt"));
+        assertTrue(files.contains("fare_products.txt"));
+        assertTrue(files.contains("fare_transfer_rules.txt"));
+        assertTrue(files.contains("networks.txt"));
+        assertTrue(files.contains("route_networks.txt"));
+        assertTrue(files.contains("stop_areas.txt"));
+        assertTrue(files.contains("timeframes.txt"));
+    }
+
+    /**
+     * This is not an exhaustive test, more of a sanity check that MobilityData can detect errors in fares v2 data.
+     * https://gtfs-validator.mobilitydata.org/rules.html
+     */
+    @ParameterizedTest
+    @MethodSource("createValidationErrorChecks")
+    void canDetectValidationErrors(Set<String> codes, String expectedCode) {
+        assertTrue(codes.contains(expectedCode));
+    }
+
+    private static Stream<Arguments> createValidationErrorChecks() {
+        ArrayList<Document> notices = (ArrayList<Document>) faresV2Version.mobilityDataResult.get("notices");
+        Set<String> codes = new HashSet<>();
+        for (Document notice : notices) {
+            codes.add(notice.getString("code"));
+        }
+        return Stream.of(
+            Arguments.of(codes, "fare_transfer_rule_duration_limit_type_without_duration_limit"),
+            Arguments.of(codes, "fare_transfer_rule_duration_limit_without_type"),
+            Arguments.of(codes, "fare_transfer_rule_missing_transfer_count"),
+            Arguments.of(codes, "fare_transfer_rule_with_forbidden_transfer_count")
+        );
+    }
+}

src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-fares-v2/agency.txt

@@ -0,0 +1,2 @@
+agency_id,agency_name,agency_url,agency_lang,agency_phone,agency_email,agency_timezone,agency_fare_url,agency_branding_url
+1,Fake Transit,http://www.fake-agency.com,,,,America/Los_Angeles,,

src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-fares-v2/areas.txt

@@ -0,0 +1,40 @@
+area_id,area_name
+area_bl,Blue Line
+area_bl_airport,Blue Line - Airport Station
+area_cf_zone_buzzards,CapeFLYER - Wareham/Buzzards Bay/Bourne
+area_cf_zone_hyannis,CapeFLYER - Hyannis
+area_commuter_rail_porter_zone_1a,Commuter Rail Zone 1A
+area_commuter_rail_sumner_tunnel_zone_1a,Commuter Rail Zone 1A
+area_commuter_rail_zone_1,Commuter Rail Zone 1
+area_commuter_rail_zone_10,Commuter Rail Zone 10
+area_commuter_rail_zone_1a,Commuter Rail Zone 1A
+area_commuter_rail_zone_2,Commuter Rail Zone 2
+area_commuter_rail_zone_3,Commuter Rail Zone 3
+area_commuter_rail_zone_4,Commuter Rail Zone 4
+area_commuter_rail_zone_5,Commuter Rail Zone 5
+area_commuter_rail_zone_6,Commuter Rail Zone 6
+area_commuter_rail_zone_7,Commuter Rail Zone 7
+area_commuter_rail_zone_8,Commuter Rail Zone 8
+area_commuter_rail_zone_9,Commuter Rail Zone 9
+area_fairmount_line_zone_1a,Commuter Rail Zone 1A - Fairmount Line
+area_gl_govt_ctr,Green Line - Government Center
+area_green_b_west_of_kenmore,Green Line B - West of Kenmore
+area_green_c_west_of_kenmore,Green Line C - West of Kenmore
+area_green_e_west_of_symphony,Green Line E - West of Symphony
+area_m_ashmont_mattapan,Mattapan Trolley - Ashmont and Mattapan
+area_ol_state,Orange Line - State
+area_red_south_station,Red Line - South Station
+area_route_354_downtown,Route 354 - Downtown
+area_route_354_outside_downtown,Route 354 - Outside Downtown
+area_route_426_downtown,Route 426 - Downtown
+area_route_426_outside_downtown,Route 426 - Outside Downtown
+area_route_450_downtown,Route 450 - Downtown
+area_route_450_outside_downtown,Route 450 - Outside Downtown
+area_sl3_north_of_airport,Silver Line - North of Airport Station
+area_sl_airport,Silver Line - Airport Station
+area_sl_courthouse,Silver Line - Courthouse
+area_sl_logan_terminal,Silver Line - Airport Terminals
+area_sl_silver_line_way,Silver Line - Silver Line Way
+area_sl_south_station,Silver Line - South Station
+area_sl_world_trade_center,Silver Line - World Trade Center
+area_ss_commuter_rail_zone_1a,Commuter Rail Zone 1A - South Station

src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-fares-v2/calendar.txt

@@ -0,0 +1,2 @@
+service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,start_date,end_date
+04100312-8fe1-46a5-a9f2-556f39478f57,1,1,1,1,1,1,1,20170915,20170917