From 38d34538ecd98571d6cfc6cfdd540d110929b1f1 Mon Sep 17 00:00:00 2001 From: sfomuseumbot Date: Mon, 8 Jul 2024 14:00:36 -0700 Subject: [PATCH 1/4] improved reporting and error checking building current --- campus/campus.go | 11 ++- campus/current.go | 174 +++++++++++++++++++++++++++++----------------- 2 files changed, 117 insertions(+), 68 deletions(-) diff --git a/campus/campus.go b/campus/campus.go index 704e2bc..1635a8d 100644 --- a/campus/campus.go +++ b/campus/campus.go @@ -3,12 +3,11 @@ package campus // type Campus is a lightweight data structure to represent the SFO campus with pointers its descendants. type Campus struct { - WhosOnFirstId int64 `json:"id"` - SFOId string `json:"sfo:id"` - Complex *Complex `json:"complex"` - Garages []*Garage `json:"garages"` - // Buildings []*Building `json:"buildings,omitempty"` - PublicArt []*PublicArt `json:"buildings,omitempty"` + WhosOnFirstId int64 `json:"id"` + SFOId string `json:"sfo:id"` + Complex *Complex `json:"complex"` + Garages []*Garage `json:"garages"` + PublicArt []*PublicArt `json:"buildings,omitempty"` } // type Garage is a lightweight data structure to represent garages at SFO with pointers its descendants. diff --git a/campus/current.go b/campus/current.go index a48427c..d2fed9b 100644 --- a/campus/current.go +++ b/campus/current.go @@ -29,18 +29,19 @@ package campus import ( "context" "database/sql" - "errors" "fmt" - "log" + "log/slog" "github.com/aaronland/go-sqlite" aa_database "github.com/aaronland/go-sqlite/database" "github.com/tidwall/gjson" "github.com/whosonfirst/go-whosonfirst-sqlite-features-index" "github.com/whosonfirst/go-whosonfirst-sqlite-features/tables" - sql_index "github.com/whosonfirst/go-whosonfirst-sqlite-index/v3" + sql_index "github.com/whosonfirst/go-whosonfirst-sqlite-index/v3" ) +// SFO Terminal Complex (1954~ to 1963~) +// https://millsfield.sfomuseum.org/buildings/1159396329/ const FIRST_SFO int64 = 1159396329 // MostRecentComplexWithIterator will return a `Complex` instance representing the most recent relationships of the SFO terminal complex @@ -52,13 +53,13 @@ func MostRecentComplexWithIterator(ctx context.Context, iterator_uri string, pat db, err := newWhosOnFirstDatabaseFromIterator(ctx, dsn, iterator_uri, paths...) if err != nil { - return nil, err + return nil, fmt.Errorf("Failed to derive new WOF database, %w", err) } sql_db, err := db.Conn() if err != nil { - return nil, err + return nil, fmt.Errorf("Failed to create database connection, %w", err) } return FindMostRecentComplexWithDatabase(ctx, sql_db) @@ -69,13 +70,13 @@ func FindMostRecentComplexWithDatabase(ctx context.Context, db *sql.DB) (*Comple sfo_id, err := findMostRecentComplexID(ctx, db, FIRST_SFO) if err != nil { - return nil, err + return nil, fmt.Errorf("Failed to derive most recent complex ID, %w", err) } terminals, err := findTerminals(ctx, db, sfo_id) if err != nil { - return nil, err + return nil, fmt.Errorf("Failed to derive terminals for complex %d, %w", sfo_id, err) } c := &Complex{ @@ -89,10 +90,12 @@ func FindMostRecentComplexWithDatabase(ctx context.Context, db *sql.DB) (*Comple func findTerminals(ctx context.Context, db *sql.DB, sfo_id int64) ([]*Terminal, error) { + slog.Info("Find terminals", "parent id", sfo_id) + terminal_ids, err := findChildIDs(ctx, db, sfo_id, "terminal") if err != nil { - log.Fatalf("Failed to find terminals, %v", err) + return nil, fmt.Errorf("Failed to find any child records (terminals) for %d, %v", sfo_id, err) } terminals := make([]*Terminal, len(terminal_ids)) @@ -102,19 +105,19 @@ func findTerminals(ctx context.Context, db *sql.DB, sfo_id int64) ([]*Terminal, commonareas, err := findCommonAreas(ctx, db, t_id) if err != nil { - return nil, err + return nil, fmt.Errorf("Failed to derive common areas for %d, %w", t_id, err) } boardingareas, err := findBoardingAreas(ctx, db, t_id) if err != nil { - return nil, err + return nil, fmt.Errorf("Failed to derive boarding areas for %d, %w", t_id, err) } - t_body, err := loadFeatureWithDB(ctx, db, t_id) + t_body, err := loadFeatureWithDBAndChecks(ctx, db, t_id) if err != nil { - return nil, err + return nil, fmt.Errorf("Failed to load feature for %d, %w", t_id, err) } var sfoid string @@ -130,7 +133,7 @@ func findTerminals(ctx context.Context, db *sql.DB, sfo_id int64) ([]*Terminal, rsp := gjson.GetBytes(t_body, "properties.sfomuseum:terminal_id") if !rsp.Exists() { - return nil, errors.New("Unable to find terminal_id") + return nil, fmt.Errorf("Missing properties.sfomuseum:terminal_id property for terminal %d", t_id) } switch rsp.String() { @@ -143,10 +146,12 @@ func findTerminals(ctx context.Context, db *sql.DB, sfo_id int64) ([]*Terminal, case "T3": sfoid = "400" // gis.BUILDING_T3 default: - return nil, fmt.Errorf("Unrecognized terminal_id '%s'", rsp.String()) + return nil, fmt.Errorf("Unrecognized terminal_id '%s' for %d", rsp.String(), t_id) } } + slog.Info("Add terminal", "parent id", sfoid, "id", t_id) + terminal := &Terminal{ WhosOnFirstId: t_id, SFOId: sfoid, @@ -168,10 +173,12 @@ func findTerminals(ctx context.Context, db *sql.DB, sfo_id int64) ([]*Terminal, func findObservationDecks(ctx context.Context, db *sql.DB, t_id int64) ([]*ObservationDeck, error) { + slog.Info("Find observation decks", "parent id", t_id) + deck_ids, err := findChildIDs(ctx, db, t_id, "observationdeck") if err != nil { - log.Fatalf("Failed to find observation decks, %v", err) + return nil, fmt.Errorf("Failed to find any child records (observation decks) for %d, %v", t_id, err) } decks := make([]*ObservationDeck, len(deck_ids)) @@ -181,19 +188,19 @@ func findObservationDecks(ctx context.Context, db *sql.DB, t_id int64) ([]*Obser galleries, err := findGalleries(ctx, db, d_id) if err != nil { - return nil, err + return nil, fmt.Errorf("Failed to derive galleries for observation deck %d, %w", d_id, err) } publicart, err := findPublicArt(ctx, db, d_id) if err != nil { - return nil, err + return nil, fmt.Errorf("Failed to derive public art for observation deck %d, %w", d_id, err) } - d_body, err := loadFeatureWithDB(ctx, db, d_id) + d_body, err := loadFeatureWithDBAndChecks(ctx, db, d_id) if err != nil { - return nil, err + return nil, fmt.Errorf("Failed to load feature for observation deck %d, %w", d_id, err) } var sfoid string @@ -201,11 +208,13 @@ func findObservationDecks(ctx context.Context, db *sql.DB, t_id int64) ([]*Obser rsp := gjson.GetBytes(d_body, "properties.sfo:id") if !rsp.Exists() { - return nil, errors.New("Unable to find sfo:id") + return nil, fmt.Errorf("Unable to find sfo:id for WOF record, %d", d_id) } sfoid = rsp.String() + slog.Info("Add observation deck", "sfo id", sfoid, "parent id", t_id, "id", d_id) + deck := &ObservationDeck{ WhosOnFirstId: d_id, SFOId: sfoid, @@ -225,12 +234,14 @@ func findObservationDecks(ctx context.Context, db *sql.DB, t_id int64) ([]*Obser return decks, nil } -func findCommonAreas(ctx context.Context, db *sql.DB, t_id int64) ([]*CommonArea, error) { +func findCommonAreas(ctx context.Context, db *sql.DB, parent_id int64) ([]*CommonArea, error) { - commonarea_ids, err := findChildIDs(ctx, db, t_id, "commonarea") + slog.Info("Find common areas", "parent", parent_id) + + commonarea_ids, err := findChildIDs(ctx, db, parent_id, "commonarea") if err != nil { - log.Fatalf("Failed to find common areas, %v", err) + return nil, fmt.Errorf("Failed to find any child records (common areas) for %d, %v", parent_id, err) } commonareas := make([]*CommonArea, len(commonarea_ids)) @@ -240,37 +251,37 @@ func findCommonAreas(ctx context.Context, db *sql.DB, t_id int64) ([]*CommonArea gates, err := findGates(ctx, db, c_id) if err != nil { - return nil, err + return nil, fmt.Errorf("Failed to derive gates for common area %d, %w", c_id, err) } checkpoints, err := findCheckpoints(ctx, db, c_id) if err != nil { - return nil, err + return nil, fmt.Errorf("Failed to derive gates for check points %d, %w", c_id, err) } galleries, err := findGalleries(ctx, db, c_id) if err != nil { - return nil, err + return nil, fmt.Errorf("Failed to derive gates for galleries %d, %w", c_id, err) } observation_decks, err := findObservationDecks(ctx, db, c_id) if err != nil { - return nil, err + return nil, fmt.Errorf("Failed to derive observation decks for galleries %d, %w", c_id, err) } publicart, err := findPublicArt(ctx, db, c_id) if err != nil { - return nil, err + return nil, fmt.Errorf("Failed to derive public art for galleries %d, %w", c_id, err) } - c_body, err := loadFeatureWithDB(ctx, db, c_id) + c_body, err := loadFeatureWithDBAndChecks(ctx, db, c_id) if err != nil { - return nil, err + return nil, fmt.Errorf("Failed to load feature %d, %w", c_id, err) } var sfoid string @@ -286,7 +297,7 @@ func findCommonAreas(ctx context.Context, db *sql.DB, t_id int64) ([]*CommonArea rsp := gjson.GetBytes(c_body, "properties.sfo:building_id") if !rsp.Exists() { - return nil, errors.New("Unable to find sfo:building_id") + return nil, fmt.Errorf("Unable to find sfo:building_id for %d", c_id) } switch rsp.String() { @@ -299,7 +310,7 @@ func findCommonAreas(ctx context.Context, db *sql.DB, t_id int64) ([]*CommonArea case "T3", "400": sfoid = "400CAD" // gis.COMMONAREA_T3_DEPARTURES default: - return nil, fmt.Errorf("Unrecognized sfo:id '%s'", rsp.String()) + return nil, fmt.Errorf("Unrecognized sfo:id '%s' for %d", rsp.String(), c_id) } } @@ -336,10 +347,12 @@ func findCommonAreas(ctx context.Context, db *sql.DB, t_id int64) ([]*CommonArea func findBoardingAreas(ctx context.Context, db *sql.DB, id int64) ([]*BoardingArea, error) { + slog.Info("Find boarding areas", "parent", id) + boardingarea_ids, err := findChildIDs(ctx, db, id, "boardingarea") if err != nil { - log.Fatalf("Failed to find boarding areas, %v", err) + return nil, fmt.Errorf("Failed to find any child records (boarding areas areas) for %d, %v", id, err) } boardingareas := make([]*BoardingArea, len(boardingarea_ids)) @@ -349,37 +362,37 @@ func findBoardingAreas(ctx context.Context, db *sql.DB, id int64) ([]*BoardingAr gates, err := findGates(ctx, db, b_id) if err != nil { - return nil, err + return nil, fmt.Errorf("Failed to derive gates for boarding area %d, %w", b_id, err) } checkpoints, err := findCheckpoints(ctx, db, b_id) if err != nil { - return nil, err + return nil, fmt.Errorf("Failed to derive check points for boarding area %d, %w", b_id, err) } galleries, err := findGalleries(ctx, db, b_id) if err != nil { - return nil, err + return nil, fmt.Errorf("Failed to derive galleries for boarding area %d, %w", b_id, err) } publicart, err := findPublicArt(ctx, db, b_id) if err != nil { - return nil, err + return nil, fmt.Errorf("Failed to derive public art for boarding area %d, %w", b_id, err) } observation_decks, err := findObservationDecks(ctx, db, b_id) if err != nil { - return nil, err + return nil, fmt.Errorf("Failed to derive observation decks for boarding area %d, %w", b_id, err) } - b_body, err := loadFeatureWithDB(ctx, db, b_id) + b_body, err := loadFeatureWithDBAndChecks(ctx, db, b_id) if err != nil { - return nil, err + return nil, fmt.Errorf("Failed to load feature for %d, %w", b_id, err) } var sfoid string @@ -395,7 +408,7 @@ func findBoardingAreas(ctx context.Context, db *sql.DB, id int64) ([]*BoardingAr rsp := gjson.GetBytes(b_body, "properties.sfo:building_id") if !rsp.Exists() { - return nil, errors.New("Missing sfo:building_id") + return nil, fmt.Errorf("Missing sfo:building_id for boarding area %d", b_id) } sfoid = rsp.String() @@ -433,22 +446,24 @@ func findBoardingAreas(ctx context.Context, db *sql.DB, id int64) ([]*BoardingAr } -func findGates(ctx context.Context, db *sql.DB, id int64) ([]*Gate, error) { +func findGates(ctx context.Context, db *sql.DB, parent_id int64) ([]*Gate, error) { - gate_ids, err := findChildIDs(ctx, db, id, "gate") + slog.Info("Find gates", "parent", parent_id) + + gate_ids, err := findChildIDs(ctx, db, parent_id, "gate") if err != nil { - return nil, err + return nil, fmt.Errorf("Failed to find any child records (gates) for %d, %w", parent_id, err) } gates := make([]*Gate, len(gate_ids)) for idx, g_id := range gate_ids { - g_body, err := loadFeatureWithDB(ctx, db, g_id) + g_body, err := loadFeatureWithDBAndChecks(ctx, db, g_id) if err != nil { - return nil, err + return nil, fmt.Errorf("Failed to load feature for date %d, %w", g_id, err) } var sfoid string @@ -464,12 +479,14 @@ func findGates(ctx context.Context, db *sql.DB, id int64) ([]*Gate, error) { rsp := gjson.GetBytes(g_body, "properties.wof:name") if !rsp.Exists() { - return nil, errors.New("Missing wof:name") + return nil, fmt.Errorf("Missing wof:name for %d", g_id) } sfoid = rsp.String() } + slog.Info("Add gate", "sfo id", sfoid, "parent_id", parent_id, "id", g_id) + g := &Gate{ WhosOnFirstId: g_id, SFOId: sfoid, @@ -481,22 +498,24 @@ func findGates(ctx context.Context, db *sql.DB, id int64) ([]*Gate, error) { return gates, nil } -func findCheckpoints(ctx context.Context, db *sql.DB, id int64) ([]*Checkpoint, error) { +func findCheckpoints(ctx context.Context, db *sql.DB, parent_id int64) ([]*Checkpoint, error) { - checkpoint_ids, err := findChildIDs(ctx, db, id, "checkpoint") + slog.Info("Find check points", "parent id", parent_id) + + checkpoint_ids, err := findChildIDs(ctx, db, parent_id, "checkpoint") if err != nil { - return nil, err + return nil, fmt.Errorf("Failed to find any child records (checkpoints) for %d, %w", parent_id, err) } checkpoints := make([]*Checkpoint, len(checkpoint_ids)) for idx, cp_id := range checkpoint_ids { - cp_body, err := loadFeatureWithDB(ctx, db, cp_id) + cp_body, err := loadFeatureWithDBAndChecks(ctx, db, cp_id) if err != nil { - return nil, err + return nil, fmt.Errorf("Failed to load feature for %d, %w", cp_id, err) } var sfoid string @@ -504,11 +523,13 @@ func findCheckpoints(ctx context.Context, db *sql.DB, id int64) ([]*Checkpoint, rsp := gjson.GetBytes(cp_body, "properties.sfo:id") if !rsp.Exists() { - return nil, errors.New("Missing sfo:id") + return nil, fmt.Errorf("Missing sfo:id for %d", cp_id) } sfoid = rsp.String() + slog.Info("Add checkpoint", "sfo id", sfoid, "parent id", parent_id, "id", cp_id) + cp := &Checkpoint{ WhosOnFirstId: cp_id, SFOId: sfoid, @@ -522,20 +543,22 @@ func findCheckpoints(ctx context.Context, db *sql.DB, id int64) ([]*Checkpoint, func findGalleries(ctx context.Context, db *sql.DB, parent_id int64) ([]*Gallery, error) { + slog.Info("Find galleries", "parent id", parent_id) + gallery_ids, err := findChildIDs(ctx, db, parent_id, "gallery") if err != nil { - return nil, err + return nil, fmt.Errorf("Failed to find any child records (galleries) for %d, %w", parent_id, err) } galleries := make([]*Gallery, len(gallery_ids)) for idx, g_id := range gallery_ids { - g_body, err := loadFeatureWithDB(ctx, db, g_id) + g_body, err := loadFeatureWithDBAndChecks(ctx, db, g_id) if err != nil { - return nil, err + return nil, fmt.Errorf("Failed load feature for gallery %d, %w", g_id, err) } var sfomid string @@ -551,12 +574,14 @@ func findGalleries(ctx context.Context, db *sql.DB, parent_id int64) ([]*Gallery rsp := gjson.GetBytes(g_body, "properties.sfomuseum:gallery_id") if !rsp.Exists() { - return nil, errors.New("Missing sfomuseum:gallery_id") + return nil, fmt.Errorf("Missing sfomuseum:gallery_id property for gallery %d", g_id) } sfomid = rsp.String() } + slog.Info("Add gallery", "sfo id", sfomid, "parent id", parent_id, "id", g_id) + g := &Gallery{ WhosOnFirstId: g_id, SFOId: sfomid, @@ -570,20 +595,22 @@ func findGalleries(ctx context.Context, db *sql.DB, parent_id int64) ([]*Gallery func findPublicArt(ctx context.Context, db *sql.DB, parent_id int64) ([]*PublicArt, error) { + slog.Info("Find public art", "parent id", parent_id) + publicart_ids, err := findChildIDs(ctx, db, parent_id, "publicart") if err != nil { - return nil, err + return nil, fmt.Errorf("Failed to find any child records (public art) for %d, %w", parent_id, err) } publicarts := make([]*PublicArt, len(publicart_ids)) for idx, p_id := range publicart_ids { - p_body, err := loadFeatureWithDB(ctx, db, p_id) + p_body, err := loadFeatureWithDBAndChecks(ctx, db, p_id) if err != nil { - return nil, err + return nil, fmt.Errorf("Failed to load feature for public art %d, %w", p_id, err) } var sfomid string @@ -599,12 +626,14 @@ func findPublicArt(ctx context.Context, db *sql.DB, parent_id int64) ([]*PublicA rsp := gjson.GetBytes(p_body, "properties.sfomuseum:object_id") if !rsp.Exists() { - return nil, errors.New("Missing sfomuseum:object_id") + return nil, fmt.Errorf("Missing sfomuseum:object_id property for public art %d, %w", p_id, err) } sfomid = rsp.String() } + slog.Info("Add public art", "sfo id", sfomid, "parent id", parent_id, "id", p_id) + g := &PublicArt{ WhosOnFirstId: p_id, SFOId: sfomid, @@ -640,7 +669,7 @@ func findMostRecentComplexID(ctx context.Context, db *sql.DB, id int64) (int64, err := rows.Scan(&superseded_by) if err != nil { - return -1, err + return -1, fmt.Errorf("Failed to scan row, %w", err) } possible = append(possible, superseded_by) @@ -708,6 +737,27 @@ func findChildIDs(ctx context.Context, db *sql.DB, parent_id int64, placetype st return children, nil } +func loadFeatureWithDBAndChecks(ctx context.Context, db *sql.DB, id int64) ([]byte, error) { + + body, err := loadFeatureWithDB(ctx, db, id) + + if err != nil { + return nil, fmt.Errorf("Failed to load feature for record %d, %w", id, err) + } + + current_rsp := gjson.GetBytes(body, "properties.mz:is_current") + + if !current_rsp.Exists() { + return nil, fmt.Errorf("Missing properties.mz:is_current property for record %d", id) + } + + if current_rsp.Int() != 1 { + return nil, fmt.Errorf("Unexpected mz:is_current property (%d) for record %d", current_rsp.Int(), id) + } + + return body, nil +} + func loadFeatureWithDB(ctx context.Context, db *sql.DB, id int64) ([]byte, error) { q := "SELECT body FROM geojson WHERE id = ?" From d81d0b728c64e65b106c37a26945be07ee72e0fd Mon Sep 17 00:00:00 2001 From: sfomuseumbot Date: Mon, 8 Jul 2024 15:18:00 -0700 Subject: [PATCH 2/4] don't store null elements --- campus/current.go | 143 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 111 insertions(+), 32 deletions(-) diff --git a/campus/current.go b/campus/current.go index d2fed9b..3be7ebb 100644 --- a/campus/current.go +++ b/campus/current.go @@ -98,9 +98,9 @@ func findTerminals(ctx context.Context, db *sql.DB, sfo_id int64) ([]*Terminal, return nil, fmt.Errorf("Failed to find any child records (terminals) for %d, %v", sfo_id, err) } - terminals := make([]*Terminal, len(terminal_ids)) + terminals := make([]*Terminal, 0) - for idx, t_id := range terminal_ids { + for _, t_id := range terminal_ids { commonareas, err := findCommonAreas(ctx, db, t_id) @@ -120,6 +120,10 @@ func findTerminals(ctx context.Context, db *sql.DB, sfo_id int64) ([]*Terminal, return nil, fmt.Errorf("Failed to load feature for %d, %w", t_id, err) } + if t_body == nil { + continue + } + var sfoid string rsp := gjson.GetBytes(t_body, "properties.sfo:id") @@ -150,7 +154,11 @@ func findTerminals(ctx context.Context, db *sql.DB, sfo_id int64) ([]*Terminal, } } - slog.Info("Add terminal", "parent id", sfoid, "id", t_id) + name_rsp := gjson.GetBytes(t_body, "properties.wof:name") + inception_rsp := gjson.GetBytes(t_body, "properties.edtf:inception") + cessation_rsp := gjson.GetBytes(t_body, "properties.edtf:cessation") + + slog.Info("Add terminal", "sfo id", sfoid, "id", t_id, "name", name_rsp.String(), "inception", inception_rsp.String(), "cessation", cessation_rsp.String()) terminal := &Terminal{ WhosOnFirstId: t_id, @@ -165,7 +173,7 @@ func findTerminals(ctx context.Context, db *sql.DB, sfo_id int64) ([]*Terminal, terminal.BoardingAreas = boardingareas } - terminals[idx] = terminal + terminals = append(terminals, terminal) } return terminals, nil @@ -181,9 +189,9 @@ func findObservationDecks(ctx context.Context, db *sql.DB, t_id int64) ([]*Obser return nil, fmt.Errorf("Failed to find any child records (observation decks) for %d, %v", t_id, err) } - decks := make([]*ObservationDeck, len(deck_ids)) + decks := make([]*ObservationDeck, 0) - for idx, d_id := range deck_ids { + for _, d_id := range deck_ids { galleries, err := findGalleries(ctx, db, d_id) @@ -203,6 +211,10 @@ func findObservationDecks(ctx context.Context, db *sql.DB, t_id int64) ([]*Obser return nil, fmt.Errorf("Failed to load feature for observation deck %d, %w", d_id, err) } + if d_body == nil { + continue + } + var sfoid string rsp := gjson.GetBytes(d_body, "properties.sfo:id") @@ -213,7 +225,11 @@ func findObservationDecks(ctx context.Context, db *sql.DB, t_id int64) ([]*Obser sfoid = rsp.String() - slog.Info("Add observation deck", "sfo id", sfoid, "parent id", t_id, "id", d_id) + name_rsp := gjson.GetBytes(d_body, "properties.wof:name") + inception_rsp := gjson.GetBytes(d_body, "properties.edtf:inception") + cessation_rsp := gjson.GetBytes(d_body, "properties.edtf:cessation") + + slog.Info("Add observation deck", "sfo id", sfoid, "parent id", t_id, "id", d_id, "name", name_rsp.String(), "inception", inception_rsp.String(), "cessation", cessation_rsp.String()) deck := &ObservationDeck{ WhosOnFirstId: d_id, @@ -228,7 +244,7 @@ func findObservationDecks(ctx context.Context, db *sql.DB, t_id int64) ([]*Obser deck.PublicArt = publicart } - decks[idx] = deck + decks = append(decks, deck) } return decks, nil @@ -244,9 +260,9 @@ func findCommonAreas(ctx context.Context, db *sql.DB, parent_id int64) ([]*Commo return nil, fmt.Errorf("Failed to find any child records (common areas) for %d, %v", parent_id, err) } - commonareas := make([]*CommonArea, len(commonarea_ids)) + commonareas := make([]*CommonArea, 0) - for idx, c_id := range commonarea_ids { + for _, c_id := range commonarea_ids { gates, err := findGates(ctx, db, c_id) @@ -284,6 +300,10 @@ func findCommonAreas(ctx context.Context, db *sql.DB, parent_id int64) ([]*Commo return nil, fmt.Errorf("Failed to load feature %d, %w", c_id, err) } + if c_body == nil { + continue + } + var sfoid string rsp := gjson.GetBytes(c_body, "properties.sfo:id") @@ -314,6 +334,12 @@ func findCommonAreas(ctx context.Context, db *sql.DB, parent_id int64) ([]*Commo } } + name_rsp := gjson.GetBytes(c_body, "properties.wof:name") + inception_rsp := gjson.GetBytes(c_body, "properties.edtf:inception") + cessation_rsp := gjson.GetBytes(c_body, "properties.edtf:cessation") + + slog.Info("Add common area", "sfo id", sfoid, "id", c_id, "name", name_rsp.String(), "inception", inception_rsp.String(), "cessation", cessation_rsp.String()) + area := &CommonArea{ WhosOnFirstId: c_id, SFOId: sfoid, @@ -339,7 +365,7 @@ func findCommonAreas(ctx context.Context, db *sql.DB, parent_id int64) ([]*Commo area.ObservationDecks = observation_decks } - commonareas[idx] = area + commonareas = append(commonareas, area) } return commonareas, nil @@ -355,9 +381,9 @@ func findBoardingAreas(ctx context.Context, db *sql.DB, id int64) ([]*BoardingAr return nil, fmt.Errorf("Failed to find any child records (boarding areas areas) for %d, %v", id, err) } - boardingareas := make([]*BoardingArea, len(boardingarea_ids)) + boardingareas := make([]*BoardingArea, 0) - for idx, b_id := range boardingarea_ids { + for _, b_id := range boardingarea_ids { gates, err := findGates(ctx, db, b_id) @@ -395,6 +421,10 @@ func findBoardingAreas(ctx context.Context, db *sql.DB, id int64) ([]*BoardingAr return nil, fmt.Errorf("Failed to load feature for %d, %w", b_id, err) } + if b_body == nil { + continue + } + var sfoid string rsp := gjson.GetBytes(b_body, "properties.sfo:id") @@ -414,6 +444,12 @@ func findBoardingAreas(ctx context.Context, db *sql.DB, id int64) ([]*BoardingAr sfoid = rsp.String() } + name_rsp := gjson.GetBytes(b_body, "properties.wof:name") + inception_rsp := gjson.GetBytes(b_body, "properties.edtf:inception") + cessation_rsp := gjson.GetBytes(b_body, "properties.edtf:cessation") + + slog.Info("Add boardinarea", "sfo id", sfoid, "id", b_id, "name", name_rsp.String(), "inception", inception_rsp.String(), "cessation", cessation_rsp.String()) + area := &BoardingArea{ WhosOnFirstId: b_id, SFOId: sfoid, @@ -439,7 +475,7 @@ func findBoardingAreas(ctx context.Context, db *sql.DB, id int64) ([]*BoardingAr area.ObservationDecks = observation_decks } - boardingareas[idx] = area + boardingareas = append(boardingareas, area) } return boardingareas, nil @@ -456,9 +492,9 @@ func findGates(ctx context.Context, db *sql.DB, parent_id int64) ([]*Gate, error return nil, fmt.Errorf("Failed to find any child records (gates) for %d, %w", parent_id, err) } - gates := make([]*Gate, len(gate_ids)) + gates := make([]*Gate, 0) - for idx, g_id := range gate_ids { + for _, g_id := range gate_ids { g_body, err := loadFeatureWithDBAndChecks(ctx, db, g_id) @@ -466,6 +502,10 @@ func findGates(ctx context.Context, db *sql.DB, parent_id int64) ([]*Gate, error return nil, fmt.Errorf("Failed to load feature for date %d, %w", g_id, err) } + if g_body == nil { + continue + } + var sfoid string rsp := gjson.GetBytes(g_body, "properties.sfo:id") @@ -485,14 +525,18 @@ func findGates(ctx context.Context, db *sql.DB, parent_id int64) ([]*Gate, error sfoid = rsp.String() } - slog.Info("Add gate", "sfo id", sfoid, "parent_id", parent_id, "id", g_id) + name_rsp := gjson.GetBytes(g_body, "properties.wof:name") + inception_rsp := gjson.GetBytes(g_body, "properties.edtf:inception") + cessation_rsp := gjson.GetBytes(g_body, "properties.edtf:cessation") + + slog.Info("Add gate", "sfo id", sfoid, "parent_id", parent_id, "id", g_id, "name", name_rsp.String(), "inception", inception_rsp.String(), "cessation", cessation_rsp.String()) g := &Gate{ WhosOnFirstId: g_id, SFOId: sfoid, } - gates[idx] = g + gates = append(gates, g) } return gates, nil @@ -508,9 +552,9 @@ func findCheckpoints(ctx context.Context, db *sql.DB, parent_id int64) ([]*Check return nil, fmt.Errorf("Failed to find any child records (checkpoints) for %d, %w", parent_id, err) } - checkpoints := make([]*Checkpoint, len(checkpoint_ids)) + checkpoints := make([]*Checkpoint, 0) - for idx, cp_id := range checkpoint_ids { + for _, cp_id := range checkpoint_ids { cp_body, err := loadFeatureWithDBAndChecks(ctx, db, cp_id) @@ -518,6 +562,10 @@ func findCheckpoints(ctx context.Context, db *sql.DB, parent_id int64) ([]*Check return nil, fmt.Errorf("Failed to load feature for %d, %w", cp_id, err) } + if cp_body == nil { + continue + } + var sfoid string rsp := gjson.GetBytes(cp_body, "properties.sfo:id") @@ -528,14 +576,18 @@ func findCheckpoints(ctx context.Context, db *sql.DB, parent_id int64) ([]*Check sfoid = rsp.String() - slog.Info("Add checkpoint", "sfo id", sfoid, "parent id", parent_id, "id", cp_id) + name_rsp := gjson.GetBytes(cp_body, "properties.wof:name") + inception_rsp := gjson.GetBytes(cp_body, "properties.edtf:inception") + cessation_rsp := gjson.GetBytes(cp_body, "properties.edtf:cessation") + + slog.Info("Add checkpoint", "sfo id", sfoid, "parent id", parent_id, "id", cp_id, "name", name_rsp.String(), "inception", inception_rsp.String(), "cessation", cessation_rsp.String()) cp := &Checkpoint{ WhosOnFirstId: cp_id, SFOId: sfoid, } - checkpoints[idx] = cp + checkpoints = append(checkpoints, cp) } return checkpoints, nil @@ -551,9 +603,9 @@ func findGalleries(ctx context.Context, db *sql.DB, parent_id int64) ([]*Gallery return nil, fmt.Errorf("Failed to find any child records (galleries) for %d, %w", parent_id, err) } - galleries := make([]*Gallery, len(gallery_ids)) + galleries := make([]*Gallery, 0) - for idx, g_id := range gallery_ids { + for _, g_id := range gallery_ids { g_body, err := loadFeatureWithDBAndChecks(ctx, db, g_id) @@ -561,6 +613,10 @@ func findGalleries(ctx context.Context, db *sql.DB, parent_id int64) ([]*Gallery return nil, fmt.Errorf("Failed load feature for gallery %d, %w", g_id, err) } + if g_body == nil { + continue + } + var sfomid string rsp := gjson.GetBytes(g_body, "properties.sfomuseum:map_id") @@ -580,14 +636,18 @@ func findGalleries(ctx context.Context, db *sql.DB, parent_id int64) ([]*Gallery sfomid = rsp.String() } - slog.Info("Add gallery", "sfo id", sfomid, "parent id", parent_id, "id", g_id) + name_rsp := gjson.GetBytes(g_body, "properties.wof:name") + inception_rsp := gjson.GetBytes(g_body, "properties.edtf:inception") + cessation_rsp := gjson.GetBytes(g_body, "properties.edtf:cessation") + + slog.Info("Add gallery", "sfo id", sfomid, "parent id", parent_id, "id", g_id, "name", name_rsp.String(), "inception", inception_rsp.String(), "cessation", cessation_rsp.String()) g := &Gallery{ WhosOnFirstId: g_id, SFOId: sfomid, } - galleries[idx] = g + galleries = append(galleries, g) } return galleries, nil @@ -603,9 +663,9 @@ func findPublicArt(ctx context.Context, db *sql.DB, parent_id int64) ([]*PublicA return nil, fmt.Errorf("Failed to find any child records (public art) for %d, %w", parent_id, err) } - publicarts := make([]*PublicArt, len(publicart_ids)) + publicarts := make([]*PublicArt, 0) - for idx, p_id := range publicart_ids { + for _, p_id := range publicart_ids { p_body, err := loadFeatureWithDBAndChecks(ctx, db, p_id) @@ -613,6 +673,10 @@ func findPublicArt(ctx context.Context, db *sql.DB, parent_id int64) ([]*PublicA return nil, fmt.Errorf("Failed to load feature for public art %d, %w", p_id, err) } + if p_body == nil { + continue + } + var sfomid string rsp := gjson.GetBytes(p_body, "properties.sfomuseum:map_id") @@ -632,14 +696,18 @@ func findPublicArt(ctx context.Context, db *sql.DB, parent_id int64) ([]*PublicA sfomid = rsp.String() } - slog.Info("Add public art", "sfo id", sfomid, "parent id", parent_id, "id", p_id) + name_rsp := gjson.GetBytes(p_body, "properties.wof:name") + inception_rsp := gjson.GetBytes(p_body, "properties.edtf:inception") + cessation_rsp := gjson.GetBytes(p_body, "properties.edtf:cessation") - g := &PublicArt{ + slog.Info("Add public art", "sfo id", sfomid, "parent id", parent_id, "id", p_id, "name", name_rsp.String(), "inception", inception_rsp.String(), "cessation", cessation_rsp.String()) + + pa := &PublicArt{ WhosOnFirstId: p_id, SFOId: sfomid, } - publicarts[idx] = g + publicarts = append(publicarts, pa) } return publicarts, nil @@ -745,6 +813,16 @@ func loadFeatureWithDBAndChecks(ctx context.Context, db *sql.DB, id int64) ([]by return nil, fmt.Errorf("Failed to load feature for record %d, %w", id, err) } + name_rsp := gjson.GetBytes(body, "properties.wof:name") + inception_rsp := gjson.GetBytes(body, "properties.edtf:inception") + cessation_rsp := gjson.GetBytes(body, "properties.edtf:cessation") + + deprecated_rsp := gjson.GetBytes(body, "properties.edtf:deprecated") + + if deprecated_rsp.Exists() && deprecated_rsp.String() != "" { + return nil, nil + } + current_rsp := gjson.GetBytes(body, "properties.mz:is_current") if !current_rsp.Exists() { @@ -752,7 +830,8 @@ func loadFeatureWithDBAndChecks(ctx context.Context, db *sql.DB, id int64) ([]by } if current_rsp.Int() != 1 { - return nil, fmt.Errorf("Unexpected mz:is_current property (%d) for record %d", current_rsp.Int(), id) + slog.Warn("Unexpected mz:is_current property", "id", id, "mz:is_current", current_rsp.Int(), "name", name_rsp.String(), "inception", inception_rsp.String(), "cessation", cessation_rsp.String()) + return nil, nil } return body, nil From 5ac9ae792d66ad9d71ae80a8f202ee7f7b83337b Mon Sep 17 00:00:00 2001 From: sfomuseumbot Date: Mon, 8 Jul 2024 16:20:34 -0700 Subject: [PATCH 3/4] start blocking out code to dump current complex as geojson --- campus/geojson.go | 94 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 campus/geojson.go diff --git a/campus/geojson.go b/campus/geojson.go new file mode 100644 index 0000000..cab54ac --- /dev/null +++ b/campus/geojson.go @@ -0,0 +1,94 @@ +package campus + +import ( + "context" + + "github.com/whosonfirst/go-reader" +) + +func ToGeoJSON(ctx context.Context, c *Complex, r reader.Reader) error { + + terminal_ids := make([]int64, 0) + boardingarea_ids := make([]int64, 0) + commonarea_ids := make([]int64, 0) + observationdeck_ids := make([]int64, 0) + gallery_ids := make([]int64, 0) + checkpoint_ids := make([]int64, 0) + gate_ids := make([]int64, 0) + publicart_ids := make([]int64, 0) + + for _, t := range c.Terminals { + + terminal_ids = append(terminal_ids, t.WhosOnFirstId) + + for _, b := range t.BoardingAreas { + boardingarea_ids = append(boardingarea_ids, b.WhosOnFirstId) + + for _, g := range b.Gates { + gate_ids = append(gate_ids, g.WhosOnFirstId) + } + + for _, c := range b.Checkpoints { + checkpoint_ids = append(checkpoint_ids, c.WhosOnFirstId) + } + + for _, g := range b.Galleries { + gallery_ids = append(gallery_ids, g.WhosOnFirstId) + } + + for _, p := range b.PublicArt { + publicart_ids = append(publicart_ids, p.WhosOnFirstId) + } + + for _, o := range b.ObservationDecks { + + observationdeck_ids = append(observationdeck_ids, o.WhosOnFirstId) + + for _, g := range o.Galleries { + gallery_ids = append(gallery_ids, g.WhosOnFirstId) + } + + for _, p := range o.PublicArt { + publicart_ids = append(publicart_ids, p.WhosOnFirstId) + } + + } + } + + for _, c := range t.CommonAreas { + + commonarea_ids = append(commonarea_ids, c.WhosOnFirstId) + + for _, g := range c.Gates { + gate_ids = append(gate_ids, g.WhosOnFirstId) + } + + for _, c := range c.Checkpoints { + checkpoint_ids = append(checkpoint_ids, c.WhosOnFirstId) + } + + for _, g := range c.Galleries { + gallery_ids = append(gallery_ids, g.WhosOnFirstId) + } + + for _, p := range c.PublicArt { + publicart_ids = append(publicart_ids, p.WhosOnFirstId) + } + + for _, o := range c.ObservationDecks { + + observationdeck_ids = append(observationdeck_ids, o.WhosOnFirstId) + + for _, g := range o.Galleries { + gallery_ids = append(gallery_ids, g.WhosOnFirstId) + } + + for _, p := range o.PublicArt { + publicart_ids = append(publicart_ids, p.WhosOnFirstId) + } + } + } + } + + return nil +} From 84881b77ce672ae2b365a936ceba3a773a95bbef Mon Sep 17 00:00:00 2001 From: sfomuseumbot Date: Mon, 8 Jul 2024 17:26:08 -0700 Subject: [PATCH 4/4] ComplexToGeoJSONLayers --- campus/geojson.go | 44 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/campus/geojson.go b/campus/geojson.go index cab54ac..87e4448 100644 --- a/campus/geojson.go +++ b/campus/geojson.go @@ -2,11 +2,14 @@ package campus import ( "context" + "fmt" + "github.com/paulmach/orb/geojson" "github.com/whosonfirst/go-reader" + wof_reader "github.com/whosonfirst/go-whosonfirst-reader" ) -func ToGeoJSON(ctx context.Context, c *Complex, r reader.Reader) error { +func ComplexToGeoJSONLayers(ctx context.Context, c *Complex, r reader.Reader) (map[string]*geojson.FeatureCollection, error) { terminal_ids := make([]int64, 0) boardingarea_ids := make([]int64, 0) @@ -90,5 +93,42 @@ func ToGeoJSON(ctx context.Context, c *Complex, r reader.Reader) error { } } - return nil + feature_ids := map[string][]int64{ + "terminals": terminal_ids, + "boardingareas": boardingarea_ids, + "commonareas": commonarea_ids, + "observationdecks": observationdeck_ids, + "gates": gate_ids, + "checkpoints": checkpoint_ids, + "galleries": gallery_ids, + "publicart": publicart_ids, + } + + features := make(map[string]*geojson.FeatureCollection) + + for k, ids := range feature_ids { + + fc := geojson.NewFeatureCollection() + + for _, id := range ids { + + body, err := wof_reader.LoadBytes(ctx, r, id) + + if err != nil { + return nil, fmt.Errorf("Failed to read %d, %w", id, err) + } + + f, err := geojson.UnmarshalFeature(body) + + if err != nil { + return nil, fmt.Errorf("Failed to unmarshal %d, %w", id, err) + } + + fc.Append(f) + } + + features[k] = fc + } + + return features, nil }