Skip to content

Commit

Permalink
Merge pull request #25 from tomaytotomato/23-build-object-links-dataset
Browse files Browse the repository at this point in the history 
23 build links in dataset binary
  • Loading branch information
@tomaytotomato
tomaytotomato authored Aug 11, 2024
2 parents a466f1d + 25ab003 commit 0f03c36
Show file tree
Hide file tree
Showing 9 changed files with 123 additions and 77 deletions.
85 changes: 81 additions & 4 deletions buildtools/src/main/java/com/tomaytotomato/JsonToBinaryConverter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
import com.tomaytotomato.model.City;
import com.tomaytotomato.model.Country;
import com.tomaytotomato.model.State;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
Expand All @@ -16,7 +18,7 @@

/**
* This tool is used to transform the opensource JSON data into a binary format for use by
* location4j
* location4j.
*/
public class JsonToBinaryConverter {

Expand All @@ -38,21 +40,33 @@ public static void main(String[] args) {

var jsonString = new String(inputStream.readAllBytes());

// fix data, for inconsistent namings of JSON properties
// Fix data for inconsistent namings of JSON properties
var modifiedJson = fixJSONPropertyNames(jsonString);

ObjectMapper mapper = new ObjectMapper();
mapper.setPropertyNamingStrategy(new SnakeCaseStrategy());
var countries = mapper.readValue(modifiedJson, new TypeReference<List<Country>>() {
List<Country> countries = mapper.readValue(modifiedJson, new TypeReference<List<Country>>() {
});

// Build links between each object
List<Country> updatedCountries = countries.stream().map(country -> {
List<State> updatedStates = country.getStates().stream().map(state -> {

List<City> cities = state.getCities().stream()
.map(city -> buildCityLinksToStateAndCountry(city, state, country)).toList();

return buildStateLinksToCountry(country, state, cities);
}).toList();
return buildCountry(country, updatedStates);
}).toList();

Path outputFile = Paths.get(OUTPUT_FILE).toAbsolutePath();
logger.log(Level.INFO,
() -> String.format("Serializing data to binary file at: %s", outputFile));

try (var fileOutputStream = new FileOutputStream(outputFile.toFile());
var objectOutputStream = new ObjectOutputStream(fileOutputStream)) {
objectOutputStream.writeObject(countries);
objectOutputStream.writeObject(updatedCountries);
logger.info("Data successfully serialized to binary format.");
}
} catch (IOException e) {
Expand All @@ -71,7 +85,70 @@ private static String fixJSONPropertyNames(String jsonString) {
modifiedJson = modifiedJson.replace("\"gmtOffsetName\"", "\"gmt_offset_name\"");
modifiedJson = modifiedJson.replace("\"tzName\"", "\"tz_name\"");
modifiedJson = modifiedJson.replace("\"emojiU\"", "\"emoji_u\"");
modifiedJson = modifiedJson.replace("\"iso2\"", "\"iso2_code\"");
modifiedJson = modifiedJson.replace("\"iso3\"", "\"iso3_code\"");

return modifiedJson;
}

private static Country buildCountry(Country country, List<State> states) {
return Country.builder()
.id(country.getId())
.name(country.getName())
.iso2Code(country.getIso2Code())
.iso3Code(country.getIso3Code())
.phoneCode(country.getPhoneCode())
.numericCode(country.getNumericCode())
.capital(country.getCapital())
.currency(country.getCurrency())
.currencyName(country.getCurrencyName())
.currencySymbol(country.getCurrencySymbol())
.tld(country.getTld())
.nativeName(country.getNativeName())
.region(country.getRegion())
.regionId(country.getRegionId())
.subregion(country.getSubregion())
.subregionId(country.getSubregionId())
.states(states)
.nationality(country.getNationality())
.timezones(country.getTimezones())
.translations(country.getTranslations())
.latitude(country.getLatitude())
.longitude(country.getLongitude())
.emoji(country.getEmoji())
.emojiU(country.getEmojiU())
.build();
}

private static State buildStateLinksToCountry(Country country, State state, List<City> cities) {
return State.builder()
.id(state.getId())
.name(state.getName())
.type(state.getType())
.countryId(country.getId())
.countryName(country.getName())
.countryIso2Code(country.getIso2Code())
.countryIso3Code(country.getIso3Code())
.stateCode(state.getStateCode())
.latitude(state.getLatitude())
.longitude(state.getLongitude())
.cities(cities)
.build();
}

private static City buildCityLinksToStateAndCountry(City city, State state, Country country) {
return City.builder()
.id(city.getId())
.countryId(country.getId())
.countryName(country.getName())
.countryIso2Code(country.getIso2Code())
.countryIso3Code(country.getIso3Code())
.stateCode(state.getStateCode())
.stateId(state.getId())
.stateName(state.getName())
.name(city.getName())
.longitude(city.getLongitude())
.latitude(city.getLatitude())
.build();
}
}
4 changes: 2 additions & 2 deletions location4j/src/main/java/com/tomaytotomato/LocationService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,8 +87,8 @@ private void buildDataStructures() {
.forEach(
translatedName ->
localisedNameToCountryMap.put(translatedName, country));
iso2CodeToCountryMap.put(keyMaker(country.getIso2()), country);
iso3CodeToCountryMap.put(keyMaker(country.getIso3()), country);
iso2CodeToCountryMap.put(keyMaker(country.getIso2Code()), country);
iso3CodeToCountryMap.put(keyMaker(country.getIso3Code()), country);

country.getStates()
.forEach(
Expand Down
33 changes: 6 additions & 27 deletions location4j/src/main/java/com/tomaytotomato/SearchLocationService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,8 +77,8 @@ private static Location buildLocationResult(Country topCountry, State topState,

.countryName(topCountry.getName())
.countryId(topCountry.getId())
.countryIso2Code(topCountry.getIso2())
.countryIso3Code(topCountry.getIso3())
.countryIso2Code(topCountry.getIso2Code())
.countryIso3Code(topCountry.getIso3Code())
.latitude(topCountry.getLatitude())
.longitude(topCountry.getLongitude());

Expand Down Expand Up @@ -148,8 +148,8 @@ private void addAliases() {
private void buildCountryLookups(Country country) {
countryNameToCountryMap.put(keyMaker(country.getName()), country);
countryIdToCountryMap.put(country.getId(), country);
iso2CodeToCountryMap.put(keyMaker(country.getIso2()), country);
iso3CodeToCountryMap.put(keyMaker(country.getIso3()), country);
iso2CodeToCountryMap.put(keyMaker(country.getIso2Code()), country);
iso3CodeToCountryMap.put(keyMaker(country.getIso3Code()), country);
}

/**
Expand All @@ -159,37 +159,16 @@ private void buildCountryLookups(Country country) {
* @param country The country the state belongs to.
*/
private void buildStateLookups(State state, Country country) {
state.setCountryId(country.getId());
state.setCountryName(country.getName());
state.setCountryIso2Code(country.getIso2());
state.setCountryIso3Code(country.getIso3());

stateIdToStateMap.put(state.getId(), state);
stateNameToStatesMap.computeIfAbsent(keyMaker(state.getName()), k -> new ArrayList<>())
.add(state);
stateCodeToStatesMap.computeIfAbsent(keyMaker(state.getStateCode()), k -> new ArrayList<>())
.add(state);

state.getCities().forEach(city -> buildCityLookups(city, state, country));
state.getCities().forEach(city -> cityNameToCitiesMap.computeIfAbsent(keyMaker(city.getName()),
k -> new ArrayList<>()).add(city));
}

/**
* Maps a city to various lookup maps.
*
* @param city The city to be mapped.
* @param state The state the city belongs to.
* @param country The country the city belongs to.
*/
private void buildCityLookups(City city, State state, Country country) {
city.setCountryId(country.getId());
city.setCountryName(country.getName());
city.setCountryIso2Code(country.getIso2());
city.setCountryIso3Code(country.getIso3());
city.setStateId(state.getId());
city.setStateName(state.getName());
city.setStateCode(state.getStateCode());
cityNameToCitiesMap.computeIfAbsent(keyMaker(city.getName()), k -> new ArrayList<>()).add(city);
}

/**
* Normalizes a key for consistent lookup.
Expand Down
4 changes: 2 additions & 2 deletions location4j/src/main/java/com/tomaytotomato/mapper/DefaultLocationMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@ public Location toLocation(Country country) {
return Location.builder()
.countryName(country.getName())
.countryId(country.getId())
.countryIso2Code(country.getIso2())
.countryIso3Code(country.getIso3())
.countryIso2Code(country.getIso2Code())
.countryIso3Code(country.getIso3Code())
.latitude(country.getLatitude())
.longitude(country.getLongitude())
.build();
Expand Down
9 changes: 5 additions & 4 deletions location4j/src/main/java/com/tomaytotomato/model/City.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,11 @@ public void setStateCode(String stateCode) {
this.stateCode = stateCode;
}

public static Builder builder() {
return new Builder();
}

@Generated("IntelliJ")
public static final class Builder {

private Integer id;
Expand All @@ -161,10 +166,6 @@ public static final class Builder {
private Builder() {
}

public static Builder aCity() {
return new Builder();
}

public Builder id(Integer id) {
this.id = id;
return this;
Expand Down
55 changes: 21 additions & 34 deletions location4j/src/main/java/com/tomaytotomato/model/Country.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,8 @@ public class Country implements Serializable {

private Integer id;
private String name;
private String iso3;
private String iso2;
private String zoneName;
private String iso3Code;
private String iso2Code;
private String phoneCode;
private String numericCode;
private String capital;
Expand Down Expand Up @@ -56,16 +55,12 @@ public String getName() {
return name;
}

public String getIso3() {
return iso3;
public String getIso3Code() {
return iso3Code;
}

public String getIso2() {
return iso2;
}

public String getZoneName() {
return zoneName;
public String getIso2Code() {
return iso2Code;
}

public String getPhoneCode() {
Expand Down Expand Up @@ -148,14 +143,17 @@ public String getEmojiU() {
return emojiU;
}

public static Builder builder() {
return new Builder();
}

@Generated("IntelliJ")
public static final class Builder {

private Integer id;
private String name;
private String iso3;
private String iso2;
private String zoneName;
private String iso3Code;
private String iso2Code;
private String phoneCode;
private String numericCode;
private String capital;
Expand All @@ -180,10 +178,6 @@ public static final class Builder {
private Builder() {
}

public static Builder aCountry() {
return new Builder();
}

public Builder id(Integer id) {
this.id = id;
return this;
Expand All @@ -194,18 +188,13 @@ public Builder name(String name) {
return this;
}

public Builder iso3(String iso3) {
this.iso3 = iso3;
return this;
}

public Builder iso2(String iso2) {
this.iso2 = iso2;
public Builder iso3Code(String iso3Code) {
this.iso3Code = iso3Code;
return this;
}

public Builder zoneName(String zoneName) {
this.zoneName = zoneName;
public Builder iso2Code(String iso2Code) {
this.iso2Code = iso2Code;
return this;
}

Expand Down Expand Up @@ -313,23 +302,22 @@ public Country build() {
Country country = new Country();
country.name = this.name;
country.phoneCode = this.phoneCode;
country.iso3 = this.iso3;
country.iso3Code = this.iso3Code;
country.longitude = this.longitude;
country.currencyName = this.currencyName;
country.numericCode = this.numericCode;
country.subregionId = this.subregionId;
country.capital = this.capital;
country.region = this.region;
country.translations = this.translations;
country.iso2 = this.iso2;
country.iso2Code = this.iso2Code;
country.currencySymbol = this.currencySymbol;
country.timezones = this.timezones;
country.tld = this.tld;
country.currency = this.currency;
country.nativeName = this.nativeName;
country.emoji = this.emoji;
country.subregion = this.subregion;
country.zoneName = this.zoneName;
country.regionId = this.regionId;
country.latitude = this.latitude;
country.id = this.id;
Expand All @@ -351,9 +339,8 @@ public boolean equals(Object o) {
}
Country country = (Country) o;
return Objects.equals(getId(), country.getId()) && Objects.equals(getName(),
country.getName()) && Objects.equals(getIso3(), country.getIso3())
&& Objects.equals(getIso2(), country.getIso2()) && Objects.equals(
getZoneName(), country.getZoneName()) && Objects.equals(getPhoneCode(),
country.getName()) && Objects.equals(getIso3Code(), country.getIso3Code())
&& Objects.equals(getIso2Code(), country.getIso2Code()) && Objects.equals(getPhoneCode(),
country.getPhoneCode()) && Objects.equals(getNumericCode(),
country.getNumericCode()) && Objects.equals(getCapital(), country.getCapital())
&& Objects.equals(getCurrency(), country.getCurrency()) && Objects.equals(
Expand All @@ -375,7 +362,7 @@ public boolean equals(Object o) {
@Generated("IntelliJ")
@Override
public int hashCode() {
return Objects.hash(getId(), getName(), getIso3(), getIso2(), getZoneName(), getPhoneCode(),
return Objects.hash(getId(), getName(), getIso3Code(), getIso2Code(), getPhoneCode(),
getNumericCode(), getCapital(), getCurrency(), getCurrencyName(), getCurrencySymbol(),
getTld(), getNativeName(), getRegion(), getRegionId(), getSubregion(), getSubregionId(),
getStates(), getNationality(), getTimezones(), getTranslations(), getLatitude(),
Expand Down
9 changes: 5 additions & 4 deletions location4j/src/main/java/com/tomaytotomato/model/State.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,11 @@ public void setCountryIso3Code(String countryIso3Code) {
this.countryIso3Code = countryIso3Code;
}

public static Builder builder() {
return new Builder();
}

@Generated("IntelliJ")
public static final class Builder {

private Integer id;
Expand All @@ -145,10 +150,6 @@ public static final class Builder {
private Builder() {
}

public static Builder aState() {
return new Builder();
}

public Builder id(Integer id) {
this.id = id;
return this;
Expand Down
1 change: 1 addition & 0 deletions location4j/src/main/java/com/tomaytotomato/model/TimeZone.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ public String getGmtOffsetName() {
return gmtOffsetName;
}

@Generated("IntelliJ")
public static final class Builder {

private String zoneName;
Expand Down
Binary file modified location4j/src/main/resources/location4j.bin
View file
Binary file not shown.

0 comments on commit 0f03c36

Please sign in to comment.