123 lines
2.6 KiB
Go
123 lines
2.6 KiB
Go
package main
|
|
|
|
import (
|
|
"encoding/csv"
|
|
"encoding/json"
|
|
"io"
|
|
"io/ioutil"
|
|
"log"
|
|
"os"
|
|
"regexp"
|
|
"strings"
|
|
)
|
|
|
|
func main() {
|
|
// Read CSV data
|
|
citiesFile, err := os.Open("data/cities15000.txt")
|
|
if err != nil {
|
|
log.Fatalf("Opening file failed: %v", err)
|
|
}
|
|
r := csv.NewReader(citiesFile)
|
|
r.Comma = '\t'
|
|
r.Comment = '#'
|
|
|
|
// Track collisions
|
|
collisions := make(map[string]bool)
|
|
|
|
// Pick out useful information
|
|
cities := make(map[string]City)
|
|
for {
|
|
record, err := r.Read()
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
if err != nil {
|
|
log.Fatalf("Unable to read CSV: %v", err)
|
|
}
|
|
key, city := CityFromRecord(record)
|
|
|
|
// TODO: Reimplement collision rewriter
|
|
|
|
// Remap collisions
|
|
if _, ok := collisions[key]; ok {
|
|
key = key + "_" + city.Admin1Code + "_" + city.CountryCode
|
|
}
|
|
|
|
// Check for collisions
|
|
if existing, ok := cities[key]; ok {
|
|
if existing.CountryCode == city.CountryCode {
|
|
log.Printf("Warning: Repeat entry with same country code for %s (please compare %s with %s)", key, city.Timezone, existing.Timezone)
|
|
} else if existing.Timezone == city.Timezone {
|
|
log.Printf("Warning: Repeat entry with same timezone for %s", key)
|
|
} else {
|
|
log.Printf("Warning: Collision entry found for %s. Rewriting (%s but there is %s)", key, city.CountryCode, existing.CountryCode)
|
|
cities[key+"_"+existing.Admin1Code+"_"+existing.CountryCode] = existing
|
|
delete(cities, "key")
|
|
collisions[key] = true
|
|
}
|
|
}
|
|
cities[key] = city
|
|
}
|
|
|
|
// Group data
|
|
data := Data{
|
|
Cities: cities,
|
|
}
|
|
|
|
// Encode JSON file
|
|
b, err := json.Marshal(data)
|
|
if err != nil {
|
|
log.Fatalf("Failed to encode: %v", err)
|
|
}
|
|
|
|
// Write JSON file
|
|
err = ioutil.WriteFile("js/data.json", b, 0644)
|
|
if err != nil {
|
|
log.Fatalf("Failed to write: %v", err)
|
|
}
|
|
}
|
|
|
|
type Data struct {
|
|
Cities map[string]City
|
|
}
|
|
|
|
type City struct {
|
|
Names []string `json:"n"`
|
|
Admin1Code string `json:"a"`
|
|
CountryCode string `json:"c"`
|
|
Timezone string `json:"t"`
|
|
}
|
|
|
|
// TODO: might be better to have IDs be City_Administrative_SG and the City struct having Names, Administrative, Country as pure text
|
|
|
|
func CityFromRecord(record []string) (string, City) {
|
|
name := normalizeName(record[2])
|
|
names := splitNames(record[3])
|
|
admin1Code := record[10]
|
|
countryCode := record[8]
|
|
timezone := record[17]
|
|
return name, City{
|
|
Names: names,
|
|
Admin1Code: admin1Code,
|
|
CountryCode: countryCode,
|
|
Timezone: timezone,
|
|
}
|
|
}
|
|
|
|
var re = regexp.MustCompile(`[^a-zA-Z1-9]`)
|
|
|
|
func normalizeName(name string) string {
|
|
return re.ReplaceAllString(name, "_")
|
|
}
|
|
|
|
func splitNames(names string) []string {
|
|
return strings.Split(names, ",")
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|