diff --git a/go.mod b/go.mod index c435a4d..bf7549e 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/smartystreets/assertions v1.2.0 // indirect github.com/smartystreets/goconvey v1.6.4 // indirect github.com/stretchr/testify v1.8.1 // indirect + github.com/zsefvlol/timezonemapper v1.0.0 // indirect golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 1fa36b2..084eb3f 100644 --- a/go.sum +++ b/go.sum @@ -38,6 +38,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/zsefvlol/timezonemapper v1.0.0 h1:HXqkOzf01gXYh2nDQcDSROikFgMaximnhE8BY9SyF6E= +github.com/zsefvlol/timezonemapper v1.0.0/go.mod h1:cVUCOLEmc/VvOMusEhpd2G/UBtadL26ZVz2syODXDoQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= diff --git a/internal/config/config.go b/internal/config/config.go index 611c35b..bac275c 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -77,7 +77,15 @@ type Geo struct { // IPCacheDB contains the path to the SQLite DB with the IP Geodata cache. IPCacheDB string `yaml:"ipCacheDB,omitempty"` - CacheType types.CacheType `yaml:"cacheType,omitempty"` + IPCacheType types.CacheType `yaml:"cacheType,omitempty"` + + // LocationCache contains the path to the Location Geodata cache. + LocationCache string `yaml:"locationCache,omitempty"` + + // LocationCacheDB contains the path to the SQLite DB with the Location Geodata cache. + LocationCacheDB string `yaml:"locationCacheDB,omitempty"` + + LocationCacheType types.CacheType `yaml:"locationCacheType,omitempty"` Nominatim []Nominatim } @@ -97,9 +105,12 @@ func Default() *Config { Size: 12800, }, Geo{ - IPCache: "/wttr.in/cache/ip2l", - IPCacheDB: "/wttr.in/cache/geoip.db", - CacheType: types.CacheTypeDB, + IPCache: "/wttr.in/cache/ip2l", + IPCacheDB: "/wttr.in/cache/geoip.db", + IPCacheType: types.CacheTypeDB, + LocationCache: "/wttr.in/cache/loc", + LocationCacheDB: "/wttr.in/cache/geoloc.db", + LocationCacheType: types.CacheTypeFiles, Nominatim: []Nominatim{ { Name: "locationiq", diff --git a/internal/geo/ip/convert.go b/internal/geo/ip/convert.go index 8582685..962d546 100644 --- a/internal/geo/ip/convert.go +++ b/internal/geo/ip/convert.go @@ -3,17 +3,18 @@ package ip import ( "fmt" "log" - "os" "path/filepath" "github.com/samonzeweb/godb" "github.com/samonzeweb/godb/adapters/sqlite" + + "github.com/chubin/wttr.in/internal/util" ) func (c *Cache) ConvertCache() error { dbfile := c.config.Geo.IPCacheDB - err := removeDBIfExists(dbfile) + err := util.RemoveFileIfExists(dbfile) if err != nil { return err } @@ -29,7 +30,7 @@ func (c *Cache) ConvertCache() error { } log.Println("listing cache entries...") - files, err := filepath.Glob(filepath.Join(c.config.Geo.IPCache, "*")) + files, err := filepath.Glob(filepath.Join(c.config.Geo.LocationCache, "*")) if err != nil { return err } @@ -72,28 +73,12 @@ func (c *Cache) ConvertCache() error { func createTable(db *godb.DB, tableName string) error { createTable := fmt.Sprintf( `create table %s ( - ip text not null primary key, - countryCode text not null, - country text not null, - region text not null, - city text not null, - latitude text not null, - longitude text not null); + name text not null primary key, + fullName text not null, + lat text not null, + long text not null); `, tableName) _, err := db.CurrentDB().Exec(createTable) return err } - -func removeDBIfExists(filename string) error { - _, err := os.Stat(filename) - if err != nil { - if !os.IsNotExist(err) { - return err - } - // no db file - return nil - } - - return os.Remove(filename) -} diff --git a/internal/geo/ip/ip.go b/internal/geo/ip/ip.go index 96ababf..b20336c 100644 --- a/internal/geo/ip/ip.go +++ b/internal/geo/ip/ip.go @@ -1,7 +1,6 @@ package ip import ( - "errors" "fmt" "log" "net/http" @@ -19,11 +18,6 @@ import ( "github.com/samonzeweb/godb/adapters/sqlite" ) -var ( - ErrNotFound = errors.New("cache entry not found") - ErrInvalidCacheEntry = errors.New("invalid cache entry format") -) - // Location information. type Location struct { IP string `db:"ip,key"` @@ -69,8 +63,8 @@ func NewCache(config *config.Config) (*Cache, error) { } // Read returns location information from the cache, if found, -// or ErrNotFound if not found. If the entry is found, but its format -// is invalid, ErrInvalidCacheEntry is returned. +// or types.ErrNotFound if not found. If the entry is found, but its format +// is invalid, types.ErrInvalidCacheEntry is returned. // // Format: // @@ -81,7 +75,7 @@ func NewCache(config *config.Config) (*Cache, error) { // DE;Germany;Free and Hanseatic City of Hamburg;Hamburg;53.5736;9.9782 // func (c *Cache) Read(addr string) (*Location, error) { - if c.config.Geo.CacheType == types.CacheTypeDB { + if c.config.Geo.IPCacheType == types.CacheTypeDB { return c.readFromCacheDB(addr) } return c.readFromCacheFile(addr) @@ -90,7 +84,7 @@ func (c *Cache) Read(addr string) (*Location, error) { func (c *Cache) readFromCacheFile(addr string) (*Location, error) { bytes, err := os.ReadFile(c.cacheFile(addr)) if err != nil { - return nil, ErrNotFound + return nil, types.ErrNotFound } return NewLocationFromString(addr, string(bytes)) } @@ -107,7 +101,7 @@ func (c *Cache) readFromCacheDB(addr string) (*Location, error) { } func (c *Cache) Put(addr string, loc *Location) error { - if c.config.Geo.CacheType == types.CacheTypeDB { + if c.config.Geo.IPCacheType == types.CacheTypeDB { return c.putToCacheDB(addr, loc) } return c.putToCacheFile(addr, loc) @@ -150,18 +144,18 @@ func NewLocationFromString(addr, s string) (*Location, error) { parts := strings.Split(s, ";") if len(parts) < 4 { - return nil, ErrInvalidCacheEntry + return nil, types.ErrInvalidCacheEntry } if len(parts) >= 6 { lat, err = strconv.ParseFloat(parts[4], 64) if err != nil { - return nil, ErrInvalidCacheEntry + return nil, types.ErrInvalidCacheEntry } long, err = strconv.ParseFloat(parts[5], 64) if err != nil { - return nil, ErrInvalidCacheEntry + return nil, types.ErrInvalidCacheEntry } } diff --git a/internal/geo/ip/ip_test.go b/internal/geo/ip/ip_test.go index 7e98f00..dcd4cb7 100644 --- a/internal/geo/ip/ip_test.go +++ b/internal/geo/ip/ip_test.go @@ -4,6 +4,8 @@ import ( "testing" "github.com/stretchr/testify/require" + + "github.com/chubin/wttr.in/internal/types" ) func TestParseCacheEntry(t *testing.T) { @@ -63,7 +65,7 @@ func TestParseCacheEntry(t *testing.T) { "1.2.3.4", "DE;Germany;Free and Hanseatic City of Hamburg;Hamburg;53.5736;XXX", Location{}, - ErrInvalidCacheEntry, + types.ErrInvalidCacheEntry, }, } diff --git a/internal/geo/location/location.go b/internal/geo/location/location.go index 86fde7f..c07cbbc 100644 --- a/internal/geo/location/location.go +++ b/internal/geo/location/location.go @@ -1,12 +1,29 @@ package location -import "github.com/chubin/wttr.in/internal/config" +import ( + "encoding/json" + "log" + + "github.com/chubin/wttr.in/internal/config" +) type Location struct { - Name string - Fullname string `json:"display_name"` - Lat string - Lon string + Name string `db:"name,key"` + Fullname string `db:"displayName" json:"display_name"` + Lat string `db:"lat"` + Lon string `db:"lon"` + Timezone string `db:"timezone"` +} + +// String returns string represenation of location +func (l *Location) String() string { + bytes, err := json.Marshal(l) + if err != nil { + // should never happen + log.Fatalln(err) + } + + return string(bytes) } type Provider interface { diff --git a/internal/types/types.go b/internal/types/types.go index d28d0c6..24f0f6d 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -1,8 +1,15 @@ package types +import "errors" + type CacheType string const ( CacheTypeDB = "db" CacheTypeFiles = "files" ) + +var ( + ErrNotFound = errors.New("cache entry not found") + ErrInvalidCacheEntry = errors.New("invalid cache entry format") +) diff --git a/srv.go b/srv.go index f47b10d..a8c3739 100644 --- a/srv.go +++ b/srv.go @@ -25,7 +25,8 @@ var cli struct { ConfigFile string `name:"config-file" arg:"" optional:"" help:"Name of configuration file"` - ConvertGeoIPCache bool `name:"convert-geo-ip-cache" help:"Convert Geo IP data cache to SQlite"` + ConvertGeoIPCache bool `name:"convert-geo-ip-cache" help:"Convert Geo IP data cache to SQlite"` + ConvertGeoLocationCache bool `name:"convert-geo-location-cache" help:"Convert Geo Location data cache to SQlite"` } const logLineStart = "LOG_LINE_START " @@ -188,6 +189,15 @@ func main() { return } + if cli.ConvertGeoLocationCache { + geoLocCache, err := geoloc.NewCache(conf) + if err != nil { + ctx.FatalIfErrorf(err) + } + ctx.FatalIfErrorf(geoLocCache.ConvertCache()) + return + } + if cli.GeoResolve != "" { sr := geoloc.NewSearcher(conf) loc, err := sr.Search(cli.GeoResolve)