From b54ba3664393b7c8db142417fc0739322f3cc91c Mon Sep 17 00:00:00 2001 From: Igor Chubin Date: Sun, 4 Dec 2022 21:16:23 +0100 Subject: [PATCH] Add geoip cache converter --- internal/config/config.go | 6 ++- internal/geo/ip/convert.go | 99 ++++++++++++++++++++++++++++++++++++++ internal/geo/ip/ip.go | 18 ++++--- internal/geo/ip/ip_test.go | 10 +++- srv.go | 9 ++++ 5 files changed, 132 insertions(+), 10 deletions(-) create mode 100644 internal/geo/ip/convert.go diff --git a/internal/config/config.go b/internal/config/config.go index de4024d..c7c618b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -72,6 +72,9 @@ type Cache struct { type Geo struct { // IPCache contains the path to the IP Geodata cache. IPCache string `yaml:"ipCache,omitempty"` + + // IPCacheDB contains the path to the SQLite DB with the IP Geodata cache. + IPCacheDB string `yaml:"ipCacheDB,omitempty"` } // Default contains the default configuration. @@ -81,7 +84,8 @@ func Default() *Config { Size: 12800, }, Geo{ - IPCache: "/wttr.in/cache/ip2l", + IPCache: "/wttr.in/cache/ip2l", + IPCacheDB: "/wttr.in/cache/geoip.db", }, Logging{ AccessLog: "/wttr.in/log/access.log", diff --git a/internal/geo/ip/convert.go b/internal/geo/ip/convert.go new file mode 100644 index 0000000..8582685 --- /dev/null +++ b/internal/geo/ip/convert.go @@ -0,0 +1,99 @@ +package ip + +import ( + "fmt" + "log" + "os" + "path/filepath" + + "github.com/samonzeweb/godb" + "github.com/samonzeweb/godb/adapters/sqlite" +) + +func (c *Cache) ConvertCache() error { + dbfile := c.config.Geo.IPCacheDB + + err := removeDBIfExists(dbfile) + if err != nil { + return err + } + + db, err := godb.Open(sqlite.Adapter, dbfile) + if err != nil { + return err + } + + err = createTable(db, "Location") + if err != nil { + return err + } + + log.Println("listing cache entries...") + files, err := filepath.Glob(filepath.Join(c.config.Geo.IPCache, "*")) + if err != nil { + return err + } + + log.Printf("going to convert %d entries\n", len(files)) + + block := []Location{} + for i, file := range files { + ip := filepath.Base(file) + loc, err := c.Read(ip) + if err != nil { + log.Println("invalid entry for", ip) + continue + } + + block = append(block, *loc) + if i%1000 != 0 || i == 0 { + continue + } + + err = db.BulkInsert(&block).Do() + if err != nil { + return err + } + block = []Location{} + log.Println("converted", i+1, "entries") + } + + // inserting the rest. + err = db.BulkInsert(&block).Do() + if err != nil { + return err + } + + log.Println("converted", len(files), "entries") + + return nil +} + +func createTable(db *godb.DB, tableName string) error { + createTable := fmt.Sprintf( + `create table %s ( + ip text not null primary key, + countryCode text not null, + country text not null, + region text not null, + city text not null, + latitude text not null, + longitude text not null); + `, tableName) + + _, err := db.CurrentDB().Exec(createTable) + return err +} + +func removeDBIfExists(filename string) error { + _, err := os.Stat(filename) + if err != nil { + if !os.IsNotExist(err) { + return err + } + // no db file + return nil + } + + return os.Remove(filename) +} diff --git a/internal/geo/ip/ip.go b/internal/geo/ip/ip.go index d74381f..dd9fbd2 100644 --- a/internal/geo/ip/ip.go +++ b/internal/geo/ip/ip.go @@ -22,12 +22,13 @@ var ( // Location information. type Location struct { - CountryCode string - Country string - Region string - City string - Latitude float64 - Longitude float64 + IP string `db:"ip,key"` + CountryCode string `db:"countryCode"` + Country string `db:"country"` + Region string `db:"region"` + City string `db:"city"` + Latitude float64 `db:"latitude"` + Longitude float64 `db:"longitude"` } // Cache provides access to the IP Geodata cache. @@ -59,7 +60,7 @@ func (c *Cache) Read(addr string) (*Location, error) { if err != nil { return nil, ErrNotFound } - return parseCacheEntry(string(bytes)) + return parseCacheEntry(addr, string(bytes)) } // cacheFile retuns path to the cache entry for addr. @@ -69,7 +70,7 @@ func (c *Cache) cacheFile(addr string) string { // parseCacheEntry parses the location cache entry s, // and return location, or error, if the cache entry is invalid. -func parseCacheEntry(s string) (*Location, error) { +func parseCacheEntry(addr, s string) (*Location, error) { var ( lat float64 = -1000 long float64 = -1000 @@ -94,6 +95,7 @@ func parseCacheEntry(s string) (*Location, error) { } return &Location{ + IP: addr, CountryCode: parts[0], Country: parts[1], Region: parts[2], diff --git a/internal/geo/ip/ip_test.go b/internal/geo/ip/ip_test.go index 8921628..bcad77c 100644 --- a/internal/geo/ip/ip_test.go +++ b/internal/geo/ip/ip_test.go @@ -8,13 +8,16 @@ import ( func TestParseCacheEntry(t *testing.T) { tests := []struct { + addr string input string expected Location err error }{ { + "1.2.3.4", "DE;Germany;Free and Hanseatic City of Hamburg;Hamburg;53.5736;9.9782", Location{ + IP: "1.2.3.4", CountryCode: "DE", Country: "Germany", Region: "Free and Hanseatic City of Hamburg", @@ -26,8 +29,10 @@ func TestParseCacheEntry(t *testing.T) { }, { + "1.2.3.4", "ES;Spain;Madrid, Comunidad de;Madrid;40.4165;-3.70256;28223;Orange Espagne SA;orange.es", Location{ + IP: "1.2.3.4", CountryCode: "ES", Country: "Spain", Region: "Madrid, Comunidad de", @@ -39,8 +44,10 @@ func TestParseCacheEntry(t *testing.T) { }, { + "1.2.3.4", "US;United States of America;California;Mountain View", Location{ + IP: "1.2.3.4", CountryCode: "US", Country: "United States of America", Region: "California", @@ -53,6 +60,7 @@ func TestParseCacheEntry(t *testing.T) { // Invalid entries { + "1.2.3.4", "DE;Germany;Free and Hanseatic City of Hamburg;Hamburg;53.5736;XXX", Location{}, ErrInvalidCacheEntry, @@ -60,7 +68,7 @@ func TestParseCacheEntry(t *testing.T) { } for _, tt := range tests { - result, err := parseCacheEntry(tt.input) + result, err := parseCacheEntry(tt.addr, tt.input) if tt.err == nil { require.NoError(t, err) require.Equal(t, *result, tt.expected) diff --git a/srv.go b/srv.go index c39b80e..7d2a134 100644 --- a/srv.go +++ b/srv.go @@ -12,6 +12,7 @@ import ( "github.com/alecthomas/kong" "github.com/chubin/wttr.in/internal/config" + geoip "github.com/chubin/wttr.in/internal/geo/ip" "github.com/chubin/wttr.in/internal/logging" "github.com/chubin/wttr.in/internal/processor" ) @@ -20,6 +21,8 @@ var cli struct { ConfigCheck bool `name:"config-check" help:"Check configuration"` ConfigDump bool `name:"config-dump" help:"Dump configuration"` ConfigFile string `name:"config-file" arg:"" optional:"" help:"Name of configuration file"` + + ConvertGeoIPCache bool `name:"convert-geo-ip-cache" help:"Convert Geo IP data cache to SQlite"` } const logLineStart = "LOG_LINE_START " @@ -173,6 +176,12 @@ func main() { return } + if cli.ConvertGeoIPCache { + geoIPCache := geoip.NewCache(conf) + ctx.FatalIfErrorf(geoIPCache.ConvertCache()) + return + } + err = serve(conf) ctx.FatalIfErrorf(err) }