Add geoip cache converter

internal/config/config.go

@@ -72,6 +72,9 @@ type Cache struct {
 type Geo struct {
 	// IPCache contains the path to the IP Geodata cache.
 	IPCache string `yaml:"ipCache,omitempty"`
+
+	// IPCacheDB contains the path to the SQLite DB with the IP Geodata cache.
+	IPCacheDB string `yaml:"ipCacheDB,omitempty"`
 }
 
 // Default contains the default configuration.
@@ -81,7 +84,8 @@ func Default() *Config {
 			Size: 12800,
 		},
 		Geo{
-			IPCache: "/wttr.in/cache/ip2l",
+			IPCache:   "/wttr.in/cache/ip2l",
+			IPCacheDB: "/wttr.in/cache/geoip.db",
 		},
 		Logging{
 			AccessLog: "/wttr.in/log/access.log",

internal/geo/ip/convert.go

@@ -0,0 +1,99 @@
+package ip
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+
+	"github.com/samonzeweb/godb"
+	"github.com/samonzeweb/godb/adapters/sqlite"
+)
+
+func (c *Cache) ConvertCache() error {
+	dbfile := c.config.Geo.IPCacheDB
+
+	err := removeDBIfExists(dbfile)
+	if err != nil {
+		return err
+	}
+
+	db, err := godb.Open(sqlite.Adapter, dbfile)
+	if err != nil {
+		return err
+	}
+
+	err = createTable(db, "Location")
+	if err != nil {
+		return err
+	}
+
+	log.Println("listing cache entries...")
+	files, err := filepath.Glob(filepath.Join(c.config.Geo.IPCache, "*"))
+	if err != nil {
+		return err
+	}
+
+	log.Printf("going to convert %d entries\n", len(files))
+
+	block := []Location{}
+	for i, file := range files {
+		ip := filepath.Base(file)
+		loc, err := c.Read(ip)
+		if err != nil {
+			log.Println("invalid entry for", ip)
+			continue
+		}
+
+		block = append(block, *loc)
+		if i%1000 != 0 || i == 0 {
+			continue
+		}
+
+		err = db.BulkInsert(&block).Do()
+		if err != nil {
+			return err
+		}
+		block = []Location{}
+		log.Println("converted", i+1, "entries")
+	}
+
+	// inserting the rest.
+	err = db.BulkInsert(&block).Do()
+	if err != nil {
+		return err
+	}
+
+	log.Println("converted", len(files), "entries")
+
+	return nil
+}
+
+func createTable(db *godb.DB, tableName string) error {
+	createTable := fmt.Sprintf(
+		`create table %s (
+	    ip             text not null primary key,
+        countryCode    text not null,
+        country        text not null,
+        region         text not null,
+        city           text not null,
+        latitude       text not null,
+        longitude      text not null);
+	`, tableName)
+
+	_, err := db.CurrentDB().Exec(createTable)
+	return err
+}
+
+func removeDBIfExists(filename string) error {
+	_, err := os.Stat(filename)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return err
+		}
+		// no db file
+		return nil
+	}
+
+	return os.Remove(filename)
+}

internal/geo/ip/ip.go

@@ -22,12 +22,13 @@ var (
 
 // Location information.
 type Location struct {
-	CountryCode string
-	Country     string
-	Region      string
-	City        string
-	Latitude    float64
-	Longitude   float64
+	IP          string  `db:"ip,key"`
+	CountryCode string  `db:"countryCode"`
+	Country     string  `db:"country"`
+	Region      string  `db:"region"`
+	City        string  `db:"city"`
+	Latitude    float64 `db:"latitude"`
+	Longitude   float64 `db:"longitude"`
 }
 
 // Cache provides access to the IP Geodata cache.
@@ -59,7 +60,7 @@ func (c *Cache) Read(addr string) (*Location, error) {
 	if err != nil {
 		return nil, ErrNotFound
 	}
-	return parseCacheEntry(string(bytes))
+	return parseCacheEntry(addr, string(bytes))
 }
 
 // cacheFile retuns path to the cache entry for addr.
@@ -69,7 +70,7 @@ func (c *Cache) cacheFile(addr string) string {
 
 // parseCacheEntry parses the location cache entry s,
 // and return location, or error, if the cache entry is invalid.
-func parseCacheEntry(s string) (*Location, error) {
+func parseCacheEntry(addr, s string) (*Location, error) {
 	var (
 		lat  float64 = -1000
 		long float64 = -1000
@@ -94,6 +95,7 @@ func parseCacheEntry(s string) (*Location, error) {
 	}
 
 	return &Location{
+		IP:          addr,
 		CountryCode: parts[0],
 		Country:     parts[1],
 		Region:      parts[2],

internal/geo/ip/ip_test.go

@@ -8,13 +8,16 @@ import (
 
 func TestParseCacheEntry(t *testing.T) {
 	tests := []struct {
+		addr     string
 		input    string
 		expected Location
 		err      error
 	}{
 		{
+			"1.2.3.4",
 			"DE;Germany;Free and Hanseatic City of Hamburg;Hamburg;53.5736;9.9782",
 			Location{
+				IP:          "1.2.3.4",
 				CountryCode: "DE",
 				Country:     "Germany",
 				Region:      "Free and Hanseatic City of Hamburg",
@@ -26,8 +29,10 @@ func TestParseCacheEntry(t *testing.T) {
 		},
 
 		{
+			"1.2.3.4",
 			"ES;Spain;Madrid, Comunidad de;Madrid;40.4165;-3.70256;28223;Orange Espagne SA;orange.es",
 			Location{
+				IP:          "1.2.3.4",
 				CountryCode: "ES",
 				Country:     "Spain",
 				Region:      "Madrid, Comunidad de",
@@ -39,8 +44,10 @@ func TestParseCacheEntry(t *testing.T) {
 		},
 
 		{
+			"1.2.3.4",
 			"US;United States of America;California;Mountain View",
 			Location{
+				IP:          "1.2.3.4",
 				CountryCode: "US",
 				Country:     "United States of America",
 				Region:      "California",
@@ -53,6 +60,7 @@ func TestParseCacheEntry(t *testing.T) {
 
 		// Invalid entries
 		{
+			"1.2.3.4",
 			"DE;Germany;Free and Hanseatic City of Hamburg;Hamburg;53.5736;XXX",
 			Location{},
 			ErrInvalidCacheEntry,
@@ -60,7 +68,7 @@ func TestParseCacheEntry(t *testing.T) {
 	}
 
 	for _, tt := range tests {
-		result, err := parseCacheEntry(tt.input)
+		result, err := parseCacheEntry(tt.addr, tt.input)
 		if tt.err == nil {
 			require.NoError(t, err)
 			require.Equal(t, *result, tt.expected)

srv.go

@@ -12,6 +12,7 @@ import (
 	"github.com/alecthomas/kong"
 
 	"github.com/chubin/wttr.in/internal/config"
+	geoip "github.com/chubin/wttr.in/internal/geo/ip"
 	"github.com/chubin/wttr.in/internal/logging"
 	"github.com/chubin/wttr.in/internal/processor"
 )
@@ -20,6 +21,8 @@ var cli struct {
 	ConfigCheck bool   `name:"config-check" help:"Check configuration"`
 	ConfigDump  bool   `name:"config-dump" help:"Dump configuration"`
 	ConfigFile  string `name:"config-file" arg:"" optional:"" help:"Name of configuration file"`
+
+	ConvertGeoIPCache bool `name:"convert-geo-ip-cache" help:"Convert Geo IP data cache to SQlite"`
 }
 
 const logLineStart = "LOG_LINE_START "
@@ -173,6 +176,12 @@ func main() {
 		return
 	}
 
+	if cli.ConvertGeoIPCache {
+		geoIPCache := geoip.NewCache(conf)
+		ctx.FatalIfErrorf(geoIPCache.ConvertCache())
+		return
+	}
+
 	err = serve(conf)
 	ctx.FatalIfErrorf(err)
 }