Add internal/geo/location/

internal/geo/location/cache.go

@@ -0,0 +1,142 @@
+package location
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/samonzeweb/godb"
+	"github.com/samonzeweb/godb/adapters/sqlite"
+	"github.com/zsefvlol/timezonemapper"
+
+	"github.com/chubin/wttr.in/internal/config"
+	"github.com/chubin/wttr.in/internal/types"
+)
+
+// Cache is an implemenation of DB/file-based cache.
+//
+// At the moment, it is an implementation for the location cache,
+// but it should be generalized to cache everything.
+type Cache struct {
+	config        *config.Config
+	db            *godb.DB
+	indexField    string
+	filesCacheDir string
+}
+
+// NewCache returns new cache reader for the specified config.
+func NewCache(config *config.Config) (*Cache, error) {
+	var (
+		db  *godb.DB
+		err error
+	)
+
+	if config.Geo.LocationCacheType == types.CacheTypeDB {
+		db, err = godb.Open(sqlite.Adapter, config.Geo.IPCacheDB)
+		if err != nil {
+			return nil, err
+		}
+
+		// Needed for "upsert" implementation in Put()
+		db.UseErrorParser()
+	}
+
+	return &Cache{
+		config:        config,
+		db:            db,
+		indexField:    "name",
+		filesCacheDir: config.Geo.LocationCache,
+	}, nil
+}
+
+// Read returns location information from the cache, if found,
+// or types.ErrNotFound if not found. If the entry is found, but its format
+// is invalid, types.ErrInvalidCacheEntry is returned.
+func (c *Cache) Read(addr string) (*Location, error) {
+	if c.config.Geo.LocationCacheType == types.CacheTypeFiles {
+		return c.readFromCacheFile(addr)
+	}
+
+	return c.readFromCacheDB(addr)
+}
+
+func (c *Cache) readFromCacheFile(name string) (*Location, error) {
+	var (
+		fileLoc = struct {
+			Latitude  float64 `json:"latitude"`
+			Longitude float64 `json:"longitude"`
+			Timezone  string  `json:"timezone"`
+			Address   string  `json:"address"`
+		}{}
+		location Location
+	)
+
+	bytes, err := os.ReadFile(c.cacheFile(name))
+	if err != nil {
+		return nil, types.ErrNotFound
+	}
+	err = json.Unmarshal(bytes, &fileLoc)
+	if err != nil {
+		return nil, err
+	}
+
+	// normalize name
+	name = strings.TrimSpace(
+		strings.TrimRight(
+			strings.TrimLeft(name, `"`), `"`))
+
+	timezone := fileLoc.Timezone
+	if timezone == "" {
+		timezone = timezonemapper.LatLngToTimezoneString(fileLoc.Latitude, fileLoc.Longitude)
+	}
+
+	location = Location{
+		Name:     name,
+		Lat:      fmt.Sprint(fileLoc.Latitude),
+		Lon:      fmt.Sprint(fileLoc.Longitude),
+		Timezone: timezone,
+		Fullname: fileLoc.Address,
+	}
+
+	return &location, nil
+}
+
+func (c *Cache) readFromCacheDB(addr string) (*Location, error) {
+	result := Location{}
+	err := c.db.Select(&result).
+		Where(c.indexField+" = ?", addr).
+		Do()
+	if err != nil {
+		return nil, err
+	}
+
+	return &result, nil
+}
+
+func (c *Cache) Put(addr string, loc *Location) error {
+	if c.config.Geo.IPCacheType == types.CacheTypeDB {
+		return c.putToCacheDB(loc)
+	}
+
+	return c.putToCacheFile(addr, loc)
+}
+
+func (c *Cache) putToCacheDB(loc *Location) error {
+	err := c.db.Insert(loc).Do()
+	if strings.Contains(fmt.Sprint(err), "UNIQUE constraint failed") {
+		return c.db.Update(loc).Do()
+	}
+
+	return err
+}
+
+func (c *Cache) putToCacheFile(addr string, loc fmt.Stringer) error {
+	return os.WriteFile(c.cacheFile(addr), []byte(loc.String()), 0o600)
+}
+
+// cacheFile returns path to the cache entry for addr.
+func (c *Cache) cacheFile(item string) string {
+	return path.Join(c.filesCacheDir, item)
+}

internal/geo/location/convert.go

@@ -0,0 +1,120 @@
+package location
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/samonzeweb/godb"
+	"github.com/samonzeweb/godb/adapters/sqlite"
+)
+
+//nolint:funlen,cyclop
+func (c *Cache) ConvertCache() error {
+	var (
+		dbfile     = c.config.Geo.LocationCacheDB
+		tableName  = "Location"
+		cacheFiles = c.filesCacheDir
+		known      = map[string]bool{}
+	)
+
+	err := removeDBIfExists(dbfile)
+	if err != nil {
+		return err
+	}
+
+	db, err := godb.Open(sqlite.Adapter, dbfile)
+	if err != nil {
+		return err
+	}
+
+	err = createTable(db, tableName)
+	if err != nil {
+		return err
+	}
+
+	log.Println("listing cache entries...")
+	files, err := filepath.Glob(filepath.Join(cacheFiles, "*"))
+	if err != nil {
+		return err
+	}
+
+	log.Printf("going to convert %d entries\n", len(files))
+
+	block := []Location{}
+	for i, file := range files {
+		ip := filepath.Base(file)
+		loc, err := c.Read(ip)
+		if err != nil {
+			log.Println("invalid entry for", ip)
+
+			continue
+		}
+
+		// Skip duplicates.
+		if known[loc.Name] {
+			log.Println("skipping", loc.Name)
+
+			continue
+		}
+		known[loc.Name] = true
+
+		// Skip some invalid names.
+		if strings.Contains(loc.Name, "\n") {
+			continue
+		}
+
+		block = append(block, *loc)
+		if i%1000 != 0 || i == 0 {
+			continue
+		}
+
+		log.Println("going to insert new entries")
+		err = db.BulkInsert(&block).Do()
+		if err != nil {
+			return err
+		}
+		block = []Location{}
+		log.Println("converted", i+1, "entries")
+	}
+
+	// inserting the rest.
+	err = db.BulkInsert(&block).Do()
+	if err != nil {
+		return err
+	}
+
+	log.Println("converted", len(files), "entries")
+
+	return nil
+}
+
+func createTable(db *godb.DB, tableName string) error {
+	createTable := fmt.Sprintf(
+		`create table %s (
+	    name           text not null primary key,
+        displayName    text not null,
+        lat            text not null,
+        lon            text not null,
+		timezone       text not null);
+	`, tableName)
+
+	_, err := db.CurrentDB().Exec(createTable)
+
+	return err
+}
+
+func removeDBIfExists(filename string) error {
+	_, err := os.Stat(filename)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return err
+		}
+		// no db file
+		return nil
+	}
+
+	return os.Remove(filename)
+}