Fix linter checks

chubin/logging
Igor Chubin 2 years ago
parent aec889e65e
commit aa3600a011

@ -21,7 +21,6 @@ type Config struct {
// Logging configuration.
type Logging struct {
// AccessLog path.
AccessLog string `yaml:"accessLog,omitempty"`
@ -34,14 +33,13 @@ type Logging struct {
// Server configuration.
type Server struct {
// PortHTTP is port where HTTP server must listen.
// If 0, HTTP is disabled.
PortHTTP int `yaml:"portHTTP,omitempty"`
PortHTTP int `yaml:"portHttp,omitempty"`
// PortHTTP is port where the HTTPS server must listen.
// If 0, HTTPS is disabled.
PortHTTPS int `yaml:"portHTTPS,omitempty"`
PortHTTPS int `yaml:"portHttps,omitempty"`
// TLSCertFile contains path to cert file for TLS Server.
TLSCertFile string `yaml:"tlsCertFile,omitempty"`
@ -75,7 +73,7 @@ type Geo struct {
IPCache string `yaml:"ipCache,omitempty"`
// IPCacheDB contains the path to the SQLite DB with the IP Geodata cache.
IPCacheDB string `yaml:"ipCacheDB,omitempty"`
IPCacheDB string `yaml:"ipCacheDb,omitempty"`
IPCacheType types.CacheType `yaml:"cacheType,omitempty"`
@ -83,7 +81,7 @@ type Geo struct {
LocationCache string `yaml:"locationCache,omitempty"`
// LocationCacheDB contains the path to the SQLite DB with the Location Geodata cache.
LocationCacheDB string `yaml:"locationCacheDB,omitempty"`
LocationCacheDB string `yaml:"locationCacheDb,omitempty"`
LocationCacheType types.CacheType `yaml:"locationCacheType,omitempty"`
@ -165,5 +163,6 @@ func (c *Config) Dump() []byte {
// should never happen.
log.Fatalln("config.Dump():", err)
}
return data
}

@ -11,6 +11,7 @@ import (
"github.com/chubin/wttr.in/internal/util"
)
//nolint:cyclop
func (c *Cache) ConvertCache() error {
dbfile := c.config.Geo.IPCacheDB
@ -43,10 +44,12 @@ func (c *Cache) ConvertCache() error {
loc, err := c.Read(ip)
if err != nil {
log.Println("invalid entry for", ip)
continue
}
block = append(block, *loc)
if i%1000 != 0 || i == 0 {
continue
}
@ -80,5 +83,6 @@ func createTable(db *godb.DB, tableName string) error {
`, tableName)
_, err := db.CurrentDB().Exec(createTable)
return err
}

@ -35,6 +35,7 @@ func (l *Location) String() string {
"%s;%s;%s;%s",
l.CountryCode, l.CountryCode, l.Region, l.City)
}
return fmt.Sprintf(
"%s;%s;%s;%s;%v;%v",
l.CountryCode, l.CountryCode, l.Region, l.City, l.Latitude, l.Longitude)
@ -68,16 +69,18 @@ func NewCache(config *config.Config) (*Cache, error) {
//
// Format:
//
// [CountryCode];Country;Region;City;[Latitude];[Longitude]
// [CountryCode];Country;Region;City;[Latitude];[Longitude]
//
// Example:
//
// DE;Germany;Free and Hanseatic City of Hamburg;Hamburg;53.5736;9.9782
// DE;Germany;Free and Hanseatic City of Hamburg;Hamburg;53.5736;9.9782
//
func (c *Cache) Read(addr string) (*Location, error) {
if c.config.Geo.IPCacheType == types.CacheTypeDB {
return c.readFromCacheDB(addr)
}
return c.readFromCacheFile(addr)
}
@ -86,6 +89,7 @@ func (c *Cache) readFromCacheFile(addr string) (*Location, error) {
if err != nil {
return nil, types.ErrNotFound
}
return NewLocationFromString(addr, string(bytes))
}
@ -97,17 +101,19 @@ func (c *Cache) readFromCacheDB(addr string) (*Location, error) {
if err != nil {
return nil, err
}
return &result, nil
}
func (c *Cache) Put(addr string, loc *Location) error {
if c.config.Geo.IPCacheType == types.CacheTypeDB {
return c.putToCacheDB(addr, loc)
return c.putToCacheDB(loc)
}
return c.putToCacheFile(addr, loc)
}
func (c *Cache) putToCacheDB(addr string, loc *Location) error {
func (c *Cache) putToCacheDB(loc *Location) error {
err := c.db.Insert(loc).Do()
// it should work like this:
//
@ -121,14 +127,15 @@ func (c *Cache) putToCacheDB(addr string, loc *Location) error {
if strings.Contains(fmt.Sprint(err), "UNIQUE constraint failed") {
return c.db.Update(loc).Do()
}
return err
}
func (c *Cache) putToCacheFile(addr string, loc *Location) error {
return os.WriteFile(c.cacheFile(addr), []byte(loc.String()), 0644)
func (c *Cache) putToCacheFile(addr string, loc fmt.Stringer) error {
return os.WriteFile(c.cacheFile(addr), []byte(loc.String()), 0o600)
}
// cacheFile retuns path to the cache entry for addr.
// cacheFile returns path to the cache entry for addr.
func (c *Cache) cacheFile(addr string) string {
return path.Join(c.config.Geo.IPCache, addr)
}
@ -170,14 +177,15 @@ func NewLocationFromString(addr, s string) (*Location, error) {
}, nil
}
// Reponse provides routing interface to the geo cache.
// Response provides routing interface to the geo cache.
//
// Temporary workaround to switch IP addresses handling to the Go server.
// Handles two queries:
//
// /:geo-ip-put?ip=IP&value=VALUE
// /:geo-ip-get?ip=IP
// - /:geo-ip-put?ip=IP&value=VALUE
// - /:geo-ip-get?ip=IP
//
//nolint:cyclop
func (c *Cache) Response(r *http.Request) *routing.Cadre {
var (
respERR = &routing.Cadre{Body: []byte("ERR")}
@ -186,6 +194,7 @@ func (c *Cache) Response(r *http.Request) *routing.Cadre {
if ip := util.ReadUserIP(r); ip != "127.0.0.1" {
log.Printf("geoIP access from %s rejected\n", ip)
return nil
}
@ -194,6 +203,7 @@ func (c *Cache) Response(r *http.Request) *routing.Cadre {
value := r.URL.Query().Get("value")
if !validIP4(ip) || value == "" {
log.Printf("invalid geoIP put query: ip='%s' value='%s'\n", ip, value)
return respERR
}
@ -206,6 +216,7 @@ func (c *Cache) Response(r *http.Request) *routing.Cadre {
if err != nil {
return respERR
}
return respOK
}
if r.URL.Path == "/:geo-ip-get" {
@ -218,12 +229,16 @@ func (c *Cache) Response(r *http.Request) *routing.Cadre {
if result == nil || err != nil {
return respERR
}
return &routing.Cadre{Body: []byte(result.String())}
}
return nil
}
func validIP4(ipAddress string) bool {
re, _ := regexp.Compile(`^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$`)
re := regexp.MustCompile(
`^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$`)
return re.MatchString(strings.Trim(ipAddress, " "))
}

@ -1,14 +1,17 @@
package ip
package ip_test
import (
"testing"
"github.com/stretchr/testify/require"
. "github.com/chubin/wttr.in/internal/geo/ip"
"github.com/chubin/wttr.in/internal/types"
)
//nolint:funlen
func TestParseCacheEntry(t *testing.T) {
t.Parallel()
tests := []struct {
addr string
input string

@ -9,13 +9,14 @@ import (
type Location struct {
Name string `db:"name,key"`
Fullname string `db:"displayName" json:"display_name"`
Lat string `db:"lat"`
Lon string `db:"lon"`
Timezone string `db:"timezone"`
//nolint:tagliatelle
Fullname string `db:"displayName" json:"display_name"`
}
// String returns string represenation of location
// String returns string representation of location.
func (l *Location) String() string {
bytes, err := json.Marshal(l)
if err != nil {

@ -7,6 +7,8 @@ import (
"log"
"net/http"
"net/url"
"github.com/chubin/wttr.in/internal/types"
)
type Nominatim struct {
@ -41,6 +43,7 @@ func (n *Nominatim) Query(location string) (*Location, error) {
if err != nil {
return nil, fmt.Errorf("%s: %w", n.name, err)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
@ -49,7 +52,7 @@ func (n *Nominatim) Query(location string) (*Location, error) {
err = json.Unmarshal(body, &errResponse)
if err == nil && errResponse.Error != "" {
return nil, fmt.Errorf("%s: %s", n.name, errResponse.Error)
return nil, fmt.Errorf("%w: %s: %s", types.ErrUpstream, n.name, errResponse.Error)
}
err = json.Unmarshal(body, &result)
@ -58,9 +61,8 @@ func (n *Nominatim) Query(location string) (*Location, error) {
}
if len(result) != 1 {
return nil, fmt.Errorf("%s: invalid response", n.name)
return nil, fmt.Errorf("%w: %s: invalid response", types.ErrUpstream, n.name)
}
return &result[0], nil
}

@ -63,6 +63,7 @@ func (rl *RequestLogger) Log(r *http.Request) error {
if time.Since(rl.lastFlush) > rl.period {
return rl.flush()
}
return nil
}
@ -85,7 +86,8 @@ func (rl *RequestLogger) flush() error {
}
// Open log file.
f, err := os.OpenFile(rl.filename, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
//nolint:nosnakecase
f, err := os.OpenFile(rl.filename, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0o600)
if err != nil {
return err
}

@ -31,11 +31,15 @@ func NewLogSuppressor(filename string, suppress []string, linePrefix string) *Lo
// Open opens log file.
func (ls *LogSuppressor) Open() error {
var err error
if ls.filename == "" {
return nil
}
var err error
ls.logFile, err = os.OpenFile(ls.filename, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
//nolint:nosnakecase
ls.logFile, err = os.OpenFile(ls.filename, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0o600)
return err
}
@ -44,15 +48,14 @@ func (ls *LogSuppressor) Close() error {
if ls.filename == "" {
return nil
}
return ls.logFile.Close()
}
// Write writes p to log, and returns number f bytes written.
// Implements io.Writer interface.
func (ls *LogSuppressor) Write(p []byte) (n int, err error) {
var (
output string
)
func (ls *LogSuppressor) Write(p []byte) (int, error) {
var output string
if ls.filename == "" {
return os.Stdin.Write(p)
@ -63,19 +66,19 @@ func (ls *LogSuppressor) Write(p []byte) (n int, err error) {
lines := strings.Split(string(p), ls.linePrefix)
for _, line := range lines {
if (func() bool {
if (func(line string) bool {
for _, suppress := range ls.suppress {
if strings.Contains(line, suppress) {
return true
}
}
return false
})() {
})(line) {
continue
}
output += line
}
n, err = ls.logFile.Write([]byte(output))
return n, err
return ls.logFile.Write([]byte(output))
}

@ -9,38 +9,49 @@ import (
"github.com/robfig/cron"
)
func (rp *RequestProcessor) startPeakHandling() {
func (rp *RequestProcessor) startPeakHandling() error {
var err error
c := cron.New()
// cronTime := fmt.Sprintf("%d,%d * * * *", 30-prefetchInterval/60, 60-prefetchInterval/60)
c.AddFunc(
err = c.AddFunc(
"24 * * * *",
func() { rp.prefetchPeakRequests(&rp.peakRequest30) },
)
c.AddFunc(
if err != nil {
return err
}
err = c.AddFunc(
"54 * * * *",
func() { rp.prefetchPeakRequests(&rp.peakRequest60) },
)
if err != nil {
return err
}
c.Start()
return nil
}
func (rp *RequestProcessor) savePeakRequest(cacheDigest string, r *http.Request) {
_, min, _ := time.Now().Clock()
if min == 30 {
if _, min, _ := time.Now().Clock(); min == 30 {
rp.peakRequest30.Store(cacheDigest, *r)
} else if min == 0 {
rp.peakRequest60.Store(cacheDigest, *r)
}
}
func (rp *RequestProcessor) prefetchRequest(r *http.Request) {
rp.ProcessRequest(r)
func (rp *RequestProcessor) prefetchRequest(r *http.Request) error {
_, err := rp.ProcessRequest(r)
return err
}
func syncMapLen(sm *sync.Map) int {
count := 0
f := func(key, value interface{}) bool {
// Not really certain about this part, don't know for sure
// if this is a good check for an entry's existence
if key == "" {
@ -65,10 +76,14 @@ func (rp *RequestProcessor) prefetchPeakRequests(peakRequestMap *sync.Map) {
sleepBetweenRequests := time.Duration(rp.config.Uplink.PrefetchInterval*1000/peakRequestLen) * time.Millisecond
peakRequestMap.Range(func(key interface{}, value interface{}) bool {
go func(r http.Request) {
rp.prefetchRequest(&r)
err := rp.prefetchRequest(&r)
if err != nil {
log.Println("prefetch request:", err)
}
}(value.(http.Request))
peakRequestMap.Delete(key)
time.Sleep(sleepBetweenRequests)
return true
})
}

@ -22,19 +22,21 @@ import (
)
// plainTextAgents contains signatures of the plain-text agents.
var plainTextAgents = []string{
"curl",
"httpie",
"lwp-request",
"wget",
"python-httpx",
"python-requests",
"openbsd ftp",
"powershell",
"fetch",
"aiohttp",
"http_get",
"xh",
func plainTextAgents() []string {
return []string{
"curl",
"httpie",
"lwp-request",
"wget",
"python-httpx",
"python-requests",
"openbsd ftp",
"powershell",
"fetch",
"aiohttp",
"http_get",
"xh",
}
}
type responseWithHeader struct {
@ -99,8 +101,8 @@ func NewRequestProcessor(config *config.Config) (*RequestProcessor, error) {
}
// Start starts async request processor jobs, such as peak handling.
func (rp *RequestProcessor) Start() {
rp.startPeakHandling()
func (rp *RequestProcessor) Start() error {
return rp.startPeakHandling()
}
func (rp *RequestProcessor) ProcessRequest(r *http.Request) (*responseWithHeader, error) {
@ -124,11 +126,13 @@ func (rp *RequestProcessor) ProcessRequest(r *http.Request) (*responseWithHeader
if resp, ok := redirectInsecure(r); ok {
rp.stats.Inc("redirects")
return resp, nil
}
if dontCache(r) {
rp.stats.Inc("uncached")
return get(r, rp.upstreamTransport)
}
@ -193,11 +197,11 @@ func (rp *RequestProcessor) ProcessRequest(r *http.Request) (*responseWithHeader
rp.lruCache.Remove(cacheDigest)
}
}
return response, nil
}
func get(req *http.Request, transport *http.Transport) (*responseWithHeader, error) {
client := &http.Client{
Transport: transport,
}
@ -226,6 +230,7 @@ func get(req *http.Request, transport *http.Transport) (*responseWithHeader, err
if err != nil {
return nil, err
}
defer res.Body.Close()
body, err := ioutil.ReadAll(res.Body)
if err != nil {
@ -241,9 +246,8 @@ func get(req *http.Request, transport *http.Transport) (*responseWithHeader, err
}, nil
}
// implementation of the cache.get_signature of original wttr.in
// getCacheDigest is an implementation of the cache.get_signature of original wttr.in.
func getCacheDigest(req *http.Request) string {
userAgent := req.Header.Get("User-Agent")
queryHost := req.Host
@ -256,11 +260,11 @@ func getCacheDigest(req *http.Request) string {
return fmt.Sprintf("%s:%s%s:%s:%s", userAgent, queryHost, queryString, clientIPAddress, lang)
}
// return true if request should not be cached
// dontCache returns true if req should not be cached.
func dontCache(req *http.Request) bool {
// dont cache cyclic requests
loc := strings.Split(req.RequestURI, "?")[0]
return strings.Contains(loc, ":")
}
@ -269,10 +273,7 @@ func dontCache(req *http.Request) bool {
//
// Insecure queries are marked by the frontend web server
// with X-Forwarded-Proto header:
//
// proxy_set_header X-Forwarded-Proto $scheme;
//
//
// `proxy_set_header X-Forwarded-Proto $scheme;`.
func redirectInsecure(req *http.Request) (*responseWithHeader, bool) {
if isPlainTextAgent(req.Header.Get("User-Agent")) {
return nil, false
@ -304,14 +305,15 @@ The document has moved
}, true
}
// isPlainTextAgent returns true if userAgent is a plain-text agent
// isPlainTextAgent returns true if userAgent is a plain-text agent.
func isPlainTextAgent(userAgent string) bool {
userAgentLower := strings.ToLower(userAgent)
for _, signature := range plainTextAgents {
for _, signature := range plainTextAgents() {
if strings.Contains(userAgentLower, signature) {
return true
}
}
return false
}
@ -325,6 +327,7 @@ func ipFromAddr(s string) string {
if pos == -1 {
return s
}
return s[:pos]
}
@ -336,5 +339,4 @@ func fromCadre(cadre *routing.Cadre) *responseWithHeader {
StatusCode: 200,
InProgress: false,
}
}

@ -56,6 +56,7 @@ func (r *Router) Route(req *http.Request) Handler {
return re.Handler
}
}
return nil
}

@ -36,6 +36,7 @@ func (c *Stats) Inc(key string) {
func (c *Stats) Get(key string) int {
c.m.Lock()
defer c.m.Unlock()
return c.v[key]
}
@ -45,14 +46,13 @@ func (c *Stats) Reset(key string) int {
defer c.m.Unlock()
result := c.v[key]
c.v[key] = 0
return result
}
// Show returns current statistics formatted as []byte.
func (c *Stats) Show() []byte {
var (
b bytes.Buffer
)
var b bytes.Buffer
c.m.Lock()
defer c.m.Unlock()
@ -63,11 +63,13 @@ func (c *Stats) Show() []byte {
fmt.Fprintf(&b, "%-20s: %d\n", "Uptime (min)", uptime/60)
fmt.Fprintf(&b, "%-20s: %d\n", "Total queries", c.v["total"])
if uptime != 0 {
fmt.Fprintf(&b, "%-20s: %d\n", "Throughput (QpM)", c.v["total"]*60/int(uptime))
}
fmt.Fprintf(&b, "%-20s: %d\n", "Cache L1 queries", c.v["cache1"])
if c.v["total"] != 0 {
fmt.Fprintf(&b, "%-20s: %d\n", "Cache L1 queries (%)", (100*c.v["cache1"])/c.v["total"])
}

@ -0,0 +1,9 @@
package types
import "errors"
var (
ErrNotFound = errors.New("cache entry not found")
ErrInvalidCacheEntry = errors.New("invalid cache entry format")
ErrUpstream = errors.New("upstream error")
)

@ -1,15 +1,8 @@
package types
import "errors"
type CacheType string
const (
CacheTypeDB = "db"
CacheTypeFiles = "files"
)
var (
ErrNotFound = errors.New("cache entry not found")
ErrInvalidCacheEntry = errors.New("invalid cache entry format")
)

@ -21,5 +21,6 @@ func ReadUserIP(r *http.Request) string {
log.Printf("ERROR: userip: %q is not IP:port\n", IPAddress)
}
}
return IPAddress
}

@ -10,5 +10,6 @@ import (
func YamlUnmarshalStrict(in []byte, out interface{}) error {
dec := yaml.NewDecoder(bytes.NewReader(in))
dec.KnownFields(true)
return dec.Decode(out)
}

@ -18,15 +18,15 @@ import (
"github.com/chubin/wttr.in/internal/processor"
)
//nolint:gochecknoglobals
var cli struct {
ConfigCheck bool `name:"config-check" help:"Check configuration"`
ConfigDump bool `name:"config-dump" help:"Dump configuration"`
GeoResolve string `name:"geo-resolve" help:"Resolve location"`
ConfigFile string `name:"config-file" arg:"" optional:"" help:"Name of configuration file"`
ConvertGeoIPCache bool `name:"convert-geo-ip-cache" help:"Convert Geo IP data cache to SQlite"`
ConvertGeoLocationCache bool `name:"convert-geo-location-cache" help:"Convert Geo Location data cache to SQlite"`
ConfigCheck bool `name:"config-check" help:"Check configuration"`
ConfigDump bool `name:"config-dump" help:"Dump configuration"`
ConvertGeoIPCache bool `name:"convert-geo-ip-cache" help:"Convert Geo IP data cache to SQlite"`
ConvertGeoLocationCache bool `name:"convert-geo-location-cache" help:"Convert Geo Location data cache to SQlite"`
GeoResolve string `name:"geo-resolve" help:"Resolve location"`
}
const logLineStart = "LOG_LINE_START "
@ -84,7 +84,7 @@ func serve(conf *config.Config) error {
rp *processor.RequestProcessor
// errs is the servers errors channel.
errs chan error = make(chan error, 1)
errs = make(chan error, 1)
// numberOfServers started. If 0, exit.
numberOfServers int
@ -110,15 +110,18 @@ func serve(conf *config.Config) error {
rp, err = processor.NewRequestProcessor(conf)
if err != nil {
log.Fatalln("log processor initialization:", err)
return fmt.Errorf("log processor initialization: %w", err)
}
err = errorsLog.Open()
if err != nil {
log.Fatalln("errors log:", err)
return err
}
rp.Start()
err = rp.Start()
if err != nil {
return err
}
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
if err := logger.Log(r); err != nil {
@ -128,17 +131,22 @@ func serve(conf *config.Config) error {
response, err := rp.ProcessRequest(r)
if err != nil {
log.Println(err)
return
}
if response.StatusCode == 0 {
log.Println("status code 0", response)
return
}
copyHeader(w.Header(), response.Header)
w.Header().Set("Access-Control-Allow-Origin", "*")
w.WriteHeader(response.StatusCode)
w.Write(response.Body)
_, err = w.Write(response.Body)
if err != nil {
log.Println(err)
}
})
if conf.Server.PortHTTP != 0 {
@ -152,6 +160,7 @@ func serve(conf *config.Config) error {
if numberOfServers == 0 {
return errors.New("no servers configured")
}
return <-errs // block until one of the servers writes an error
}
@ -185,7 +194,9 @@ func main() {
if err != nil {
ctx.FatalIfErrorf(err)
}
ctx.FatalIfErrorf(geoIPCache.ConvertCache())
return
}
@ -194,7 +205,9 @@ func main() {
if err != nil {
ctx.FatalIfErrorf(err)
}
ctx.FatalIfErrorf(geoLocCache.ConvertCache())
return
}
@ -204,7 +217,6 @@ func main() {
ctx.FatalIfErrorf(err)
if loc != nil {
fmt.Println(*loc)
}
}

Loading…
Cancel
Save