From fd43c61bd128ad77b22db0537a9a4eb58490b0b5 Mon Sep 17 00:00:00 2001 From: Ben Morrison Date: Wed, 5 Jun 2019 15:36:23 -0400 Subject: moved bulk of code to its own package to clean up source tree --- svc/cache.go | 130 +++++++++++ svc/cache_test.go | 32 +++ svc/db.go | 135 ++++++++++++ svc/db_test.go | 92 ++++++++ svc/go.mod | 13 ++ svc/go.sum | 163 ++++++++++++++ svc/handlers.go | 228 ++++++++++++++++++++ svc/handlers_test.go | 116 ++++++++++ svc/http.go | 82 +++++++ svc/http_test.go | 47 ++++ svc/init.go | 600 +++++++++++++++++++++++++++++++++++++++++++++++++++ svc/init_test.go | 27 +++ svc/post.go | 65 ++++++ svc/post_test.go | 77 +++++++ svc/query.go | 146 +++++++++++++ svc/query_test.go | 75 +++++++ svc/svc.go | 120 +++++++++++ svc/types.go | 67 ++++++ 18 files changed, 2215 insertions(+) create mode 100644 svc/cache.go create mode 100644 svc/cache_test.go create mode 100644 svc/db.go create mode 100644 svc/db_test.go create mode 100644 svc/go.mod create mode 100644 svc/go.sum create mode 100644 svc/handlers.go create mode 100644 svc/handlers_test.go create mode 100644 svc/http.go create mode 100644 svc/http_test.go create mode 100644 svc/init.go create mode 100644 svc/init_test.go create mode 100644 svc/post.go create mode 100644 svc/post_test.go create mode 100644 svc/query.go create mode 100644 svc/query_test.go create mode 100644 svc/svc.go create mode 100644 svc/types.go (limited to 'svc') diff --git a/svc/cache.go b/svc/cache.go new file mode 100644 index 0000000..ce3813c --- /dev/null +++ b/svc/cache.go @@ -0,0 +1,130 @@ +package svc // import "github.com/getwtxt/getwtxt/svc" + +import ( + "bytes" + "io/ioutil" + "log" + "os" + "time" +) + +func checkCacheTime() bool { + confObj.Mu.RLock() + answer := time.Since(confObj.LastCache) > confObj.CacheInterval + confObj.Mu.RUnlock() + + return answer +} + +// Launched by init as a coroutine to watch +// for the update intervals to pass. +func cacheAndPush() { + for { + if checkCacheTime() { + refreshCache() + } + if checkDBtime() { + if err := pushDatabase(); err != nil { + log.Printf("Error pushing cache to database: %v\n", err.Error()) + } + } + time.Sleep(1000 * time.Millisecond) + } +} + +func refreshCache() { + + // This clusterfuck of mutex read locks is + // necessary to avoid deadlock. This mess + // also avoids a panic that would occur + // should twtxtCache be written to during + // this loop. + twtxtCache.Mu.RLock() + for k := range twtxtCache.Users { + twtxtCache.Mu.RUnlock() + err := twtxtCache.UpdateUser(k) + if err != nil { + log.Printf("%v\n", err.Error()) + } + twtxtCache.Mu.RLock() + } + twtxtCache.Mu.RUnlock() + + remoteRegistries.Mu.RLock() + for _, v := range remoteRegistries.List { + err := twtxtCache.CrawlRemoteRegistry(v) + if err != nil { + log.Printf("Error while refreshing local copy of remote registry user data: %v\n", err.Error()) + } + } + remoteRegistries.Mu.RUnlock() + confObj.Mu.Lock() + confObj.LastCache = time.Now() + confObj.Mu.Unlock() +} + +// pingAssets checks if the local static assets +// need to be re-cached. If they do, they are +// pulled back into memory from disk. +func pingAssets() { + + confObj.Mu.RLock() + assetsDir := confObj.AssetsDir + confObj.Mu.RUnlock() + + cssStat, err := os.Stat(assetsDir + "/style.css") + if err != nil { + log.Printf("%v\n", err.Error()) + } + + indexStat, err := os.Stat(assetsDir + "/tmpl/index.html") + if err != nil { + log.Printf("%v\n", err.Error()) + } + + indexMod := staticCache.indexMod + cssMod := staticCache.cssMod + + if !indexMod.Equal(indexStat.ModTime()) { + tmpls = initTemplates() + + var b []byte + buf := bytes.NewBuffer(b) + + confObj.Mu.RLock() + err = tmpls.ExecuteTemplate(buf, "index.html", confObj.Instance) + confObj.Mu.RUnlock() + if err != nil { + log.Printf("%v\n", err.Error()) + } + + staticCache.index = buf.Bytes() + staticCache.indexMod = indexStat.ModTime() + } + + if !cssMod.Equal(cssStat.ModTime()) { + + css, err := ioutil.ReadFile(assetsDir + "/style.css") + if err != nil { + log.Printf("%v\n", err.Error()) + } + + staticCache.css = css + staticCache.cssMod = cssStat.ModTime() + } +} + +// Simple function to deduplicate entries in a []string +func dedupe(list []string) []string { + var out []string + var seen = map[string]bool{} + + for _, e := range list { + if !seen[e] { + out = append(out, e) + seen[e] = true + } + } + + return out +} diff --git a/svc/cache_test.go b/svc/cache_test.go new file mode 100644 index 0000000..d3dba29 --- /dev/null +++ b/svc/cache_test.go @@ -0,0 +1,32 @@ +package svc // import github.com/getwtxt/getwtxt/svc + +import ( + "testing" +) + +func Test_refreshCache(t *testing.T) { + initTestConf() + confObj.Mu.RLock() + prevtime := confObj.LastCache + confObj.Mu.RUnlock() + + t.Run("Cache Time Check", func(t *testing.T) { + refreshCache() + confObj.Mu.RLock() + newtime := confObj.LastCache + confObj.Mu.RUnlock() + + if !newtime.After(prevtime) || newtime == prevtime { + t.Errorf("Cache time did not update, check refreshCache() logic\n") + } + }) +} + +func Benchmark_refreshCache(b *testing.B) { + initTestConf() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + refreshCache() + } +} diff --git a/svc/db.go b/svc/db.go new file mode 100644 index 0000000..8c3f278 --- /dev/null +++ b/svc/db.go @@ -0,0 +1,135 @@ +package svc // import "github.com/getwtxt/getwtxt/svc" + +import ( + "log" + "net" + "strings" + "time" + + "github.com/getwtxt/registry" + "github.com/syndtr/goleveldb/leveldb" +) + +func checkDBtime() bool { + confObj.Mu.RLock() + answer := time.Since(confObj.LastPush) > confObj.DBInterval + confObj.Mu.RUnlock() + + return answer +} + +// Pushes the registry's cache data to a local +// database for safe keeping. +func pushDatabase() error { + db := <-dbChan + dbChan <- db + + return db.push() +} + +func pullDatabase() { + db := <-dbChan + dbChan <- db + db.pull() +} + +func (lvl dbLevel) push() error { + twtxtCache.Mu.RLock() + var dbBasket = &leveldb.Batch{} + for k, v := range twtxtCache.Users { + dbBasket.Put([]byte(k+"*Nick"), []byte(v.Nick)) + dbBasket.Put([]byte(k+"*URL"), []byte(v.URL)) + dbBasket.Put([]byte(k+"*IP"), []byte(v.IP.String())) + dbBasket.Put([]byte(k+"*Date"), []byte(v.Date)) + for i, e := range v.Status { + rfc := i.Format(time.RFC3339) + dbBasket.Put([]byte(k+"*Status*"+rfc), []byte(e)) + } + } + twtxtCache.Mu.RUnlock() + + remoteRegistries.Mu.RLock() + for k, v := range remoteRegistries.List { + dbBasket.Put([]byte("remote*"+string(k)), []byte(v)) + } + remoteRegistries.Mu.RUnlock() + + if err := lvl.db.Write(dbBasket, nil); err != nil { + return err + } + + confObj.Mu.Lock() + confObj.LastPush = time.Now() + confObj.Mu.Unlock() + + return nil +} + +func (lvl dbLevel) pull() { + + iter := lvl.db.NewIterator(nil, nil) + + for iter.Next() { + key := string(iter.Key()) + val := string(iter.Value()) + + split := strings.Split(key, "*") + urls := split[0] + field := split[1] + + if urls == "remote" { + remoteRegistries.Mu.Lock() + remoteRegistries.List = append(remoteRegistries.List, val) + remoteRegistries.Mu.Unlock() + continue + } + + data := registry.NewUser() + twtxtCache.Mu.RLock() + if _, ok := twtxtCache.Users[urls]; ok { + data = twtxtCache.Users[urls] + } + twtxtCache.Mu.RUnlock() + + switch field { + case "IP": + data.IP = net.ParseIP(val) + case "Nick": + data.Nick = val + case "URL": + data.URL = val + case "Date": + data.Date = val + case "Status": + thetime, err := time.Parse(time.RFC3339, split[2]) + if err != nil { + log.Printf("%v\n", err.Error()) + } + data.Status[thetime] = val + } + + twtxtCache.Mu.Lock() + twtxtCache.Users[urls] = data + twtxtCache.Mu.Unlock() + + } + + remoteRegistries.Mu.Lock() + remoteRegistries.List = dedupe(remoteRegistries.List) + remoteRegistries.Mu.Unlock() + + iter.Release() + err := iter.Error() + if err != nil { + log.Printf("Error while pulling DB into registry cache: %v\n", err.Error()) + } +} + +func (lite dbSqlite) push() error { + + return nil +} + +func (lite dbSqlite) pull() { + +} diff --git a/svc/db_test.go b/svc/db_test.go new file mode 100644 index 0000000..830c7a6 --- /dev/null +++ b/svc/db_test.go @@ -0,0 +1,92 @@ +package svc // import github.com/getwtxt/getwtxt/svc + +import ( + "net" + "testing" + + "github.com/getwtxt/registry" +) + +/* +func Test_pushpullDatabase(t *testing.T) { + initTestConf() + initDatabase() + out, _, err := registry.GetTwtxt("https://gbmor.dev/twtxt.txt") + if err != nil { + t.Errorf("Couldn't set up test: %v\n", err) + } + statusmap, err := registry.ParseUserTwtxt(out, "gbmor", "https://gbmor.dev/twtxt.txt") + if err != nil { + t.Errorf("Couldn't set up test: %v\n", err) + } + twtxtCache.AddUser("gbmor", "https://gbmor.dev/twtxt.txt", "", net.ParseIP("127.0.0.1"), statusmap) + remoteRegistries.Mu.Lock() + remoteRegistries.List = append(remoteRegistries.List, "https://twtxt.tilde.institute/api/plain/users") + remoteRegistries.Mu.Unlock() + + t.Run("Push to Database", func(t *testing.T) { + err := pushDatabase() + if err != nil { + t.Errorf("%v\n", err) + } + }) + + t.Run("Clearing Registry", func(t *testing.T) { + err := twtxtCache.DelUser("https://gbmor.dev/twtxt.txt") + if err != nil { + t.Errorf("%v", err) + } + }) + + t.Run("Pulling from Database", func(t *testing.T) { + pullDatabase() + twtxtCache.Mu.RLock() + if _, ok := twtxtCache.Users["https://gbmor.dev/twtxt.txt"]; !ok { + t.Errorf("Missing user previously pushed to database\n") + } + twtxtCache.Mu.RUnlock() + + }) +} +*/ +func Benchmark_pushDatabase(b *testing.B) { + initTestConf() + + if len(dbChan) < 1 { + initDatabase() + } + + if _, ok := twtxtCache.Users["https://gbmor.dev/twtxt.txt"]; !ok { + out, _, err := registry.GetTwtxt("https://gbmor.dev/twtxt.txt") + if err != nil { + b.Errorf("Couldn't set up benchmark: %v\n", err) + } + + statusmap, err := registry.ParseUserTwtxt(out, "gbmor", "https://gbmor.dev/twtxt.txt") + if err != nil { + b.Errorf("Couldn't set up benchmark: %v\n", err) + } + + twtxtCache.AddUser("gbmor", "https://gbmor.dev/twtxt.txt", "", net.ParseIP("127.0.0.1"), statusmap) + } + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + err := pushDatabase() + if err != nil { + b.Errorf("%v\n", err) + } + } +} +func Benchmark_pullDatabase(b *testing.B) { + initTestConf() + + if len(dbChan) < 1 { + initDatabase() + } + + for i := 0; i < b.N; i++ { + pullDatabase() + } +} diff --git a/svc/go.mod b/svc/go.mod new file mode 100644 index 0000000..b6ac3e3 --- /dev/null +++ b/svc/go.mod @@ -0,0 +1,13 @@ +module github.com/getwtxt/getwtxt/svc + +go 1.12 + +require ( + github.com/fsnotify/fsnotify v1.4.7 + github.com/getwtxt/registry v0.2.3 + github.com/gorilla/handlers v1.4.0 + github.com/gorilla/mux v1.7.2 + github.com/spf13/pflag v1.0.3 + github.com/spf13/viper v1.4.0 + github.com/syndtr/goleveldb v1.0.0 +) diff --git a/svc/go.sum b/svc/go.sum new file mode 100644 index 0000000..6c5bcf6 --- /dev/null +++ b/svc/go.sum @@ -0,0 +1,163 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/getwtxt/registry v0.2.3 h1:gOc6fSBljD/6QKwr+UXtY2bPepsifVei49XQaR5i1mk= +github.com/getwtxt/registry v0.2.3/go.mod h1:BGSIALOFqIRj+ACLB8etWGUOgFAKN8oFDpCsw6YOdYQ= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/gorilla/handlers v1.4.0 h1:XulKRWSQK5uChr4pEgSE4Tc/OcmnU9GJuSwdog/tZsA= +github.com/gorilla/handlers v1.4.0/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= +github.com/gorilla/mux v1.7.2 h1:zoNxOV7WjqXptQOVngLmcSQgXmgk4NMz1HibBchjl/I= +github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= +github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/svc/handlers.go b/svc/handlers.go new file mode 100644 index 0000000..45f3022 --- /dev/null +++ b/svc/handlers.go @@ -0,0 +1,228 @@ +package svc // import "github.com/getwtxt/getwtxt/svc" + +import ( + "crypto/sha256" + "fmt" + "net/http" + "strconv" + "strings" + + "github.com/getwtxt/registry" + "github.com/gorilla/mux" +) + +// handles "/" +func indexHandler(w http.ResponseWriter, r *http.Request) { + + pingAssets() + + etag := fmt.Sprintf("%x", sha256.Sum256([]byte(staticCache.indexMod.String()))) + + // Take the hex-encoded sha256 sum of the index template's mod time + // to send as an ETag. If an error occurred when grabbing the file info, + // the ETag will be empty. + w.Header().Set("ETag", "\""+etag+"\"") + w.Header().Set("Content-Type", htmlutf8) + + _, err := w.Write(staticCache.index) + if err != nil { + log500(w, r, err) + return + } + + log200(r) +} + +// handles "/api" +func apiBaseHandler(w http.ResponseWriter, r *http.Request) { + indexHandler(w, r) +} + +// handles "/api/plain" +// maybe add json/xml support later +func apiFormatHandler(w http.ResponseWriter, r *http.Request) { + indexHandler(w, r) +} + +func apiAllTweetsHandler(w http.ResponseWriter, r *http.Request) { + out, err := twtxtCache.QueryAllStatuses() + if err != nil { + log500(w, r, err) + } + + data := parseQueryOut(out) + if err != nil { + data = []byte("") + } + + etag := fmt.Sprintf("%x", sha256.Sum256(data)) + w.Header().Set("ETag", etag) + w.Header().Set("Content-Type", txtutf8) + + _, err = w.Write(data) + if err != nil { + log500(w, r, err) + return + } + + log200(r) +} + +// handles "/api/plain/(users|mentions|tweets)" +func apiEndpointHandler(w http.ResponseWriter, r *http.Request) { + + err := r.ParseForm() + if err != nil { + log500(w, r, err) + return + } + + if r.FormValue("q") != "" || r.FormValue("url") != "" { + err := apiEndpointQuery(w, r) + if err != nil { + log500(w, r, err) + return + } + log200(r) + return + } + + page := 1 + pageVal := r.FormValue("page") + if pageVal != "" { + page, err = strconv.Atoi(pageVal) + if err != nil || page == 0 { + page = 1 + } + } + + // if there's no query, return everything in + // registry for a given endpoint + var out []string + switch r.URL.Path { + case "/api/plain/users": + out, err = twtxtCache.QueryUser("") + out = registry.ReduceToPage(page, out) + + case "/api/plain/mentions": + out, err = twtxtCache.QueryInStatus("@<") + out = registry.ReduceToPage(page, out) + + default: + out, err = twtxtCache.QueryAllStatuses() + out = registry.ReduceToPage(page, out) + } + + data := parseQueryOut(out) + if err != nil { + data = []byte("") + } + + etag := fmt.Sprintf("%x", sha256.Sum256(data)) + w.Header().Set("ETag", etag) + w.Header().Set("Content-Type", txtutf8) + + _, err = w.Write(data) + if err != nil { + log500(w, r, err) + return + } + + log200(r) +} + +// handles POST for "/api/plain/users" +func apiEndpointPOSTHandler(w http.ResponseWriter, r *http.Request) { + apiPostUser(w, r) +} + +// handles "/api/plain/tags" +func apiTagsBaseHandler(w http.ResponseWriter, r *http.Request) { + + out, err := twtxtCache.QueryInStatus("#") + if err != nil { + log500(w, r, err) + return + } + + out = registry.ReduceToPage(1, out) + data := parseQueryOut(out) + + etag := fmt.Sprintf("%x", sha256.Sum256(data)) + w.Header().Set("ETag", etag) + w.Header().Set("Content-Type", txtutf8) + + _, err = w.Write(data) + if err != nil { + log500(w, r, err) + return + } + + log200(r) +} + +// handles "/api/plain/tags/[a-zA-Z0-9]+" +func apiTagsHandler(w http.ResponseWriter, r *http.Request) { + + vars := mux.Vars(r) + tags := vars["tags"] + + tags = strings.ToLower(tags) + out, err := twtxtCache.QueryInStatus("#" + tags) + if err != nil { + log500(w, r, err) + return + } + tags = strings.Title(tags) + out2, err := twtxtCache.QueryInStatus("#" + tags) + if err != nil { + log500(w, r, err) + return + } + tags = strings.ToUpper(tags) + out3, err := twtxtCache.QueryInStatus("#" + tags) + if err != nil { + log500(w, r, err) + return + } + + out = append(out, out2...) + out = append(out, out3...) + out = uniq(out) + + out = registry.ReduceToPage(1, out) + data := parseQueryOut(out) + + etag := fmt.Sprintf("%x", sha256.Sum256(data)) + w.Header().Set("ETag", etag) + w.Header().Set("Content-Type", txtutf8) + + _, err = w.Write(data) + if err != nil { + log500(w, r, err) + return + } + + log200(r) +} + +// Serving the stylesheet virtually because +// files aren't served directly in getwtxt. +func cssHandler(w http.ResponseWriter, r *http.Request) { + + // Sending the sha256 sum of the modtime in hexadecimal for the ETag header + etag := fmt.Sprintf("%x", sha256.Sum256([]byte(staticCache.cssMod.String()))) + + w.Header().Set("ETag", "\""+etag+"\"") + w.Header().Set("Content-Type", cssutf8) + + pingAssets() + + n, err := w.Write(staticCache.css) + if err != nil || n == 0 { + log500(w, r, err) + return + } + + log200(r) +} diff --git a/svc/handlers_test.go b/svc/handlers_test.go new file mode 100644 index 0000000..ab9b20f --- /dev/null +++ b/svc/handlers_test.go @@ -0,0 +1,116 @@ +package svc // import github.com/getwtxt/getwtxt/svc + +import ( + "bytes" + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" +) + +// Currently, these only test for a 200 status code. +// More in-depth unit tests are planned, however, several +// of these will quickly turn into integration tests as +// they'll need more than a barebones test environment to +// get any real information. The HTTP responses are being +// tested by me by hand, mostly. +func Test_indexHandler(t *testing.T) { + initTestConf() + t.Run("indexHandler", func(t *testing.T) { + w := httptest.NewRecorder() + req := httptest.NewRequest("GET", "localhost"+testport+"/", nil) + indexHandler(w, req) + resp := w.Result() + if resp.StatusCode != http.StatusOK { + t.Errorf(fmt.Sprintf("%v", resp.StatusCode)) + } + }) +} +func Test_apiBaseHandler(t *testing.T) { + initTestConf() + t.Run("apiBaseHandler", func(t *testing.T) { + w := httptest.NewRecorder() + req := httptest.NewRequest("GET", "localhost"+testport+"/api", nil) + apiBaseHandler(w, req) + resp := w.Result() + if resp.StatusCode != http.StatusOK { + t.Errorf(fmt.Sprintf("%v", resp.StatusCode)) + } + }) +} +func Test_apiFormatHandler(t *testing.T) { + initTestConf() + t.Run("apiFormatHandler", func(t *testing.T) { + w := httptest.NewRecorder() + req := httptest.NewRequest("GET", "localhost"+testport+"/api/plain", nil) + apiFormatHandler(w, req) + resp := w.Result() + if resp.StatusCode != http.StatusOK { + t.Errorf(fmt.Sprintf("%v", resp.StatusCode)) + } + }) +} +func Test_apiEndpointHandler(t *testing.T) { + initTestConf() + t.Run("apiEndpointHandler", func(t *testing.T) { + w := httptest.NewRecorder() + req := httptest.NewRequest("GET", "localhost"+testport+"/api/plain/users", nil) + apiEndpointHandler(w, req) + resp := w.Result() + if resp.StatusCode != http.StatusOK { + t.Errorf(fmt.Sprintf("%v", resp.StatusCode)) + } + }) +} + +/* +func Test_apiTagsBaseHandler(t *testing.T) { + initTestConf() + t.Run("apiTagsBaseHandler", func(t *testing.T) { + w := httptest.NewRecorder() + req := httptest.NewRequest("GET", "localhost"+testport+"/api/plain/tags", nil) + apiTagsBaseHandler(w, req) + resp := w.Result() + if resp.StatusCode != http.StatusOK { + t.Errorf(fmt.Sprintf("%v", resp.StatusCode)) + } + }) +} +func Test_apiTagsHandler(t *testing.T) { + initTestConf() + t.Run("apiTagsHandler", func(t *testing.T) { + w := httptest.NewRecorder() + req := httptest.NewRequest("GET", "localhost"+testport+"/api/plain/tags/tag", nil) + apiTagsHandler(w, req) + resp := w.Result() + if resp.StatusCode != http.StatusOK { + t.Errorf(fmt.Sprintf("%v", resp.StatusCode)) + } + }) +} +*/ +func Test_cssHandler(t *testing.T) { + initTestConf() + + name := "CSS Handler Test" + css, err := ioutil.ReadFile("assets/style.css") + if err != nil { + t.Errorf("Couldn't read assets/style.css: %v\n", err) + } + + w := httptest.NewRecorder() + req := httptest.NewRequest("GET", "localhost"+testport+"/css", nil) + + t.Run(name, func(t *testing.T) { + cssHandler(w, req) + resp := w.Result() + body, _ := ioutil.ReadAll(resp.Body) + if resp.StatusCode != 200 { + t.Errorf("cssHandler(): %v\n", resp.StatusCode) + } + if !bytes.Equal(body, css) { + t.Errorf("cssHandler(): Byte mismatch\n") + } + }) +} diff --git a/svc/http.go b/svc/http.go new file mode 100644 index 0000000..ddf8669 --- /dev/null +++ b/svc/http.go @@ -0,0 +1,82 @@ +package svc // import "github.com/getwtxt/getwtxt/svc" + +import ( + "context" + "log" + "net" + "net/http" + "strings" +) + +// Attaches a request's IP address to the request's context. +// If getwtxt is behind a reverse proxy, get the last entry +// in the X-Forwarded-For or X-Real-IP HTTP header as the user IP. +func newCtxUserIP(ctx context.Context, r *http.Request) context.Context { + + base := strings.Split(r.RemoteAddr, ":") + uip := base[0] + + if _, ok := r.Header["X-Forwarded-For"]; ok { + proxied := r.Header["X-Forwarded-For"] + base = strings.Split(proxied[len(proxied)-1], ":") + uip = base[0] + } + + xRealIP := http.CanonicalHeaderKey("X-Real-IP") + if _, ok := r.Header[xRealIP]; ok { + proxied := r.Header[xRealIP] + base = strings.Split(proxied[len(proxied)-1], ":") + uip = base[0] + } + + return context.WithValue(ctx, ctxKey, uip) +} + +// Retrieves a request's IP address from the request's context +func getIPFromCtx(ctx context.Context) net.IP { + + uip, ok := ctx.Value(ctxKey).(string) + if !ok { + log.Printf("Couldn't retrieve IP from request\n") + } + + return net.ParseIP(uip) +} + +// Shim function to modify/pass context value to a handler +func ipMiddleware(hop http.Handler) http.Handler { + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx := newCtxUserIP(r.Context(), r) + hop.ServeHTTP(w, r.WithContext(ctx)) + }) +} + +func log200(r *http.Request) { + useragent := r.Header["User-Agent"] + + uip := getIPFromCtx(r.Context()) + log.Printf("*** %v :: 200 :: %v %v :: %v\n", uip, r.Method, r.URL, useragent) +} + +func log400(w http.ResponseWriter, r *http.Request, err string) { + uip := getIPFromCtx(r.Context()) + log.Printf("*** %v :: 400 :: %v %v :: %v\n", uip, r.Method, r.URL, err) + http.Error(w, "400 Bad Request: "+err, http.StatusBadRequest) +} + +func log404(w http.ResponseWriter, r *http.Request, err error) { + useragent := r.Header["User-Agent"] + + uip := getIPFromCtx(r.Context()) + log.Printf("*** %v :: 404 :: %v %v :: %v :: %v\n", uip, r.Method, r.URL, useragent, err.Error()) + http.Error(w, err.Error(), http.StatusNotFound) +} + +func log500(w http.ResponseWriter, r *http.Request, err error) { + useragent := r.Header["User-Agent"] + + uip := getIPFromCtx(r.Context()) + log.Printf("*** %v :: 500 :: %v %v :: %v :: %v\n", uip, r.Method, r.URL, useragent, err.Error()) + http.Error(w, err.Error(), http.StatusInternalServerError) +} diff --git a/svc/http_test.go b/svc/http_test.go new file mode 100644 index 0000000..261c738 --- /dev/null +++ b/svc/http_test.go @@ -0,0 +1,47 @@ +package svc // import github.com/getwtxt/getwtxt/svc + +import ( + "errors" + "net/http" + "net/http/httptest" + "testing" +) + +func Test_log400(t *testing.T) { + initTestConf() + t.Run("log400", func(t *testing.T) { + w := httptest.NewRecorder() + req := httptest.NewRequest("POST", "/400", nil) + log400(w, req, "400 Test") + resp := w.Result() + if resp.StatusCode != http.StatusBadRequest { + t.Errorf("Didn't receive 400, received: %v\n", resp.StatusCode) + } + }) +} + +func Test_log404(t *testing.T) { + initTestConf() + t.Run("log404", func(t *testing.T) { + w := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/404", nil) + log404(w, req, errors.New("404 Test")) + resp := w.Result() + if resp.StatusCode != http.StatusNotFound { + t.Errorf("Didn't receive 404, received: %v\n", resp.StatusCode) + } + }) +} + +func Test_log500(t *testing.T) { + initTestConf() + t.Run("log500", func(t *testing.T) { + w := httptest.NewRecorder() + req := httptest.NewRequest("POST", "/500", nil) + log500(w, req, errors.New("500 Test")) + resp := w.Result() + if resp.StatusCode != http.StatusInternalServerError { + t.Errorf("Didn't receive 500, received: %v\n", resp.StatusCode) + } + }) +} diff --git a/svc/init.go b/svc/init.go new file mode 100644 index 0000000..18fe0b6 --- /dev/null +++ b/svc/init.go @@ -0,0 +1,600 @@ +package svc // import "github.com/getwtxt/getwtxt/svc" + +import ( + "database/sql" + "fmt" + "html/template" + "log" + "os" + "os/signal" + "path/filepath" + "strings" + "time" + + "github.com/fsnotify/fsnotify" + "github.com/getwtxt/registry" + "github.com/spf13/pflag" + "github.com/spf13/viper" + "github.com/syndtr/goleveldb/leveldb" +) + +const getwtxt = "0.2.2" + +var ( + flagVersion *bool = pflag.BoolP("version", "v", false, "Display version information, then exit.") + flagHelp *bool = pflag.BoolP("help", "h", false, "Display the quick-help screen.") + flagMan *bool = pflag.BoolP("manual", "m", false, "Display the configuration manual.") + flagConfFile *string = pflag.StringP("config", "c", "", "The name/path of the configuration file you wish to use.") + flagAssets *string = pflag.StringP("assets", "a", "", "The location of the getwtxt assets directory") + flagDBPath *string = pflag.StringP("db", "d", "", "Path to the getwtxt database") + flagDBType *string = pflag.StringP("dbtype", "t", "", "Type of database being used") +) + +var confObj = &Configuration{} + +// signals to close the log file +var closeLog = make(chan bool, 1) + +// used to transmit database pointer after +// initialization +var dbChan = make(chan dbase, 1) + +var tmpls *template.Template + +var twtxtCache = registry.NewIndex() + +var remoteRegistries = &RemoteRegistries{} + +var staticCache = &struct { + index []byte + indexMod time.Time + css []byte + cssMod time.Time +}{ + index: nil, + indexMod: time.Time{}, + css: nil, + cssMod: time.Time{}, +} + +func init() { + checkFlags() + titleScreen() + initConfig() + initLogging() + tmpls = initTemplates() + initDatabase() + go cacheAndPush() + watchForInterrupt() +} + +func checkFlags() { + pflag.Parse() + if *flagVersion { + titleScreen() + os.Exit(0) + } + if *flagHelp { + titleScreen() + helpScreen() + os.Exit(0) + } + if *flagMan { + titleScreen() + helpScreen() + manualScreen() + os.Exit(0) + } +} + +func initConfig() { + + if *flagConfFile == "" { + viper.SetConfigName("getwtxt") + viper.SetConfigType("yml") + viper.AddConfigPath(".") + viper.AddConfigPath("/usr/local/getwtxt") + viper.AddConfigPath("/etc") + viper.AddConfigPath("/usr/local/etc") + + } else { + path, file := filepath.Split(*flagConfFile) + if path == "" { + path = "." + } + if file == "" { + file = *flagConfFile + } + filename := strings.Split(file, ".") + viper.SetConfigName(filename[0]) + viper.SetConfigType(filename[1]) + viper.AddConfigPath(path) + } + + log.Printf("Loading configuration ...\n") + if err := viper.ReadInConfig(); err != nil { + log.Printf("%v\n", err.Error()) + log.Printf("Using defaults ...\n") + } else { + viper.WatchConfig() + viper.OnConfigChange(func(e fsnotify.Event) { + log.Printf("Config file change detected. Reloading...\n") + rebindConfig() + }) + } + + viper.SetDefault("ListenPort", 9001) + viper.SetDefault("LogFile", "getwtxt.log") + viper.SetDefault("DatabasePath", "getwtxt.db") + viper.SetDefault("AssetsDirectory", "assets") + viper.SetDefault("DatabaseType", "leveldb") + viper.SetDefault("StdoutLogging", false) + viper.SetDefault("ReCacheInterval", "1h") + viper.SetDefault("DatabasePushInterval", "5m") + + viper.SetDefault("Instance.SiteName", "getwtxt") + viper.SetDefault("Instance.OwnerName", "Anonymous Microblogger") + viper.SetDefault("Instance.Email", "nobody@knows") + viper.SetDefault("Instance.URL", "https://twtxt.example.com") + viper.SetDefault("Instance.Description", "A fast, resilient twtxt registry server written in Go!") + + confObj.Mu.Lock() + + confObj.Port = viper.GetInt("ListenPort") + confObj.LogFile = viper.GetString("LogFile") + + if *flagDBType == "" { + confObj.DBType = strings.ToLower(viper.GetString("DatabaseType")) + } else { + confObj.DBType = *flagDBType + } + + if *flagDBPath == "" { + confObj.DBPath = viper.GetString("DatabasePath") + } else { + confObj.DBPath = *flagDBPath + } + log.Printf("Using %v database: %v\n", confObj.DBType, confObj.DBPath) + + if *flagAssets == "" { + confObj.AssetsDir = viper.GetString("AssetsDirectory") + } else { + confObj.AssetsDir = *flagAssets + } + + confObj.StdoutLogging = viper.GetBool("StdoutLogging") + if confObj.StdoutLogging { + log.Printf("Logging to stdout\n") + } else { + log.Printf("Logging to %v\n", confObj.LogFile) + } + + confObj.CacheInterval = viper.GetDuration("StatusFetchInterval") + log.Printf("User status fetch interval: %v\n", confObj.CacheInterval) + + confObj.DBInterval = viper.GetDuration("DatabasePushInterval") + log.Printf("Database push interval: %v\n", confObj.DBInterval) + + confObj.LastCache = time.Now() + confObj.LastPush = time.Now() + confObj.Version = getwtxt + + confObj.Instance.Vers = getwtxt + confObj.Instance.Name = viper.GetString("Instance.SiteName") + confObj.Instance.URL = viper.GetString("Instance.URL") + confObj.Instance.Owner = viper.GetString("Instance.OwnerName") + confObj.Instance.Mail = viper.GetString("Instance.Email") + confObj.Instance.Desc = viper.GetString("Instance.Description") + + confObj.Mu.Unlock() + +} + +func initLogging() { + + confObj.Mu.RLock() + + if confObj.StdoutLogging { + log.SetOutput(os.Stdout) + + } else { + + logfile, err := os.OpenFile(confObj.LogFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) + if err != nil { + log.Printf("Could not open log file: %v\n", err.Error()) + } + + // Listen for the signal to close the log file + // in a separate thread. Passing it as an argument + // to prevent race conditions when the config is + // reloaded. + go func(logfile *os.File) { + + <-closeLog + log.Printf("Closing log file ...\n") + + err = logfile.Close() + if err != nil { + log.Printf("Couldn't close log file: %v\n", err.Error()) + } + }(logfile) + + log.SetOutput(logfile) + } + + confObj.Mu.RUnlock() +} + +func rebindConfig() { + + confObj.Mu.RLock() + if !confObj.StdoutLogging { + closeLog <- true + } + confObj.Mu.RUnlock() + + confObj.Mu.Lock() + + confObj.LogFile = viper.GetString("LogFile") + confObj.DBType = strings.ToLower(viper.GetString("DatabaseType")) + confObj.DBPath = viper.GetString("DatabasePath") + confObj.StdoutLogging = viper.GetBool("StdoutLogging") + confObj.CacheInterval = viper.GetDuration("StatusFetchInterval") + confObj.DBInterval = viper.GetDuration("DatabasePushInterval") + + confObj.Instance.Name = viper.GetString("Instance.SiteName") + confObj.Instance.URL = viper.GetString("Instance.URL") + confObj.Instance.Owner = viper.GetString("Instance.OwnerName") + confObj.Instance.Mail = viper.GetString("Instance.Email") + confObj.Instance.Desc = viper.GetString("Instance.Description") + + confObj.Mu.Unlock() + + initLogging() +} + +func initTemplates() *template.Template { + confObj.Mu.RLock() + assetsDir := confObj.AssetsDir + confObj.Mu.RUnlock() + + return template.Must(template.ParseFiles(assetsDir + "/tmpl/index.html")) +} + +// Pull DB data into cache, if available. +func initDatabase() { + var db dbase + var err error + + confObj.Mu.RLock() + switch confObj.DBType { + + case "leveldb": + var lvl *leveldb.DB + lvl, err = leveldb.OpenFile(confObj.DBPath, nil) + db = &dbLevel{db: lvl} + + case "sqlite": + var lite *sql.DB + db = &dbSqlite{db: lite} + + } + confObj.Mu.RUnlock() + + if err != nil { + log.Fatalf("%v\n", err.Error()) + } + + dbChan <- db + + pullDatabase() +} + +// Watch for SIGINT aka ^C +// Close the log file then exit +func watchForInterrupt() { + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt) + + go func() { + for sigint := range c { + + log.Printf("\n\nCaught %v. Cleaning up ...\n", sigint) + confObj.Mu.RLock() + log.Printf("Closing database connection to %v...\n", confObj.DBPath) + + db := <-dbChan + + switch dbType := db.(type) { + + case *dbLevel: + lvl := dbType + if err := lvl.db.Close(); err != nil { + log.Printf("%v\n", err.Error()) + } + + } + + if !confObj.StdoutLogging { + closeLog <- true + } + + confObj.Mu.RUnlock() + close(dbChan) + close(closeLog) + + // Let everything catch up + time.Sleep(100 * time.Millisecond) + os.Exit(0) + } + }() +} + +func titleScreen() { + fmt.Printf(` + + _ _ _ + __ _ ___| |___ _| |___ _| |_ + / _ |/ _ \ __\ \ /\ / / __\ \/ / __| + | (_| | __/ |_ \ V V /| |_ > <| |_ + \__, |\___|\__| \_/\_/ \__/_/\_\\__| + |___/ + version ` + getwtxt + ` + github.com/getwtxt/getwtxt + GPL v3 + +`) +} + +func helpScreen() { + fmt.Printf(` + getwtxt Help + + + :: Command Line Options :: + + Command line options are used to explicitly override defaults, + or what has been specified in the configuration file. + + -h [--help] Print this help screen. + -m [--manual] Print the manual. + -v [--version] Print the version information and quit. + -c [--config] Path to an alternate configuration file + to use. May be relative or absolute. + -a [--assets] Path to the assets directory, containing + style.css and tmpl/index.html + -d [--db] Path getwtxt should use for the database. + -t [--dbtype] Type of database to use. + Options: leveldb + +`) +} +func manualScreen() { + fmt.Printf(` + :: Sections :: + + >> Configuration File + Covers syntax and location of default configuration, + passing a specific configuration file to getwtxt, + and acceptable formats for configuration files. + + >> Customizing the Landing Page + Covers the location of the landing page template, + format of the template, and optional preprocessor + tags available to use when creating a new landing + page template. + + >> Interacting With the Registry + Explains all API endpoints, their parameters, + and expected output. + + + :: Configuration File :: + + The default configuration file is in YAML format, chosen for + its clarity and its support of commenting (unlike JSON). It may + be placed in any of the following locations by default: + + The same directory as the getwtxt executable + /usr/local/getwtxt/ + /etc/ + /usr/local/etc/ + + The paths are searched in that order. The first configuration + file found is used by getwtxt, while the locations further down + are ignored. + + Multiple configuration files may be used, however, with the + '-c' command line flag. The path passed to getwtxt via '-c' may + be relative or absolute. For example, both of the following are + allowed: + + ./getwtxt -c myconfig.json + ./getwtxt -c /etc/ExtraConfigsDir/mysecondconfig.toml + + The supported configuration types are: + YAML, TOML, JSON, HCL + + The configuration file contains several options used to + customize your instance of getwtxt. None are required, they will + simply use their default value unless otherwise specified. + + ListenPort: Defines the port getwtxt should bind to. + Default: 9001 + + DatabaseType: The type of back-end getwtxt should use + to store registry data. Currently, only leveldb + is available, with more options in development. + Default: leveldb + + DatabasePath: The location of the LevelDB structure + used by getwtxt to back up registry data. This + can be a relative or absolute path. + Default: getwtxt.db + + AssetsDirectory: This is the directory where getwtxt + can find style.css and tmpl/index.html -- the + stylesheet for the landing page and the landing + page template, respectively. + Default: assets + + StdoutLogging: Boolean used to determine whether + getwtxt should send logging output to stdout. + This is useful for debugging, but you should + probably save your logs once your instance + is running. + Default: false + + LogFile: The location of getwtxt's log file. This, + like DatabasePath, can be relative or absolute. + Default: getwtxt.log + + DatabasePushInterval: The interval on which getwtxt + will push registry data from the in-memory cache + to the on-disk LevelDB database. The following + time suffixes may be used: + ns, us, ms, s, m, h + Default: 5m + + StatusFetchInterval: The interval on which getwtxt + will crawl all users' twtxt files to retrieve + new statuses. The same time suffixes as + DatabasePushInterval may be used. + Default: 1h + + Instance: Signifies the start of instance-specific + meta information. The following are used only + for the summary and use information displayed + by the default web page for getwtxt. If desired, + the assets/tmpl/index.html file may be + customized. Keep in mind that in YAML, the + following options must be preceded by two spaces + so that they are interpreted as sub-options. + + SiteName: The name of your getwtxt instance. + Default: getwtxt + + URL: The publicly-accessible URL of your + getwtxt instance. + Default: https://twtxt.example.com + + OwnerName: Your name. + Default: Anonymous Microblogger + + Email: Your email address. + Default: nobody@knows + + Description: A short description of your getwtxt + instance or your site. As this likely includes + whitespace, it should be in double-quotes. + This can include XHTML or HTML line breaks if + desired: +
+
+ Default: "A fast, resilient twtxt registry + server written in Go!" + + + :: Customizing the Landing Page :: + + If you like, feel free to customize the landing page + template provided at + + assets/tmpl/index.html + + It must be standard HTML or XHTML. There are a few special + tags available to use that will be replaced with specific values + when the template is parsed by getwtxt. + + Values are derived from the "Instance" section of the + configuration file, except for the version of getwtxt used. The + following will be in the form of: + + {{.TemplateTag}} What it will be replaced with when + the template is processed and the landing page is + served to a visitor. + + The surrounding double braces and prefixed period are required + if you choose to use these tags in your modified landing page. The + tags themselves are not required; access to them is provided simply + for convenience. + + {{.Vers}} The version of getwtxt used. Does not include + the preceding 'v'. Ex: 0.2.0 + + {{.Name}} The name of the instance. + + {{.Owner}} The instance owner's name. + + {{.Mail}} Email address used for contacting the instance + owner if the need arises. + + {{.Desc}} Short description placed in the configuration + file. This is why HTML tags are allowed. + + {{.URL}} The publicly-accessible URL of your instance. In + the default landing page, example API calls are shown + using this URL for the convenience of the user. + + + :: Interacting with the Registry :: + + The registry API is rather simple, and can be interacted with + via the command line using cURL. Example output of the calls will + not be provided. + + Pseudo line-breaks will be represented with a backslash. + Examples with line-breaks are not syntactically correct and will + be rejected by cURL. Please concatenate the example calls without + the backslash. This is only present to maintain consistent + formatting for this manual text. + + Ex: + /api/plain/users\ + ?q=FOO + Should be: + /api/plain/users?q=FOO + + All queries (every call except adding users) accept the + ?page=N parameter, where N > 0. The output is provided in groups + of 20 results. For example, indexed at 1, ?page=2 (or &page=2 if + it is not the first parameter) appended to any query will return + results 21 through 40. If the page requested will exceed the + bounds of the query output, the last 20 query results are returned. + + Adding a user: + curl -X POST 'http://localhost:9001/api/plain/users\ + ?url=https://gbmor.dev/twtxt.txt&nickname=gbmor' + + Retrieve user list: + curl 'http://localhost:9001/api/plain/users' + + Retrieve all statuses: + curl 'http://localhost:9001/api/plain/tweets' + + Retrieve all statuses with mentions: + curl 'http://localhost:9001/api/plain/mentions' + + Retrieve all statuses with tags: + curl 'http://localhost:9001/api/plain/tags' + + Query for users by keyword: + curl 'http://localhost:9001/api/plain/users?q=FOO' + + Query for users by URL: + curl 'http://localhost:9001/api/plain/users\ + ?url=https://gbmor.dev/twtxt.txt' + + Query for statuses by substring: + curl 'http://localhost:9001/api/plain/tweets\ + ?q=SUBSTRING' + + Query for statuses mentioning a user: + curl 'http://localhost:9001/api/plain/mentions\ + ?url=https://gbmor.dev/twtxt.txt' + + Query for statuses with a given tag: + curl 'http://localhost:9001/api/plain/tags/myTagHere' + +`) +} diff --git a/svc/init_test.go b/svc/init_test.go new file mode 100644 index 0000000..58f39f9 --- /dev/null +++ b/svc/init_test.go @@ -0,0 +1,27 @@ +package svc // import github.com/getwtxt/getwtxt/svc + +import ( + "fmt" + "log" + "os" +) + +var testport = fmt.Sprintf(":%v", confObj.Port) +var hasInit = false + +func initTestConf() { + if !hasInit { + initConfig() + tmpls = initTemplates() + logToNull() + hasInit = true + } +} + +func logToNull() { + hush, err := os.Open("/dev/null") + if err != nil { + log.Printf("%v\n", err) + } + log.SetOutput(hush) +} diff --git a/svc/post.go b/svc/post.go new file mode 100644 index 0000000..0355afa --- /dev/null +++ b/svc/post.go @@ -0,0 +1,65 @@ +package svc // import "github.com/getwtxt/getwtxt/svc" + +import ( + "fmt" + "log" + "net/http" + + "github.com/getwtxt/registry" +) + +// Requests to apiEndpointPOSTHandler are passed off to this +// function. apiPostUser then fetches the twtxt data, then if +// it's an individual user's file, adds it. If it's registry +// output, it scrapes the users/urls/statuses from the remote +// registry before adding each user to the local cache. +func apiPostUser(w http.ResponseWriter, r *http.Request) { + if err := r.ParseForm(); err != nil { + log400(w, r, "Error Parsing Values: "+err.Error()) + return + } + + nick := r.FormValue("nickname") + urls := r.FormValue("url") + if nick == "" || urls == "" { + log400(w, r, "Nickname or URL missing") + return + } + + uip := getIPFromCtx(r.Context()) + + out, remoteRegistry, err := registry.GetTwtxt(urls) + if err != nil { + log400(w, r, "Error Fetching twtxt Data: "+err.Error()) + return + } + + if remoteRegistry { + remoteRegistries.Mu.Lock() + remoteRegistries.List = append(remoteRegistries.List, urls) + remoteRegistries.Mu.Unlock() + + if err := twtxtCache.CrawlRemoteRegistry(urls); err != nil { + log400(w, r, "Error Crawling Remote Registry: "+err.Error()) + return + } + log200(r) + return + } + + statuses, err := registry.ParseUserTwtxt(out, nick, urls) + if err != nil { + log.Printf("Error Parsing User Data: %v\n", err.Error()) + } + + if err := twtxtCache.AddUser(nick, urls, "", uip, statuses); err != nil { + log400(w, r, "Error Adding User to Cache: "+err.Error()) + return + } + + log200(r) + _, err = w.Write([]byte(fmt.Sprintf("200 OK\n"))) + if err != nil { + log.Printf("%v\n", err.Error()) + } +} diff --git a/svc/post_test.go b/svc/post_test.go new file mode 100644 index 0000000..c25efe3 --- /dev/null +++ b/svc/post_test.go @@ -0,0 +1,77 @@ +package svc // import github.com/getwtxt/getwtxt/svc + +import ( + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + "github.com/getwtxt/registry" +) + +var apiPostUserCases = []struct { + name string + nick string + uri string + wantErr bool +}{ + { + name: "Known Good User", + nick: "gbmor", + uri: "https://gbmor.dev/twtxt.txt", + wantErr: false, + }, + { + name: "Missing URI", + nick: "missinguri", + uri: "", + wantErr: true, + }, + { + name: "Missing Nickname", + nick: "", + uri: "https://example.com/twtxt.txt", + wantErr: true, + }, + { + name: "Missing URI and Nickname", + nick: "", + uri: "", + wantErr: true, + }, +} + +func Test_apiPostUser(t *testing.T) { + initTestConf() + portnum := fmt.Sprintf(":%v", confObj.Port) + twtxtCache = registry.NewIndex() + + for _, tt := range apiPostUserCases { + t.Run(tt.name, func(t *testing.T) { + params := url.Values{} + params.Set("url", tt.uri) + params.Set("nickname", tt.nick) + + req, err := http.NewRequest("POST", "https://localhost"+portnum+"/api/plain/users", strings.NewReader(params.Encode())) + if err != nil { + t.Errorf("%v\n", err) + } + + req.Header.Add("Content-Type", "application/x-www-form-urlencoded") + rr := httptest.NewRecorder() + apiEndpointPOSTHandler(rr, req) + + if !tt.wantErr { + if rr.Code != http.StatusOK { + t.Errorf("Received unexpected non-200 response: %v\n", rr.Code) + } + } else { + if rr.Code != http.StatusBadRequest { + t.Errorf("Expected 400 Bad Request, but received: %v\n", rr.Code) + } + } + }) + } +} diff --git a/svc/query.go b/svc/query.go new file mode 100644 index 0000000..037b0e5 --- /dev/null +++ b/svc/query.go @@ -0,0 +1,146 @@ +package svc // import "github.com/getwtxt/getwtxt/svc" + +import ( + "crypto/sha256" + "fmt" + "log" + "net/http" + "strconv" + "strings" + + "github.com/getwtxt/registry" + "github.com/gorilla/mux" +) + +func apiErrCheck(err error, r *http.Request) { + if err != nil { + uip := getIPFromCtx(r.Context()) + log.Printf("*** %v :: %v %v :: %v\n", uip, r.Method, r.URL, err.Error()) + } +} + +// Takes the output of queries and formats it for +// an HTTP response. Iterates over the string slice, +// appending each entry to a byte slice, and adding +// newlines where appropriate. +func parseQueryOut(out []string) []byte { + var data []byte + + for i, e := range out { + data = append(data, []byte(e)...) + + if !strings.HasSuffix(e, "\n") && i != len(out)-1 { + data = append(data, byte('\n')) + } + } + + return data +} + +// Removes duplicate statuses from query output +func uniq(str []string) []string { + keys := make(map[string]bool) + out := []string{} + for _, e := range str { + if _, ok := keys[e]; !ok { + keys[e] = true + out = append(out, e) + } + } + return out +} + +// apiUserQuery is called via apiEndpointHandler when +// the endpoint is "users" and r.FormValue("q") is not empty. +// It queries the registry cache for users or user URLs +// matching the term supplied via r.FormValue("q") +func apiEndpointQuery(w http.ResponseWriter, r *http.Request) error { + query := r.FormValue("q") + urls := r.FormValue("url") + pageVal := r.FormValue("page") + var out []string + var err error + + pageVal = strings.TrimSpace(pageVal) + page, err := strconv.Atoi(pageVal) + if err != nil { + log.Printf("%v\n", err.Error()) + } + + vars := mux.Vars(r) + endpoint := vars["endpoint"] + + // Handle user URL queries first, then nickname queries. + // Concatenate both outputs if they're both set. + // Also handle mention queries and status queries. + // If we made it this far and 'default' is matched, + // something went very wrong. + switch endpoint { + case "users": + var out2 []string + if query != "" { + out, err = twtxtCache.QueryUser(query) + apiErrCheck(err, r) + } + if urls != "" { + out2, err = twtxtCache.QueryUser(urls) + apiErrCheck(err, r) + } + + if query != "" && urls != "" { + out = joinQueryOuts(out2) + } + + case "mentions": + if urls == "" { + return fmt.Errorf("missing URL in mention query") + } + urls += ">" + out, err = twtxtCache.QueryInStatus(urls) + apiErrCheck(err, r) + + case "tweets": + out = compositeStatusQuery(query, r) + + default: + return fmt.Errorf("endpoint query, no cases match") + } + + out = registry.ReduceToPage(page, out) + data := parseQueryOut(out) + + etag := fmt.Sprintf("%x", sha256.Sum256(data)) + w.Header().Set("ETag", etag) + w.Header().Set("Content-Type", txtutf8) + + _, err = w.Write(data) + + return err +} + +func joinQueryOuts(data ...[]string) []string { + single := []string{} + for _, e := range data { + single = append(single, e...) + } + single = uniq(single) + + return single +} + +func compositeStatusQuery(query string, r *http.Request) []string { + query = strings.ToLower(query) + out, err := twtxtCache.QueryInStatus(query) + apiErrCheck(err, r) + + query = strings.Title(query) + out2, err := twtxtCache.QueryInStatus(query) + apiErrCheck(err, r) + + query = strings.ToUpper(query) + out3, err := twtxtCache.QueryInStatus(query) + apiErrCheck(err, r) + + final := joinQueryOuts(out, out2, out3) + return final +} diff --git a/svc/query_test.go b/svc/query_test.go new file mode 100644 index 0000000..f93cd92 --- /dev/null +++ b/svc/query_test.go @@ -0,0 +1,75 @@ +package svc // import github.com/getwtxt/getwtxt/svc + +import ( + "net" + "reflect" + "strings" + "testing" + + "github.com/getwtxt/registry" +) + +func Test_parseQueryOut(t *testing.T) { + initTestConf() + + urls := "https://gbmor.dev/twtxt.txt" + nick := "gbmor" + + out, _, err := registry.GetTwtxt(urls) + if err != nil { + t.Errorf("Couldn't set up test: %v\n", err) + } + + statusmap, err := registry.ParseUserTwtxt(out, nick, urls) + if err != nil { + t.Errorf("Couldn't set up test: %v\n", err) + } + + twtxtCache.AddUser(nick, urls, "", net.ParseIP("127.0.0.1"), statusmap) + + t.Run("Parsing Status Query", func(t *testing.T) { + data, err := twtxtCache.QueryAllStatuses() + if err != nil { + t.Errorf("%v\n", err) + } + + out := parseQueryOut(data) + + conv := strings.Split(string(out), "\n") + + if !reflect.DeepEqual(data, conv) { + t.Errorf("Pre- and Post- parseQueryOut data are inequal:\n%#v\n%#v\n", data, conv) + } + }) +} + +func Benchmark_parseQueryOut(b *testing.B) { + initTestConf() + + urls := "https://gbmor.dev/twtxt.txt" + nick := "gbmor" + + out, _, err := registry.GetTwtxt(urls) + if err != nil { + b.Errorf("Couldn't set up test: %v\n", err) + } + + statusmap, err := registry.ParseUserTwtxt(out, nick, urls) + if err != nil { + b.Errorf("Couldn't set up test: %v\n", err) + } + + twtxtCache.AddUser(nick, urls, "", net.ParseIP("127.0.0.1"), statusmap) + + data, err := twtxtCache.QueryAllStatuses() + if err != nil { + b.Errorf("%v\n", err) + } + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + parseQueryOut(data) + } + +} diff --git a/svc/svc.go b/svc/svc.go new file mode 100644 index 0000000..2592122 --- /dev/null +++ b/svc/svc.go @@ -0,0 +1,120 @@ +package svc // import "github.com/getwtxt/getwtxt/svc" + +import ( + "fmt" + "log" + "net/http" + "time" + + "github.com/gorilla/handlers" + "github.com/gorilla/mux" +) + +// Start is the initialization function for getwtxt +func Start() { + + // StrictSlash(true) allows /api and /api/ + // to serve the same content without duplicating + // handlers/paths + index := mux.NewRouter().StrictSlash(true) + api := index.PathPrefix("/api").Subrouter() + + index.Path("/"). + Methods("GET", "HEAD"). + HandlerFunc(indexHandler) + + index.Path("/css"). + Methods("GET", "HEAD"). + HandlerFunc(cssHandler) + + index.Path("/api"). + Methods("GET", "HEAD"). + HandlerFunc(apiBaseHandler) + + // twtxt will add support for other formats later. + // Maybe json? Making this future-proof. + api.Path("/{format:(?:plain)}"). + Methods("GET", "HEAD"). + HandlerFunc(apiFormatHandler) + + // Non-standard API call to list *all* tweets + // in a single request. + api.Path("/{format:(?:plain)}/tweets/all"). + Methods("GET", "HEAD"). + HandlerFunc(apiAllTweetsHandler) + + // Specifying the endpoint with and without query information. + // Will return 404 on empty queries otherwise. + api.Path("/{format:(?:plain)}/{endpoint:(?:mentions|users|tweets)}"). + Methods("GET", "HEAD"). + HandlerFunc(apiEndpointHandler) + + api.Path("/{format:(?:plain)}/{endpoint:(?:mentions|users|tweets)}"). + Queries("url", "{url}", "q", "{query}", "page", "{[0-9]+}"). + Methods("GET", "HEAD"). + HandlerFunc(apiEndpointHandler) + + // This is for submitting new users. Both query variables must exist + // in the request for this to match. + api.Path("/{format:(?:plain)}/{endpoint:users}"). + Queries("url", "{url}", "nickname", "{nickname:[a-zA-Z0-9_-]+}"). + Methods("POST"). + HandlerFunc(apiEndpointPOSTHandler) + + // This is for submitting new users incorrectly + // and letting the requester know about their error. + api.Path("/{format:(?:plain)}/{endpoint:users}"). + Queries("url", "{url}"). + Methods("POST"). + HandlerFunc(apiEndpointPOSTHandler) + + // This is also for submitting new users incorrectly + // and letting the requester know about their error. + api.Path("/{format:(?:plain)}/{endpoint:users}"). + Queries("nickname", "{nickname:[a-zA-Z0-9_-]+}"). + Methods("POST"). + HandlerFunc(apiEndpointPOSTHandler) + + // Show all observed tags + api.Path("/{format:(?:plain)}/tags"). + Methods("GET", "HEAD"). + HandlerFunc(apiTagsBaseHandler) + + // Show Nth page of all observed tags + api.Path("/{format:(?:plain)}/tags"). + Queries("page", "{[0-9]+}"). + Methods("GET", "HEAD"). + HandlerFunc(apiTagsBaseHandler) + + // Requests statuses with a specific tag + api.Path("/{format:(?:plain)}/tags/{tags:[a-zA-Z0-9_-]+}"). + Methods("GET", "HEAD"). + HandlerFunc(apiTagsHandler) + + // Requests Nth page of statuses with a specific tag + api.Path("/{format:(?:plain)}/tags/{tags:[a-zA-Z0-9_-]+}"). + Queries("page", "{[0-9]+}"). + Methods("GET", "HEAD"). + HandlerFunc(apiTagsHandler) + + confObj.Mu.RLock() + portnum := fmt.Sprintf(":%v", confObj.Port) + confObj.Mu.RUnlock() + + // handlers.CompressHandler gzips all responses. + // Write/Read timeouts are self explanatory. + server := &http.Server{ + Handler: handlers.CompressHandler(ipMiddleware(index)), + Addr: portnum, + WriteTimeout: 15 * time.Second, + ReadTimeout: 15 * time.Second, + } + + log.Printf("Listening on %v\n", portnum) + err := server.ListenAndServe() + if err != nil { + log.Printf("%v\n", err.Error()) + } + + closeLog <- true +} diff --git a/svc/types.go b/svc/types.go new file mode 100644 index 0000000..978e0df --- /dev/null +++ b/svc/types.go @@ -0,0 +1,67 @@ +package svc // import "github.com/getwtxt/getwtxt/svc" + +import ( + "database/sql" + "sync" + "time" + + "github.com/syndtr/goleveldb/leveldb" +) + +// content-type consts +const txtutf8 = "text/plain; charset=utf-8" +const htmlutf8 = "text/html; charset=utf-8" +const cssutf8 = "text/css; charset=utf-8" + +// Configuration object definition +type Configuration struct { + Mu sync.RWMutex + Port int `yaml:"ListenPort"` + LogFile string `yaml:"LogFile"` + DBType string `yaml:"DatabaseType"` + DBPath string `yaml:"DatabasePath"` + AssetsDir string `yaml:"-"` + StdoutLogging bool `yaml:"StdoutLogging"` + Version string `yaml:"-"` + CacheInterval time.Duration `yaml:"StatusFetchInterval"` + DBInterval time.Duration `yaml:"DatabasePushInterval"` + LastCache time.Time `yaml:"-"` + LastPush time.Time `yaml:"-"` + Instance `yaml:"Instance"` +} + +// Instance refers to this specific instance of getwtxt +type Instance struct { + Vers string `yaml:"-"` + Name string `yaml:"Instance.SiteName"` + URL string `yaml:"Instance.URL"` + Owner string `yaml:"Instance.OwnerName"` + Mail string `yaml:"Instance.Email"` + Desc string `yaml:"Instance.Description"` +} + +type dbLevel struct { + db *leveldb.DB +} + +type dbSqlite struct { + db *sql.DB +} + +type dbase interface { + push() error + pull() +} + +// RemoteRegistries holds a list of remote registries to +// periodically scrape for new users. The remote registries +// must have been added via POST like a user. +type RemoteRegistries struct { + Mu sync.RWMutex + List []string +} + +// ipCtxKey is the Context value key for user IP addresses +type ipCtxKey int + +const ctxKey ipCtxKey = iota -- cgit 1.4.1-2-gfad0