diff options
| author | Ben Morrison <ben@gbmor.dev> | 2020-06-20 02:27:31 -0400 |
|---|---|---|
| committer | Ben Morrison <ben@gbmor.dev> | 2020-06-20 02:27:31 -0400 |
| commit | 538e305925b9b04102ef0a4fb7cca19a6c116142 (patch) | |
| tree | 8e68b426c40c3151f39da1be874d2938f31ab5f9 | |
| parent | 0a69c582ec8b88b1d3af70ef43c3eeb1b99f973d (diff) | |
| download | getwtxt-538e305925b9b04102ef0a4fb7cca19a6c116142.tar.gz | |
updating module to live at sourcehut
Also moving the 'registry' library into this repo, rather than maintaining them separately. It will still be decoupled, just live in this repository.
38 files changed, 2493 insertions, 89 deletions
diff --git a/.build.yml b/.build.yml new file mode 100644 index 0000000..d4677a8 --- /dev/null +++ b/.build.yml @@ -0,0 +1,10 @@ +image: alpine/edge +packages: + - go +sources: + - https://git.sr.ht/~gbmor/getwtxt +tasks: + - build: | + cd getwtxt + go test -v + go test -v --bench . --benchmem diff --git a/Makefile b/Makefile index 9d83b15..319357b 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ PREFIX?=/usr/local _INSTDIR=$(PREFIX) BINDIR?=$(_INSTDIR)/getwtxt VERSION?=$(shell git tag | grep ^v | sort -V | tail -n 1) -GOFLAGS?=-ldflags '-X github.com/getwtxt/getwtxt/svc.Vers=${VERSION}' +GOFLAGS?=-ldflags '-X git.sr.ht/~gbmor/getwtxt/svc.Vers=${VERSION}' getwtxt: getwtxt.go go.mod go.sum @echo diff --git a/assets/tmpl/index.html b/assets/tmpl/index.html index 9006bec..0318810 100644 --- a/assets/tmpl/index.html +++ b/assets/tmpl/index.html @@ -90,7 +90,7 @@ foo_barrington https://example3.com/twtxt.txt 2019-02-26T11:06:44.000Z foo https://example.com/twtxt.txt 2019-02-26T11:06:44.000Z @<foo_barrington https://example3.com/twtxt.txt> Hey!! Are you still working on that project?</code></pre> </div> <div id="foot"> - powered by <a href="https://github.com/getwtxt/getwtxt">getwtxt</a> + powered by <a href="https://sr.ht/~gbmor/getwtxt">getwtxt</a> </div> </div> </body> diff --git a/getwtxt.go b/getwtxt.go index 45d04c3..5e8e52c 100644 --- a/getwtxt.go +++ b/getwtxt.go @@ -19,7 +19,7 @@ along with Getwtxt. If not, see <https://www.gnu.org/licenses/>. package main -import "github.com/getwtxt/getwtxt/svc" +import "git.sr.ht/~gbmor/getwtxt/svc" func main() { svc.Start() diff --git a/go.mod b/go.mod index 1d60931..2013e93 100644 --- a/go.mod +++ b/go.mod @@ -1,15 +1,14 @@ -module github.com/getwtxt/getwtxt +module git.sr.ht/~gbmor/getwtxt go 1.11 require ( github.com/fsnotify/fsnotify v1.4.9 - github.com/getwtxt/registry v0.4.2 github.com/gorilla/handlers v1.4.2 github.com/gorilla/mux v1.7.4 github.com/mattn/go-sqlite3 v2.0.3+incompatible github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.6.2 + github.com/spf13/viper v1.7.0 github.com/syndtr/goleveldb v1.0.0 - golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 + golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 ) diff --git a/go.sum b/go.sum index 647704a..fdd4551 100644 --- a/go.sum +++ b/go.sum @@ -1,30 +1,47 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/getwtxt/registry v0.4.2 h1:6qp7KkBi60CZaJpFC21ez29QwiIcCnzazAG3w9OdXaU= -github.com/getwtxt/registry v0.4.2/go.mod h1:BGSIALOFqIRj+ACLB8etWGUOgFAKN8oFDpCsw6YOdYQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= @@ -34,28 +51,60 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/handlers v1.4.2 h1:0QniY0USkHQ1RGCLfKxeNHK9bkDHGRYGNDFBCS+YARg= github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= @@ -70,11 +119,22 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -82,11 +142,14 @@ github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -97,6 +160,9 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= @@ -113,62 +179,139 @@ github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb6 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.6.2 h1:7aKfF+e8/k68gda3LOjo5RxiUqddoFxVq4BKBPrxk5E= -github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= +github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM= +github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM/fAoGlaiiHYiFYdm80= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= @@ -181,3 +324,7 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/registry/README.md b/registry/README.md new file mode 100644 index 0000000..34cd37e --- /dev/null +++ b/registry/README.md @@ -0,0 +1,48 @@ +# `getwtxt/registry` + +### twtxt Registry Library for Go + +`getwtxt/registry` helps you implement twtxt registries in Go. +It uses no third-party dependencies whatsoever, only the standard library, +and has no global state. +Specifying your own `http.Client` for requests is encouraged, with a sensible +default available by passing `nil` to the constructor. + +## Using the Library + +Just add it to your imports list in the file(s) where it's needed. + +```go +import ( + "git.sr.ht/~gbmor/getwtxt/registry" +) +``` + +## Documentation + +The code is commented, so feel free to browse the files themselves. +Alternatively, the generated documentation can be found at: + +[pkg.go.dev/git.sr.ht/~gbmor/getwtxt/registry](https://pkg.go.dev/git.sr.ht/~gbmor/getwtxt/registry) + +## Contributions + +All contributions are very welcome! Please specify that you are referring to `getwtxt/registry` +when using the following: + +* Mailing list (patches, discussion) + * [https://lists.sr.ht/~gbmor/getwtxt](https://lists.sr.ht/~gbmor/getwtxt) +* Ticket tracker + * [https://todo.sr.ht/~gbmor/getwtxt](https://todo.sr.ht/~gbmor/getwtxt) + +## Notes + +* getwtxt - parent project: + * [sr.ht/~gbmor/getwtxt](https://sr.ht/~gbmor/getwtxt) + +* twtxt repository: + * [github.com/buckket/twtxt](https://github.com/buckket/twtxt) +* twtxt documentation: + * [twtxt.readthedocs.io/en/latest/](https://twtxt.readthedocs.io/en/latest/) +* twtxt registry documentation: + * [twtxt.readthedocs.io/en/latest/user/registry.html](https://twtxt.readthedocs.io/en/latest/user/registry.html) diff --git a/registry/fetch.go b/registry/fetch.go new file mode 100644 index 0000000..9adf4ec --- /dev/null +++ b/registry/fetch.go @@ -0,0 +1,277 @@ +/* +Copyright (c) 2019 Ben Morrison (gbmor) + +This file is part of Registry. + +Registry is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +Registry is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Registry. If not, see <https://www.gnu.org/licenses/>. +*/ + +package registry // import "git.sr.ht/~gbmor/getwtxt/registry" + +import ( + "bufio" + "bytes" + "fmt" + "io/ioutil" + "net/http" + "regexp" + "strings" + "sync" + "time" +) + +// GetTwtxt fetches the raw twtxt file data from the user's +// provided URL, after validating the URL. If the returned +// boolean value is false, the fetched URL is a single user's +// twtxt file. If true, the fetched URL is the output of +// another registry's /api/plain/tweets. The output of +// GetTwtxt should be passed to either ParseUserTwtxt or +// ParseRegistryTwtxt, respectively. +// Generally, the *http.Client inside a given Registry instance should +// be passed to GetTwtxt. If the *http.Client passed is nil, +// Registry will use a preconstructed client with a +// timeout of 10s and all other values set to default. +func GetTwtxt(urlKey string, client *http.Client) ([]byte, bool, error) { + if !strings.HasPrefix(urlKey, "http://") && !strings.HasPrefix(urlKey, "https://") { + return nil, false, fmt.Errorf("invalid URL: %v", urlKey) + } + + res, err := doReq(urlKey, "GET", "", client) + if err != nil { + return nil, false, err + } + defer res.Body.Close() + + var textPlain bool + for _, v := range res.Header["Content-Type"] { + if strings.Contains(v, "text/plain") { + textPlain = true + break + } + } + if !textPlain { + return nil, false, fmt.Errorf("received non-text/plain response body from %v", urlKey) + } + + if res.StatusCode != http.StatusOK { + return nil, false, fmt.Errorf("didn't get 200 from remote server, received %v: %v", res.StatusCode, urlKey) + } + + twtxt, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, false, fmt.Errorf("error reading response body from %v: %v", urlKey, err) + } + + // Signal that we're adding another twtxt registry as a "user" + if strings.HasSuffix(urlKey, "/api/plain/tweets") || strings.HasSuffix(urlKey, "/api/plain/tweets/all") { + return twtxt, true, nil + } + + return twtxt, false, nil +} + +// DiffTwtxt issues a HEAD request on the user's +// remote twtxt data. It then checks the Content-Length +// header. If it's different from the stored result of +// the previous Content-Length header, update the stored +// value for a given user and return true. +// Otherwise, return false. In some error conditions, +// such as the user not being in the registry, it returns true. +// In other error conditions considered "unrecoverable," +// such as the supplied URL being invalid, it returns false. +func (registry *Registry) DiffTwtxt(urlKey string) (bool, error) { + if !strings.HasPrefix(urlKey, "http://") && !strings.HasPrefix(urlKey, "https://") { + return false, fmt.Errorf("invalid URL: %v", urlKey) + } + + registry.Mu.Lock() + user, ok := registry.Users[urlKey] + if !ok { + return true, fmt.Errorf("user not in registry") + } + + user.Mu.Lock() + + defer func() { + registry.Users[urlKey] = user + user.Mu.Unlock() + registry.Mu.Unlock() + }() + + res, err := doReq(urlKey, "HEAD", user.LastModified, registry.HTTPClient) + if err != nil { + return false, err + } + + switch res.StatusCode { + case http.StatusOK: + for _, e := range res.Header["Last-Modified"] { + if e != "" { + user.LastModified = e + break + } + } + return true, nil + + case http.StatusNotModified: + return false, nil + } + + return false, nil +} + +// internal function. boilerplate for http requests. +func doReq(urlKey, method, modTime string, client *http.Client) (*http.Response, error) { + if client == nil { + client = &http.Client{ + Transport: nil, + CheckRedirect: nil, + Jar: nil, + Timeout: 10 * time.Second, + } + } + + var b []byte + buf := bytes.NewBuffer(b) + req, err := http.NewRequest(method, urlKey, buf) + if err != nil { + return nil, err + } + + if modTime != "" { + req.Header.Set("If-Modified-Since", modTime) + } + + res, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("couldn't %v %v: %v", method, urlKey, err) + } + + return res, nil +} + +// ParseUserTwtxt takes a fetched twtxt file in the form of +// a slice of bytes, parses it, and returns it as a +// TimeMap. The output may then be passed to Index.AddUser() +func ParseUserTwtxt(twtxt []byte, nickname, urlKey string) (TimeMap, error) { + var erz []byte + if len(twtxt) == 0 { + return nil, fmt.Errorf("no data to parse in twtxt file") + } + + reader := bytes.NewReader(twtxt) + scanner := bufio.NewScanner(reader) + timemap := NewTimeMap() + + for scanner.Scan() { + nopadding := strings.TrimSpace(scanner.Text()) + if strings.HasPrefix(nopadding, "#") || nopadding == "" { + continue + } + + columns := strings.Split(nopadding, "\t") + if len(columns) != 2 { + return nil, fmt.Errorf("improperly formatted data in twtxt file") + } + + normalizedDatestamp := fixTimestamp(columns[0]) + thetime, err := time.Parse(time.RFC3339, normalizedDatestamp) + if err != nil { + erz = append(erz, []byte(fmt.Sprintf("unable to retrieve date: %v\n", err))...) + } + + timemap[thetime] = nickname + "\t" + urlKey + "\t" + nopadding + } + + if len(erz) == 0 { + return timemap, nil + } + return timemap, fmt.Errorf("%v", string(erz)) +} + +func fixTimestamp(ts string) string { + normalizeTimestamp := regexp.MustCompile(`[\+][\d][\d][:][\d][\d]`) + return strings.TrimSpace(normalizeTimestamp.ReplaceAllString(ts, "Z")) +} + +// ParseRegistryTwtxt takes output from a remote registry and outputs +// the accessible user data via a slice of Users. +func ParseRegistryTwtxt(twtxt []byte) ([]*User, error) { + var erz []byte + if len(twtxt) == 0 { + return nil, fmt.Errorf("received no data") + } + + reader := bytes.NewReader(twtxt) + scanner := bufio.NewScanner(reader) + userdata := []*User{} + + for scanner.Scan() { + + nopadding := strings.TrimSpace(scanner.Text()) + + if strings.HasPrefix(nopadding, "#") || nopadding == "" { + continue + } + + columns := strings.Split(nopadding, "\t") + if len(columns) != 4 { + return nil, fmt.Errorf("improperly formatted data") + } + + thetime, err := time.Parse(time.RFC3339, columns[2]) + if err != nil { + erz = append(erz, []byte(fmt.Sprintf("%v\n", err))...) + continue + } + + parsednickname := columns[0] + dataIndex := 0 + parsedurl := columns[1] + inIndex := false + + for i, e := range userdata { + if e.Nick == parsednickname || e.URL == parsedurl { + dataIndex = i + inIndex = true + break + } + } + + if inIndex { + tmp := userdata[dataIndex] + tmp.Status[thetime] = nopadding + userdata[dataIndex] = tmp + } else { + timeNowRFC := time.Now().Format(time.RFC3339) + if err != nil { + erz = append(erz, []byte(fmt.Sprintf("%v\n", err))...) + } + + tmp := &User{ + Mu: sync.RWMutex{}, + Nick: parsednickname, + URL: parsedurl, + Date: timeNowRFC, + Status: TimeMap{ + thetime: nopadding, + }, + } + + userdata = append(userdata, tmp) + } + } + + return userdata, fmt.Errorf("%v", erz) +} diff --git a/registry/fetch_test.go b/registry/fetch_test.go new file mode 100644 index 0000000..4eab2a4 --- /dev/null +++ b/registry/fetch_test.go @@ -0,0 +1,286 @@ +/* +Copyright (c) 2019 Ben Morrison (gbmor) + +This file is part of Registry. + +Registry is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +Registry is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Registry. If not, see <https://www.gnu.org/licenses/>. +*/ + +package registry + +import ( + "bufio" + "fmt" + "net/http" + "os" + "strings" + "testing" + "time" +) + +func constructTwtxt() []byte { + registry := initTestEnv() + var resp []byte + // iterates through each mock user's mock statuses + for _, v := range registry.Users { + for _, e := range v.Status { + split := strings.Split(e, "\t") + status := []byte(split[2] + "\t" + split[3] + "\n") + resp = append(resp, status...) + } + } + return resp +} + +// this is just dumping all the mock statuses. +// it'll be served under fake paths as +// "remote" twtxt.txt files +func twtxtHandler(w http.ResponseWriter, _ *http.Request) { + // prepare the response + resp := constructTwtxt() + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + n, err := w.Write(resp) + if err != nil || n == 0 { + fmt.Printf("Got error or wrote zero bytes: %v bytes, %v\n", n, err) + } +} + +var getTwtxtCases = []struct { + name string + url string + wantErr bool + localOnly bool +}{ + { + name: "Constructed Local twtxt.txt", + url: "http://localhost:8080/twtxt.txt", + wantErr: false, + localOnly: true, + }, + { + name: "Inaccessible Site With twtxt.txt", + url: "https://example33333333333.com/twtxt.txt", + wantErr: true, + localOnly: false, + }, + { + name: "Inaccessible Site Without twtxt.txt", + url: "https://example333333333333.com", + wantErr: true, + localOnly: false, + }, + { + name: "Local File Inclusion 1", + url: "file://init_test.go", + wantErr: true, + localOnly: false, + }, + { + name: "Local File Inclusion 2", + url: "/etc/passwd", + wantErr: true, + localOnly: false, + }, + { + name: "Remote File Inclusion", + url: "https://example.com/file.cgi", + wantErr: true, + localOnly: false, + }, + { + name: "Remote Registry", + url: "https://twtxt.tilde.institute/api/plain/tweets/", + wantErr: false, + localOnly: false, + }, + { + name: "Garbage Data", + url: "this will be replaced with garbage data", + wantErr: true, + localOnly: true, + }, +} + +// Test the function that yoinks the /twtxt.txt file +// for a given user. +func Test_GetTwtxt(t *testing.T) { + var buf = make([]byte, 256) + // read random data into case 4 + rando, _ := os.Open("/dev/random") + reader := bufio.NewReader(rando) + n, err := reader.Read(buf) + if err != nil || n == 0 { + t.Errorf("Couldn't set up test: %v\n", err) + } + getTwtxtCases[7].url = string(buf) + + if !getTwtxtCases[0].localOnly { + http.Handle("/twtxt.txt", http.HandlerFunc(twtxtHandler)) + go fmt.Println(http.ListenAndServe(":8080", nil)) + } + + for _, tt := range getTwtxtCases { + t.Run(tt.name, func(t *testing.T) { + if tt.localOnly { + t.Skipf("Local-only test. Skipping ... \n") + } + out, _, err := GetTwtxt(tt.url, nil) + if tt.wantErr && err == nil { + t.Errorf("Expected error: %v\n", tt.url) + } + if !tt.wantErr && err != nil { + t.Errorf("Unexpected error: %v %v\n", tt.url, err) + } + if !tt.wantErr && out == nil { + t.Errorf("Incorrect data received: %v\n", out) + } + }) + } + +} + +// running the benchmarks separately for each case +// as they have different properties (allocs, time) +func Benchmark_GetTwtxt(b *testing.B) { + + for i := 0; i < b.N; i++ { + _, _, err := GetTwtxt("https://gbmor.dev/twtxt.txt", nil) + if err != nil { + continue + } + } +} + +var parseTwtxtCases = []struct { + name string + data []byte + wantErr bool + localOnly bool +}{ + { + name: "Constructed twtxt file", + data: constructTwtxt(), + wantErr: false, + localOnly: false, + }, + { + name: "Incorrectly formatted date", + data: []byte("2019 April 23rd\tI love twtxt!!!11"), + wantErr: true, + localOnly: false, + }, + { + name: "No data", + data: []byte{}, + wantErr: true, + localOnly: false, + }, + { + name: "Variant rfc3339 datestamp", + data: []byte("2020-02-04T21:28:21.868659+00:00\tWill this work?"), + wantErr: false, + localOnly: false, + }, + { + name: "Random/garbage data", + wantErr: true, + localOnly: true, + }, +} + +// See if we can break ParseTwtxt or get it +// to throw an unexpected error +func Test_ParseUserTwtxt(t *testing.T) { + var buf = make([]byte, 256) + // read random data into case 4 + rando, _ := os.Open("/dev/random") + reader := bufio.NewReader(rando) + n, err := reader.Read(buf) + if err != nil || n == 0 { + t.Errorf("Couldn't set up test: %v\n", err) + } + parseTwtxtCases[4].data = buf + + for _, tt := range parseTwtxtCases { + if tt.localOnly { + t.Skipf("Local-only test: Skipping ... \n") + } + t.Run(tt.name, func(t *testing.T) { + timemap, errs := ParseUserTwtxt(tt.data, "testuser", "testurl") + if errs == nil && tt.wantErr { + t.Errorf("Expected error(s), received none.\n") + } + + if !tt.wantErr { + if errs != nil { + t.Errorf("Unexpected error: %v\n", errs) + } + + for k, v := range timemap { + if k == (time.Time{}) || v == "" { + t.Errorf("Empty status or empty timestamp: %v, %v\n", k, v) + } + } + } + }) + } +} + +func Benchmark_ParseUserTwtxt(b *testing.B) { + var buf = make([]byte, 256) + // read random data into case 4 + rando, _ := os.Open("/dev/random") + reader := bufio.NewReader(rando) + n, err := reader.Read(buf) + if err != nil || n == 0 { + b.Errorf("Couldn't set up benchmark: %v\n", err) + } + parseTwtxtCases[3].data = buf + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + for _, tt := range parseTwtxtCases { + _, _ = ParseUserTwtxt(tt.data, "testuser", "testurl") + } + } +} + +var timestampCases = []struct { + name string + orig string + expected string +}{ + { + name: "Timezone appended", + orig: "2020-01-13T16:08:25.544735+00:00", + expected: "2020-01-13T16:08:25.544735Z", + }, + { + name: "It's fine already", + orig: "2020-01-14T00:19:45.092344Z", + expected: "2020-01-14T00:19:45.092344Z", + }, +} + +func Test_fixTimestamp(t *testing.T) { + for _, tt := range timestampCases { + t.Run(tt.name, func(t *testing.T) { + tsout := fixTimestamp(tt.orig) + if tsout != tt.expected { + t.Errorf("Failed :: %s :: got %s expected %s", tt.name, tsout, tt.expected) + } + }) + } +} diff --git a/registry/init_test.go b/registry/init_test.go new file mode 100644 index 0000000..ab9d494 --- /dev/null +++ b/registry/init_test.go @@ -0,0 +1,93 @@ +/* +Copyright (c) 2019 Ben Morrison (gbmor) + +This file is part of Registry. + +Registry is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +Registry is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Registry. If not, see <https://www.gnu.org/licenses/>. +*/ + +package registry //import "git.sr.ht/~gbmor/getwtxt/registry" + +import ( + "fmt" + "log" + "os" + "time" +) + +func quickErr(err error) { + if err != nil { + fmt.Printf("%v\n", err) + } +} + +// Sets up mock users and statuses +func initTestEnv() *Registry { + hush, err := os.Open("/dev/null") + quickErr(err) + log.SetOutput(hush) + + // this is a bit tedious, but set up fake dates + // for the mock users' join and status timestamps + timeMonthPrev := time.Now().AddDate(0, -1, 0) + timeMonthPrevRFC := timeMonthPrev.Format(time.RFC3339) + + timeTwoMonthsPrev := time.Now().AddDate(0, -2, 0) + timeTwoMonthsPrevRFC := timeTwoMonthsPrev.Format(time.RFC3339) + + timeThreeMonthsPrev := time.Now().AddDate(0, -3, 0) + timeThreeMonthsPrevRFC := timeThreeMonthsPrev.Format(time.RFC3339) + + timeFourMonthsPrev := time.Now().AddDate(0, -4, 0) + timeFourMonthsPrevRFC := timeFourMonthsPrev.Format(time.RFC3339) + + var mockusers = []struct { + url string + nick string + date string + apidate []byte + status TimeMap + }{ + { + url: "https://example3.com/twtxt.txt", + nick: "foo_barrington", + date: timeTwoMonthsPrevRFC, + status: TimeMap{ + timeTwoMonthsPrev: "foo_barrington\thttps://example3.com/twtxt.txt\t" + timeTwoMonthsPrevRFC + "\tJust got started with #twtxt!", + timeMonthPrev: "foo_barrington\thttps://example3.com/twtxt.txt\t" + timeMonthPrevRFC + "\tHey <@foo https://example.com/twtxt.txt>, I love programming. Just FYI.", + }, + }, + { + url: "https://example.com/twtxt.txt", + nick: "foo", + date: timeFourMonthsPrevRFC, + status: TimeMap{ + timeFourMonthsPrev: "foo\thttps://example.com/twtxt.txt\t" + timeFourMonthsPrevRFC + "\tThis is so much better than #twitter", + timeThreeMonthsPrev: "foo\thttps://example.com/twtxt.txt\t" + timeThreeMonthsPrevRFC + "\tI can't wait to start on my next programming #project with <@foo_barrington https://example3.com/twtxt.txt>", + }, + }, + } + registry := New(nil) + + // fill the test registry with the mock users + for _, e := range mockusers { + data := &User{} + data.Nick = e.nick + data.Date = e.date + data.Status = e.status + registry.Users[e.url] = data + } + + return registry +} diff --git a/registry/integ_test.go b/registry/integ_test.go new file mode 100644 index 0000000..2cfbb13 --- /dev/null +++ b/registry/integ_test.go @@ -0,0 +1,98 @@ +/* +Copyright (c) 2019 Ben Morrison (gbmor) + +This file is part of Registry. + +Registry is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +Registry is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Registry. If not, see <https://www.gnu.org/licenses/>. +*/ + +package registry + +import ( + "strings" + "testing" +) + +// This tests all the operations on an registry. +func Test_Integration(t *testing.T) { + var integration = func(t *testing.T) { + t.Logf("Creating registry object ...\n") + registry := New(nil) + + t.Logf("Fetching remote twtxt file ...\n") + mainregistry, _, err := GetTwtxt("https://gbmor.dev/twtxt.txt", nil) + if err != nil { + t.Errorf("%v\n", err) + } + + t.Logf("Parsing remote twtxt file ...\n") + parsed, errz := ParseUserTwtxt(mainregistry, "gbmor", "https://gbmor.dev/twtxt.txt") + if errz != nil { + t.Errorf("%v\n", errz) + } + + t.Logf("Adding new user to registry ...\n") + err = registry.AddUser("TestRegistry", "https://gbmor.dev/twtxt.txt", nil, parsed) + if err != nil { + t.Errorf("%v\n", err) + } + + t.Logf("Querying user statuses ...\n") + queryuser, err := registry.QueryUser("TestRegistry") + if err != nil { + t.Errorf("%v\n", err) + } + for _, e := range queryuser { + if !strings.Contains(e, "TestRegistry") { + t.Errorf("QueryUser() returned incorrect data\n") + } + } + + t.Logf("Querying for keyword in statuses ...\n") + querystatus, err := registry.QueryInStatus("morning") + if err != nil { + t.Errorf("%v\n", err) + } + for _, e := range querystatus { + if !strings.Contains(e, "morning") { + t.Errorf("QueryInStatus() returned incorrect data\n") + } + } + + t.Logf("Querying for all statuses ...\n") + allstatus, err := registry.QueryAllStatuses() + if err != nil { + t.Errorf("%v\n", err) + } + if len(allstatus) == 0 || allstatus == nil { + t.Errorf("Got nil/zero from QueryAllStatuses") + } + + t.Logf("Querying for all users ...\n") + allusers, err := registry.QueryUser("") + if err != nil { + t.Errorf("%v\n", err) + } + if len(allusers) == 0 || allusers == nil { + t.Errorf("Got nil/zero users on empty QueryUser() query") + } + + t.Logf("Deleting user ...\n") + err = registry.DelUser("https://gbmor.dev/twtxt.txt") + if err != nil { + t.Errorf("%v\n", err) + } + } + t.Run("Integration Test", integration) +} diff --git a/registry/query.go b/registry/query.go new file mode 100644 index 0000000..604b974 --- /dev/null +++ b/registry/query.go @@ -0,0 +1,196 @@ +/* +Copyright (c) 2019 Ben Morrison (gbmor) + +This file is part of Registry. + +Registry is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +Registry is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Registry. If not, see <https://www.gnu.org/licenses/>. +*/ + +package registry // import "git.sr.ht/~gbmor/getwtxt/registry" + +import ( + "fmt" + "sort" + "strings" + "time" +) + +// QueryUser checks the Registry for usernames +// or user URLs that contain the term provided as an argument. Entries +// are returned sorted by the date they were added to the Registry. If +// the argument provided is blank, return all users. +func (registry *Registry) QueryUser(term string) ([]string, error) { + if registry == nil { + return nil, fmt.Errorf("can't query empty registry for user") + } + + term = strings.ToLower(term) + timekey := NewTimeMap() + keys := make(TimeSlice, 0) + var users []string + + registry.Mu.RLock() + defer registry.Mu.RUnlock() + + for k, v := range registry.Users { + if registry.Users[k] == nil { + continue + } + v.Mu.RLock() + if strings.Contains(strings.ToLower(v.Nick), term) || strings.Contains(strings.ToLower(k), term) { + thetime, err := time.Parse(time.RFC3339, v.Date) + if err != nil { + v.Mu.RUnlock() + continue + } + timekey[thetime] = v.Nick + "\t" + k + "\t" + v.Date + "\n" + keys = append(keys, thetime) + } + v.Mu.RUnlock() + } + + sort.Sort(keys) + for _, e := range keys { + users = append(users, timekey[e]) + } + + return users, nil +} + +// QueryInStatus returns all statuses in the Registry +// that contain the provided substring (tag, mention URL, etc). +func (registry *Registry) QueryInStatus(substring string) ([]string, error) { + if substring == "" { + return nil, fmt.Errorf("cannot query for empty tag") + } else if registry == nil { + return nil, fmt.Errorf("can't query statuses of empty registry") + } + + statusmap := make([]TimeMap, 0) + + registry.Mu.RLock() + defer registry.Mu.RUnlock() + + for _, v := range registry.Users { + statusmap = append(statusmap, v.FindInStatus(substring)) + } + + sorted, err := SortByTime(statusmap...) + if err != nil { + return nil, err + } + + return sorted, nil +} + +// QueryAllStatuses returns all statuses in the Registry +// as a slice of strings sorted by timestamp. +func (registry *Registry) QueryAllStatuses() ([]string, error) { + if registry == nil { + return nil, fmt.Errorf("can't get latest statuses from empty registry") + } + + statusmap, err := registry.GetStatuses() + if err != nil { + return nil, err + } + + sorted, err := SortByTime(statusmap) + if err != nil { + return nil, err + } + + if sorted == nil { + sorted = make([]string, 1) + } + + return sorted, nil +} + +// ReduceToPage returns the passed 'page' worth of output. +// One page is twenty items. For example, if 2 is passed, +// it will return data[20:40]. According to the twtxt +// registry specification, queries should accept a "page" +// value. +func ReduceToPage(page int, data []string) []string { + end := 20 * page + if end > len(data) || end < 1 { + end = len(data) + } + + beg := end - 20 + if beg > len(data)-1 || beg < 0 { + beg = 0 + } + + return data[beg:end] +} + +// FindInStatus takes a user's statuses and looks for a given substring. +// Returns the statuses that include the substring as a TimeMap. +func (userdata *User) FindInStatus(substring string) TimeMap { + if userdata == nil { + return nil + } else if len(substring) > 140 { + return nil + } + + substring = strings.ToLower(substring) + statuses := NewTimeMap() + + userdata.Mu.RLock() + defer userdata.Mu.RUnlock() + + for k, e := range userdata.Status { + if _, ok := userdata.Status[k]; !ok { + continue + } + + parts := strings.Split(strings.ToLower(e), "\t") + if strings.Contains(parts[3], substring) { + statuses[k] = e + } + } + + return statuses +} + +// SortByTime returns a string slice of the query results, +// sorted by timestamp in descending order (newest first). +func SortByTime(tm ...TimeMap) ([]string, error) { + if tm == nil { + return nil, fmt.Errorf("can't sort nil TimeMaps") + } + + var times = make(TimeSlice, 0) + var data []string + + for _, e := range tm { + for k := range e { + times = append(times, k) + } + } + + sort.Sort(times) + + for k := range tm { + for _, e := range times { + if _, ok := tm[k][e]; ok { + data = append(data, tm[k][e]) + } + } + } + + return data, nil +} diff --git a/registry/query_test.go b/registry/query_test.go new file mode 100644 index 0000000..7eed2cd --- /dev/null +++ b/registry/query_test.go @@ -0,0 +1,459 @@ +/* +Copyright (c) 2019 Ben Morrison (gbmor) + +This file is part of Registry. + +Registry is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +Registry is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Registry. If not, see <https://www.gnu.org/licenses/>. +*/ + +package registry + +import ( + "bufio" + "os" + "strings" + "testing" + "time" +) + +var queryUserCases = []struct { + name string + term string + wantErr bool +}{ + { + name: "Valid User", + term: "foo", + wantErr: false, + }, + { + name: "Empty Query", + term: "", + wantErr: false, + }, + { + name: "Nonexistent User", + term: "doesntexist", + wantErr: true, + }, + { + name: "Garbage Data", + term: "will be replaced with garbage data", + wantErr: true, + }, +} + +// Checks if Registry.QueryUser() returns users that +// match the provided substring. +func Test_Registry_QueryUser(t *testing.T) { + registry := initTestEnv() + var buf = make([]byte, 256) + // read random data into case 8 + rando, _ := os.Open("/dev/random") + reader := bufio.NewReader(rando) + n, err := reader.Read(buf) + if err != nil || n == 0 { + t.Errorf("Couldn't set up test: %v\n", err) + } + queryUserCases[3].term = string(buf) + + for n, tt := range queryUserCases { + + t.Run(tt.name, func(t *testing.T) { + out, err := registry.QueryUser(tt.term) + + if out == nil && err != nil && !tt.wantErr { + t.Errorf("Received nil output or an error when unexpected. Case %v, %v, %v\n", n, tt.term, err) + } + + if out != nil && tt.wantErr { + t.Errorf("Received unexpected nil output when an error was expected. Case %v, %v\n", n, tt.term) + } + + for _, e := range out { + one := strings.Split(e, "\t") + + if !strings.Contains(one[0], tt.term) && !strings.Contains(one[1], tt.term) { + t.Errorf("Received incorrect output: %v != %v\n", tt.term, e) + } + } + }) + } +} +func Benchmark_Registry_QueryUser(b *testing.B) { + registry := initTestEnv() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + for _, tt := range queryUserCases { + _, err := registry.QueryUser(tt.term) + if err != nil { + b.Errorf("%v\n", err) + } + } + } +} + +var queryInStatusCases = []struct { + name string + substr string + wantNil bool + wantErr bool +}{ + { + name: "Tag in Status", + substr: "twtxt", + wantNil: false, + wantErr: false, + }, + { + name: "Valid URL", + substr: "https://example.com/twtxt.txt", + wantNil: false, + wantErr: false, + }, + { + name: "Multiple Words in Status", + substr: "next programming", + wantNil: false, + wantErr: false, + }, + { + name: "Multiple Words, Not in Status", + substr: "explosive bananas from antarctica", + wantNil: true, + wantErr: false, + }, + { + name: "Empty Query", + substr: "", + wantNil: true, + wantErr: true, + }, + { + name: "Nonsense", + substr: "ahfiurrenkhfkajdhfao", + wantNil: true, + wantErr: false, + }, + { + name: "Invalid URL", + substr: "https://doesnt.exist/twtxt.txt", + wantNil: true, + wantErr: false, + }, + { + name: "Garbage Data", + substr: "will be replaced with garbage data", + wantNil: true, + wantErr: false, + }, +} + +// This tests whether we can find a substring in all of +// the known status messages, disregarding the metadata +// stored with each status. +func Test_Registry_QueryInStatus(t *testing.T) { + registry := initTestEnv() + var buf = make([]byte, 256) + // read random data into case 8 + rando, _ := os.Open("/dev/random") + reader := bufio.NewReader(rando) + n, err := reader.Read(buf) + if err != nil || n == 0 { + t.Errorf("Couldn't set up test: %v\n", err) + } + queryInStatusCases[7].substr = string(buf) + + for _, tt := range queryInStatusCases { + + t.Run(tt.name, func(t *testing.T) { + + out, err := registry.QueryInStatus(tt.substr) + if err != nil && !tt.wantErr { + t.Errorf("Caught unexpected error: %v\n", err) + } + + if !tt.wantErr && out == nil && !tt.wantNil { + t.Errorf("Got nil when expecting output\n") + } + + if err == nil && tt.wantErr { + t.Errorf("Expecting error, got nil.\n") + } + + for _, e := range out { + split := strings.Split(strings.ToLower(e), "\t") + + if e != "" { + if !strings.Contains(split[3], strings.ToLower(tt.substr)) { + t.Errorf("Status without substring returned\n") + } + } + } + }) + } + +} +func Benchmark_Registry_QueryInStatus(b *testing.B) { + registry := initTestEnv() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + for _, tt := range queryInStatusCases { + _, err := registry.QueryInStatus(tt.substr) + if err != nil { + continue + } + } + } +} + +// Tests whether we can retrieve the 20 most +// recent statuses in the registry +func Test_QueryAllStatuses(t *testing.T) { + registry := initTestEnv() + t.Run("Latest Statuses", func(t *testing.T) { + out, err := registry.QueryAllStatuses() + if out == nil || err != nil { + t.Errorf("Got no statuses, or more than 20: %v, %v\n", len(out), err) + } + }) +} +func Benchmark_QueryAllStatuses(b *testing.B) { + registry := initTestEnv() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := registry.QueryAllStatuses() + if err != nil { + continue + } + } +} + +var get20cases = []struct { + name string + page int + wantErr bool +}{ + { + name: "First Page", + page: 1, + wantErr: false, + }, + { + name: "High Page Number", + page: 256, + wantErr: false, + }, + { + name: "Illegal Page Number", + page: -23, + wantErr: false, + }, +} + +func Test_ReduceToPage(t *testing.T) { + registry := initTestEnv() + for _, tt := range get20cases { + t.Run(tt.name, func(t *testing.T) { + out, err := registry.QueryAllStatuses() + if err != nil && !tt.wantErr { + t.Errorf("%v\n", err.Error()) + } + out = ReduceToPage(tt.page, out) + if len(out) > 20 || len(out) == 0 { + t.Errorf("Page-Reduce Malfunction: length of data %v\n", len(out)) + } + }) + } +} + +func Benchmark_ReduceToPage(b *testing.B) { + registry := initTestEnv() + out, _ := registry.QueryAllStatuses() + b.ResetTimer() + for i := 0; i < b.N; i++ { + for _, tt := range get20cases { + ReduceToPage(tt.page, out) + } + } +} + +// This tests whether we can find a substring in the +// given user's status messages, disregarding the metadata +// stored with each status. +func Test_User_FindInStatus(t *testing.T) { + registry := initTestEnv() + var buf = make([]byte, 256) + // read random data into case 8 + rando, _ := os.Open("/dev/random") + reader := bufio.NewReader(rando) + n, err := reader.Read(buf) + if err != nil || n == 0 { + t.Errorf("Couldn't set up test: %v\n", err) + } + queryInStatusCases[7].substr = string(buf) + + data := make([]*User, 0) + + for _, v := range registry.Users { + data = append(data, v) + } + + for _, tt := range queryInStatusCases { + t.Run(tt.name, func(t *testing.T) { + for _, e := range data { + + tag := e.FindInStatus(tt.substr) + if tag == nil && !tt.wantNil { + t.Errorf("Got nil tag\n") + } + } + }) + } + +} +func Benchmark_User_FindInStatus(b *testing.B) { + registry := initTestEnv() + data := make([]*User, 0) + + for _, v := range registry.Users { + data = append(data, v) + } + b.ResetTimer() + + for i := 0; i < b.N; i++ { + for _, tt := range data { + for _, v := range queryInStatusCases { + tt.FindInStatus(v.substr) + } + } + } +} + +func Test_SortByTime_Slice(t *testing.T) { + registry := initTestEnv() + + statusmap, err := registry.GetStatuses() + if err != nil { + t.Errorf("Failed to finish test initialization: %v\n", err) + } + + t.Run("Sort By Time ([]TimeMap)", func(t *testing.T) { + sorted, err := SortByTime(statusmap) + if err != nil { + t.Errorf("%v\n", err) + } + split := strings.Split(sorted[0], "\t") + firsttime, _ := time.Parse("RFC3339", split[0]) + + for i := range sorted { + if i < len(sorted)-1 { + + nextsplit := strings.Split(sorted[i+1], "\t") + nexttime, _ := time.Parse("RFC3339", nextsplit[0]) + + if firsttime.Before(nexttime) { + t.Errorf("Timestamps out of order: %v\n", sorted) + } + + firsttime = nexttime + } + } + }) +} + +// Benchmarking a sort of 1000000 statuses by timestamp. +// Right now it's at roughly 2000ns per 2 statuses. +// Set sortMultiplier to be the number of desired +// statuses divided by four. +func Benchmark_SortByTime_Slice(b *testing.B) { + // I set this to 250,000,000 and it hard-locked + // my laptop. Oops. + sortMultiplier := 250 + b.Logf("Benchmarking SortByTime with a constructed slice of %v statuses ...\n", sortMultiplier*4) + registry := initTestEnv() + + statusmap, err := registry.GetStatuses() + if err != nil { + b.Errorf("Failed to finish benchmark initialization: %v\n", err) + } + + // Constructed registry has four statuses. This + // makes a TimeMapSlice of 1000000 statuses. + statusmaps := make([]TimeMap, sortMultiplier*4) + for i := 0; i < sortMultiplier; i++ { + statusmaps = append(statusmaps, statusmap) + } + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, err := SortByTime(statusmaps...) + if err != nil { + b.Errorf("%v\n", err) + } + } +} + +func Test_SortByTime_Single(t *testing.T) { + registry := initTestEnv() + + statusmap, err := registry.GetStatuses() + if err != nil { + t.Errorf("Failed to finish test initialization: %v\n", err) + } + + t.Run("Sort By Time (TimeMap)", func(t *testing.T) { + sorted, err := SortByTime(statusmap) + if err != nil { + t.Errorf("%v\n", err) + } + split := strings.Split(sorted[0], "\t") + firsttime, _ := time.Parse("RFC3339", split[0]) + + for i := range sorted { + if i < len(sorted)-1 { + + nextsplit := strings.Split(sorted[i+1], "\t") + nexttime, _ := time.Parse("RFC3339", nextsplit[0]) + + if firsttime.Before(nexttime) { + t.Errorf("Timestamps out of order: %v\n", sorted) + } + + firsttime = nexttime + } + } + }) +} + +func Benchmark_SortByTime_Single(b *testing.B) { + registry := initTestEnv() + + statusmap, err := registry.GetStatuses() + if err != nil { + b.Errorf("Failed to finish benchmark initialization: %v\n", err) + } + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, err := SortByTime(statusmap) + if err != nil { + b.Errorf("%v\n", err) + } + } +} diff --git a/registry/revive.toml b/registry/revive.toml new file mode 100644 index 0000000..f9e2405 --- /dev/null +++ b/registry/revive.toml @@ -0,0 +1,30 @@ +ignoreGeneratedHeader = false +severity = "warning" +confidence = 0.8 +errorCode = 0 +warningCode = 0 + +[rule.blank-imports] +[rule.context-as-argument] +[rule.context-keys-type] +[rule.dot-imports] +[rule.error-return] +[rule.error-strings] +[rule.error-naming] +[rule.exported] +[rule.if-return] +[rule.increment-decrement] +[rule.var-naming] +[rule.var-declaration] +[rule.package-comments] +[rule.range] +[rule.receiver-naming] +[rule.time-naming] +[rule.unexported-return] +[rule.indent-error-flow] +[rule.errorf] +[rule.empty-block] +[rule.superfluous-else] +[rule.unused-parameter] +[rule.unreachable-code] +[rule.redefines-builtin-id] diff --git a/registry/types.go b/registry/types.go new file mode 100644 index 0000000..eb8eee1 --- /dev/null +++ b/registry/types.go @@ -0,0 +1,148 @@ +/* +Copyright (c) 2019 Ben Morrison (gbmor) + +This file is part of Registry. + +Registry is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +Registry is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Registry. If not, see <https://www.gnu.org/licenses/>. +*/ + +// Package registry implements functions and types that assist +// in the creation and management of a twtxt registry. +package registry // import "git.sr.ht/~gbmor/getwtxt/registry" + +import ( + "net" + "net/http" + "sync" + "time" +) + +// Registrar implements the minimum amount of methods +// for a functioning Registry. +type Registrar interface { + Put(user *User) error + Get(urlKey string) (*User, error) + DelUser(urlKey string) error + UpdateUser(urlKey string) error + GetUserStatuses(urlKey string) (TimeMap, error) + GetStatuses() (TimeMap, error) +} + +// User holds a given user's information +// and statuses. +type User struct { + // Provided to aid in concurrency-safe + // reads and writes. In most cases, the + // mutex in the associated Index should be + // used instead. This mutex is provided + // should the library user need to access + // a User independently of an Index. + Mu sync.RWMutex + + // Nick is the user-specified nickname. + Nick string + + // The URL of the user's twtxt file + URL string + + // The reported last modification date + // of the user's twtxt.txt file. + LastModified string + + // The IP address of the user is optionally + // recorded when submitted via POST. + IP net.IP + + // The timestamp, in RFC3339 format, + // reflecting when the user was added. + Date string + + // A TimeMap of the user's statuses + // from their twtxt file. + Status TimeMap +} + +// Registry enables the bulk of a registry's +// user data storage and access. +type Registry struct { + // Provided to aid in concurrency-safe + // reads and writes to a given registry + // Users map. + Mu sync.RWMutex + + // The registry's user data is contained + // in this map. The functions within this + // library expect the key to be the URL of + // a given user's twtxt file. + Users map[string]*User + + // The client to use for HTTP requests. + // If nil is passed to NewIndex(), a + // client with a 10 second timeout + // and all other values as default is + // used. + HTTPClient *http.Client +} + +// TimeMap holds extracted and processed user data as a +// string. A time.Time value is used as the key. +type TimeMap map[time.Time]string + +// TimeSlice is a slice of time.Time used for sorting +// a TimeMap by timestamp. +type TimeSlice []time.Time + +// NewUser returns a pointer to an initialized User +func NewUser() *User { + return &User{ + Mu: sync.RWMutex{}, + Status: NewTimeMap(), + } +} + +// New returns an initialized Registry instance. +func New(client *http.Client) *Registry { + return &Registry{ + Mu: sync.RWMutex{}, + Users: make(map[string]*User), + HTTPClient: client, + } +} + +// NewTimeMap returns an initialized TimeMap. +func NewTimeMap() TimeMap { + return make(TimeMap) +} + +// Len returns the length of the TimeSlice to be sorted. +// This helps satisfy sort.Interface. +func (t TimeSlice) Len() int { + return len(t) +} + +// Less returns true if the timestamp at index i is after +// the timestamp at index j in a given TimeSlice. This results +// in a descending (reversed) sort order for timestamps rather +// than ascending. +// This helps satisfy sort.Interface. +func (t TimeSlice) Less(i, j int) bool { + return t[i].After(t[j]) +} + +// Swap transposes the timestamps at the two given indices +// for the TimeSlice receiver. +// This helps satisfy sort.Interface. +func (t TimeSlice) Swap(i, j int) { + t[i], t[j] = t[j], t[i] +} diff --git a/registry/user.go b/registry/user.go new file mode 100644 index 0000000..329b6e3 --- /dev/null +++ b/registry/user.go @@ -0,0 +1,270 @@ +/* +Copyright (c) 2019 Ben Morrison (gbmor) + +This file is part of Registry. + +Registry is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +Registry is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Registry. If not, see <https://www.gnu.org/licenses/>. +*/ + +package registry // import "git.sr.ht/~gbmor/getwtxt/registry" + +import ( + "fmt" + "net" + "strings" + "sync" + "time" +) + +// AddUser inserts a new user into the Registry. +func (registry *Registry) AddUser(nickname, urlKey string, ipAddress net.IP, statuses TimeMap) error { + + if registry == nil { + return fmt.Errorf("can't add user to uninitialized registry") + + } else if nickname == "" || urlKey == "" { + return fmt.Errorf("both URL and Nick must be specified") + + } else if !strings.HasPrefix(urlKey, "http") { + return fmt.Errorf("invalid URL: %v", urlKey) + } + + registry.Mu.Lock() + defer registry.Mu.Unlock() + + if _, ok := registry.Users[urlKey]; ok { + return fmt.Errorf("user %v already exists", urlKey) + } + + registry.Users[urlKey] = &User{ + Mu: sync.RWMutex{}, + Nick: nickname, + URL: urlKey, + LastModified: "", + IP: ipAddress, + Date: time.Now().Format(time.RFC3339), + Status: statuses} + + return nil +} + +// Put inserts a given User into an Registry. The User +// being pushed need only have the URL field filled. +// All other fields may be empty. +// This can be destructive: an existing User in the +// Registry will be overwritten if its User.URL is the +// same as the User.URL being pushed. +func (registry *Registry) Put(user *User) error { + if user == nil { + return fmt.Errorf("can't push nil data to registry") + } + if registry == nil || registry.Users == nil { + return fmt.Errorf("can't push data to registry: registry uninitialized") + } + user.Mu.RLock() + if user.URL == "" { + user.Mu.RUnlock() + return fmt.Errorf("can't push data to registry: missing URL for key") + } + urlKey := user.URL + registry.Mu.Lock() + registry.Users[urlKey] = user + registry.Mu.Unlock() + user.Mu.RUnlock() + + return nil +} + +// Get returns the User associated with the +// provided URL key in the Registry. +func (registry *Registry) Get(urlKey string) (*User, error) { + if registry == nil { + return nil, fmt.Errorf("can't pop from nil registry") + } + if urlKey == "" { + return nil, fmt.Errorf("can't pop unless provided a key") + } + + registry.Mu.RLock() + defer registry.Mu.RUnlock() + + if _, ok := registry.Users[urlKey]; !ok { + return nil, fmt.Errorf("provided url key doesn't exist in registry") + } + + registry.Users[urlKey].Mu.RLock() + userGot := registry.Users[urlKey] + registry.Users[urlKey].Mu.RUnlock() + + return userGot, nil +} + +// DelUser removes a user and all associated data from +// the Registry. +func (registry *Registry) DelUser(urlKey string) error { + + if registry == nil { + return fmt.Errorf("can't delete user from empty registry") + + } else if urlKey == "" { + return fmt.Errorf("can't delete blank user") + + } else if !strings.HasPrefix(urlKey, "http") { + return fmt.Errorf("invalid URL: %v", urlKey) + } + + registry.Mu.Lock() + defer registry.Mu.Unlock() + + if _, ok := registry.Users[urlKey]; !ok { + return fmt.Errorf("can't delete user %v, user doesn't exist", urlKey) + } + + delete(registry.Users, urlKey) + + return nil +} + +// UpdateUser scrapes an existing user's remote twtxt.txt +// file. Any new statuses are added to the user's entry +// in the Registry. If the remote twtxt data's reported +// Content-Length does not differ from what is stored, +// an error is returned. +func (registry *Registry) UpdateUser(urlKey string) error { + if urlKey == "" || !strings.HasPrefix(urlKey, "http") { + return fmt.Errorf("invalid URL: %v", urlKey) + } + + diff, err := registry.DiffTwtxt(urlKey) + if err != nil { + return err + } else if !diff { + return fmt.Errorf("no new statuses available for %v", urlKey) + } + + out, isRemoteRegistry, err := GetTwtxt(urlKey, registry.HTTPClient) + if err != nil { + return err + } + + if isRemoteRegistry { + return fmt.Errorf("attempting to update registry URL - users should be updated individually") + } + + registry.Mu.Lock() + defer registry.Mu.Unlock() + user := registry.Users[urlKey] + + user.Mu.Lock() + defer user.Mu.Unlock() + nick := user.Nick + + data, err := ParseUserTwtxt(out, nick, urlKey) + if err != nil { + return err + } + + for i, e := range data { + user.Status[i] = e + } + + registry.Users[urlKey] = user + + return nil +} + +// CrawlRemoteRegistry scrapes all nicknames and user URLs +// from a provided registry. The urlKey passed to this function +// must be in the form of https://registry.example.com/api/plain/users +func (registry *Registry) CrawlRemoteRegistry(urlKey string) error { + if urlKey == "" || !strings.HasPrefix(urlKey, "http") { + return fmt.Errorf("invalid URL: %v", urlKey) + } + + out, isRemoteRegistry, err := GetTwtxt(urlKey, registry.HTTPClient) + if err != nil { + return err + } + + if !isRemoteRegistry { + return fmt.Errorf("can't add single user via call to CrawlRemoteRegistry") + } + + users, err := ParseRegistryTwtxt(out) + if err != nil { + return err + } + + // only add new users so we don't overwrite data + // we already have (and lose statuses, etc) + registry.Mu.Lock() + defer registry.Mu.Unlock() + for _, e := range users { + if _, ok := registry.Users[e.URL]; !ok { + registry.Users[e.URL] = e + } + } + + return nil +} + +// GetUserStatuses returns a TimeMap containing single user's statuses +func (registry *Registry) GetUserStatuses(urlKey string) (TimeMap, error) { + if registry == nil { + return nil, fmt.Errorf("can't get statuses from an empty registry") + } else if urlKey == "" || !strings.HasPrefix(urlKey, "http") { + return nil, fmt.Errorf("invalid URL: %v", urlKey) + } + + registry.Mu.RLock() + defer registry.Mu.RUnlock() + if _, ok := registry.Users[urlKey]; !ok { + return nil, fmt.Errorf("can't retrieve statuses of nonexistent user") + } + + registry.Users[urlKey].Mu.RLock() + status := registry.Users[urlKey].Status + registry.Users[urlKey].Mu.RUnlock() + + return status, nil +} + +// GetStatuses returns a TimeMap containing all statuses +// from all users in the Registry. +func (registry *Registry) GetStatuses() (TimeMap, error) { + if registry == nil { + return nil, fmt.Errorf("can't get statuses from an empty registry") + } + + statuses := NewTimeMap() + + registry.Mu.RLock() + defer registry.Mu.RUnlock() + + for _, v := range registry.Users { + v.Mu.RLock() + if v.Status == nil || len(v.Status) == 0 { + v.Mu.RUnlock() + continue + } + for a, b := range v.Status { + if _, ok := v.Status[a]; ok { + statuses[a] = b + } + } + v.Mu.RUnlock() + } + + return statuses, nil +} diff --git a/registry/user_test.go b/registry/user_test.go new file mode 100644 index 0000000..f0c9622 --- /dev/null +++ b/registry/user_test.go @@ -0,0 +1,349 @@ +/* +Copyright (c) 2019 Ben Morrison (gbmor) + +This file is part of Registry. + +Registry is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +Registry is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Registry. If not, see <https://www.gnu.org/licenses/>. +*/ + +package registry // import "git.sr.ht/~gbmor/getwtxt/registry" + +import ( + "bufio" + "fmt" + "net/http" + "os" + "reflect" + "testing" +) + +var addUserCases = []struct { + name string + nick string + url string + wantErr bool + localOnly bool +}{ + { + name: "Legitimate User (Local Only)", + nick: "testuser1", + url: "http://localhost:8080/twtxt.txt", + wantErr: false, + localOnly: true, + }, + { + name: "Empty Query", + nick: "", + url: "", + wantErr: true, + localOnly: false, + }, + { + name: "Invalid URL", + nick: "foo", + url: "foobarringtons", + wantErr: true, + localOnly: false, + }, + { + name: "Garbage Data", + nick: "", + url: "", + wantErr: true, + localOnly: false, + }, +} + +// Tests if we can successfully add a user to the registry +func Test_Registry_AddUser(t *testing.T) { + registry := initTestEnv() + if !addUserCases[0].localOnly { + http.Handle("/twtxt.txt", http.HandlerFunc(twtxtHandler)) + go fmt.Println(http.ListenAndServe(":8080", nil)) + } + var buf = make([]byte, 256) + // read random data into case 5 + rando, _ := os.Open("/dev/random") + reader := bufio.NewReader(rando) + n, err := reader.Read(buf) + if err != nil || n == 0 { + t.Errorf("Couldn't set up test: %v\n", err) + } + addUserCases[3].nick = string(buf) + addUserCases[3].url = string(buf) + + statuses, err := registry.GetStatuses() + if err != nil { + t.Errorf("Error setting up test: %v\n", err) + } + + for n, tt := range addUserCases { + t.Run(tt.name, func(t *testing.T) { + if tt.localOnly { + t.Skipf("Local-only test. Skipping ... ") + } + + err := registry.AddUser(tt.nick, tt.url, nil, statuses) + + // only run some checks if we don't want an error + if !tt.wantErr { + if err != nil { + t.Errorf("Got error: %v\n", err) + } + + // make sure we have *something* in the registry + if reflect.ValueOf(registry.Users[tt.url]).IsNil() { + t.Errorf("Failed to add user %v registry.\n", tt.url) + } + + // see if the nick in the registry is the same + // as the test case. verifies the URL and the nick + // since the URL is used as the key + data := registry.Users[tt.url] + if data.Nick != tt.nick { + t.Errorf("Incorrect user data added to registry for user %v.\n", tt.url) + } + } + // check for the cases that should throw an error + if tt.wantErr && err == nil { + t.Errorf("Expected error for case %v, got nil\n", n) + } + }) + } +} +func Benchmark_Registry_AddUser(b *testing.B) { + registry := initTestEnv() + statuses, err := registry.GetStatuses() + if err != nil { + b.Errorf("Error setting up test: %v\n", err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + for _, tt := range addUserCases { + err := registry.AddUser(tt.nick, tt.url, nil, statuses) + if err != nil { + continue + } + registry.Users[tt.url] = &User{} + } + } +} + +var delUserCases = []struct { + name string + url string + wantErr bool +}{ + { + name: "Valid User", + url: "https://example.com/twtxt.txt", + wantErr: false, + }, + { + name: "Valid User", + url: "https://example3.com/twtxt.txt", + wantErr: false, + }, + { + name: "Already Deleted User", + url: "https://example3.com/twtxt.txt", + wantErr: true, + }, + { + name: "Empty Query", + url: "", + wantErr: true, + }, + { + name: "Garbage Data", + url: "", + wantErr: true, + }, +} + +// Tests if we can successfully delete a user from the registry +func Test_Registry_DelUser(t *testing.T) { + registry := initTestEnv() + var buf = make([]byte, 256) + // read random data into case 5 + rando, _ := os.Open("/dev/random") + reader := bufio.NewReader(rando) + n, err := reader.Read(buf) + if err != nil || n == 0 { + t.Errorf("Couldn't set up test: %v\n", err) + } + delUserCases[4].url = string(buf) + + for n, tt := range delUserCases { + t.Run(tt.name, func(t *testing.T) { + + err := registry.DelUser(tt.url) + if !reflect.ValueOf(registry.Users[tt.url]).IsNil() { + t.Errorf("Failed to delete user %v from registry.\n", tt.url) + } + if tt.wantErr && err == nil { + t.Errorf("Expected error but did not receive. Case %v\n", n) + } + if !tt.wantErr && err != nil { + t.Errorf("Unexpected error for case %v: %v\n", n, err) + } + }) + } +} +func Benchmark_Registry_DelUser(b *testing.B) { + registry := initTestEnv() + + data1 := &User{ + Nick: registry.Users[delUserCases[0].url].Nick, + Date: registry.Users[delUserCases[0].url].Date, + Status: registry.Users[delUserCases[0].url].Status, + } + + data2 := &User{ + Nick: registry.Users[delUserCases[1].url].Nick, + Date: registry.Users[delUserCases[1].url].Date, + Status: registry.Users[delUserCases[1].url].Status, + } + b.ResetTimer() + + for i := 0; i < b.N; i++ { + for _, tt := range delUserCases { + err := registry.DelUser(tt.url) + if err != nil { + continue + } + } + + registry.Users[delUserCases[0].url] = data1 + registry.Users[delUserCases[1].url] = data2 + } +} + +var getUserStatusCases = []struct { + name string + url string + wantErr bool +}{ + { + name: "Valid User", + url: "https://example.com/twtxt.txt", + wantErr: false, + }, + { + name: "Valid User", + url: "https://example3.com/twtxt.txt", + wantErr: false, + }, + { + name: "Nonexistent User", + url: "https://doesn't.exist/twtxt.txt", + wantErr: true, + }, + { + name: "Empty Query", + url: "", + wantErr: true, + }, + { + name: "Garbage Data", + url: "", + wantErr: true, + }, +} + +// Checks if we can retrieve a single user's statuses +func Test_Registry_GetUserStatuses(t *testing.T) { + registry := initTestEnv() + var buf = make([]byte, 256) + // read random data into case 5 + rando, _ := os.Open("/dev/random") + reader := bufio.NewReader(rando) + n, err := reader.Read(buf) + if err != nil || n == 0 { + t.Errorf("Couldn't set up test: %v\n", err) + } + getUserStatusCases[4].url = string(buf) + + for n, tt := range getUserStatusCases { + t.Run(tt.name, func(t *testing.T) { + + statuses, err := registry.GetUserStatuses(tt.url) + + if !tt.wantErr { + if reflect.ValueOf(statuses).IsNil() { + t.Errorf("Failed to pull statuses for user %v\n", tt.url) + } + // see if the function returns the same data + // that we already have + data := registry.Users[tt.url] + if !reflect.DeepEqual(data.Status, statuses) { + t.Errorf("Incorrect data retrieved as statuses for user %v.\n", tt.url) + } + } + + if tt.wantErr && err == nil { + t.Errorf("Expected error, received nil for case %v: %v\n", n, tt.url) + } + }) + } +} +func Benchmark_Registry_GetUserStatuses(b *testing.B) { + registry := initTestEnv() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + for _, tt := range getUserStatusCases { + _, err := registry.GetUserStatuses(tt.url) + if err != nil { + continue + } + } + } +} + +// Tests if we can retrieve all user statuses at once +func Test_Registry_GetStatuses(t *testing.T) { + registry := initTestEnv() + t.Run("Registry.GetStatuses()", func(t *testing.T) { + + statuses, err := registry.GetStatuses() + if reflect.ValueOf(statuses).IsNil() || err != nil { + t.Errorf("Failed to pull all statuses. %v\n", err) + } + + // Now do the same query manually to see + // if we get the same result + unionmap := NewTimeMap() + for _, v := range registry.Users { + for i, e := range v.Status { + unionmap[i] = e + } + } + if !reflect.DeepEqual(statuses, unionmap) { + t.Errorf("Incorrect data retrieved as statuses.\n") + } + }) +} +func Benchmark_Registry_GetStatuses(b *testing.B) { + registry := initTestEnv() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, err := registry.GetStatuses() + if err != nil { + continue + } + } +} diff --git a/svc/README.md b/svc/README.md deleted file mode 100644 index 797b7ab..0000000 --- a/svc/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# `getwtxt|svc` - -This is an internal package created for code organization -purposes. - -The file `getwtxt.go` in the parent directory calls -`svc.go::Start()`, which starts the registry server. - diff --git a/svc/cache.go b/svc/cache.go index f887d05..16287bf 100644 --- a/svc/cache.go +++ b/svc/cache.go @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with Getwtxt. If not, see <https://www.gnu.org/licenses/>. */ -package svc // import "github.com/getwtxt/getwtxt/svc" +package svc // import "git.sr.ht/~gbmor/getwtxt/svc" import ( "bytes" diff --git a/svc/cache_test.go b/svc/cache_test.go index 96c2638..c37f099 100644 --- a/svc/cache_test.go +++ b/svc/cache_test.go @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with Getwtxt. If not, see <https://www.gnu.org/licenses/>. */ -package svc // import "github.com/getwtxt/getwtxt/svc" +package svc // import "git.sr.ht/~gbmor/getwtxt/svc" import ( "bytes" @@ -27,7 +27,7 @@ import ( "reflect" "testing" - "github.com/getwtxt/registry" + "git.sr.ht/~gbmor/getwtxt/registry" ) func Test_initTemplates(t *testing.T) { @@ -49,7 +49,7 @@ func Test_cacheUpdate(t *testing.T) { killStatuses() cacheUpdate() - urls := "https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt" + urls := testTwtxtURL newStatus := twtxtCache.Users[urls].Status t.Run("Checking for any data", func(t *testing.T) { diff --git a/svc/conf.go b/svc/conf.go index 8968ed4..7365b2b 100644 --- a/svc/conf.go +++ b/svc/conf.go @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with Getwtxt. If not, see <https://www.gnu.org/licenses/>. */ -package svc // import "github.com/getwtxt/getwtxt/svc" +package svc // import "git.sr.ht/~gbmor/getwtxt/svc" import ( "log" diff --git a/svc/db.go b/svc/db.go index 34b7c34..8cd05d1 100644 --- a/svc/db.go +++ b/svc/db.go @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with Getwtxt. If not, see <https://www.gnu.org/licenses/>. */ -package svc // import "github.com/getwtxt/getwtxt/svc" +package svc // import "git.sr.ht/~gbmor/getwtxt/svc" import ( "log" diff --git a/svc/db_test.go b/svc/db_test.go index f54d610..21fc972 100644 --- a/svc/db_test.go +++ b/svc/db_test.go @@ -17,30 +17,30 @@ You should have received a copy of the GNU General Public License along with Getwtxt. If not, see <https://www.gnu.org/licenses/>. */ -package svc // import "github.com/getwtxt/getwtxt/svc" +package svc // import "git.sr.ht/~gbmor/getwtxt/svc" import ( "net" "testing" - "github.com/getwtxt/registry" + "git.sr.ht/~gbmor/getwtxt/registry" ) func Test_pushpullDatabase(t *testing.T) { initTestConf() initTestDB() - out, _, err := registry.GetTwtxt("https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt", nil) + out, _, err := registry.GetTwtxt(testTwtxtURL, nil) if err != nil { t.Errorf("Couldn't set up test: %v\n", err) } - statusmap, err := registry.ParseUserTwtxt(out, "getwtxttest", "https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt") + statusmap, err := registry.ParseUserTwtxt(out, "getwtxttest", testTwtxtURL) if err != nil { t.Errorf("Couldn't set up test: %v\n", err) } - twtxtCache.AddUser("getwtxttest", "https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt", net.ParseIP("127.0.0.1"), statusmap) + twtxtCache.AddUser("getwtxttest", testTwtxtURL, net.ParseIP("127.0.0.1"), statusmap) remoteRegistries.List = append(remoteRegistries.List, "https://twtxt.tilde.institute/api/plain/users") @@ -52,7 +52,7 @@ func Test_pushpullDatabase(t *testing.T) { }) t.Run("Clearing Registry", func(t *testing.T) { - err := twtxtCache.DelUser("https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt") + err := twtxtCache.DelUser(testTwtxtURL) if err != nil { t.Errorf("%v", err) } @@ -62,7 +62,7 @@ func Test_pushpullDatabase(t *testing.T) { pullDB() twtxtCache.Mu.RLock() - if _, ok := twtxtCache.Users["https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt"]; !ok { + if _, ok := twtxtCache.Users[testTwtxtURL]; !ok { t.Errorf("Missing user previously pushed to database\n") } twtxtCache.Mu.RUnlock() @@ -73,18 +73,18 @@ func Benchmark_pushDatabase(b *testing.B) { initTestConf() initTestDB() - if _, ok := twtxtCache.Users["https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt"]; !ok { - out, _, err := registry.GetTwtxt("https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt", nil) + if _, ok := twtxtCache.Users[testTwtxtURL]; !ok { + out, _, err := registry.GetTwtxt(testTwtxtURL, nil) if err != nil { b.Errorf("Couldn't set up benchmark: %v\n", err) } - statusmap, err := registry.ParseUserTwtxt(out, "getwtxttest", "https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt") + statusmap, err := registry.ParseUserTwtxt(out, "getwtxttest", testTwtxtURL) if err != nil { b.Errorf("Couldn't set up benchmark: %v\n", err) } - twtxtCache.AddUser("getwtxttest", "https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt", net.ParseIP("127.0.0.1"), statusmap) + twtxtCache.AddUser("getwtxttest", testTwtxtURL, net.ParseIP("127.0.0.1"), statusmap) } b.ResetTimer() diff --git a/svc/handlers.go b/svc/handlers.go index 7ce4b77..cb07349 100644 --- a/svc/handlers.go +++ b/svc/handlers.go @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with Getwtxt. If not, see <https://www.gnu.org/licenses/>. */ -package svc // import "github.com/getwtxt/getwtxt/svc" +package svc // import "git.sr.ht/~gbmor/getwtxt/svc" import ( "fmt" @@ -27,7 +27,7 @@ import ( "strings" "time" - "github.com/getwtxt/registry" + "git.sr.ht/~gbmor/getwtxt/registry" "github.com/gorilla/mux" ) diff --git a/svc/handlers_test.go b/svc/handlers_test.go index a2d77e2..46151d7 100644 --- a/svc/handlers_test.go +++ b/svc/handlers_test.go @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with Getwtxt. If not, see <https://www.gnu.org/licenses/>. */ -package svc // import "github.com/getwtxt/getwtxt/svc" +package svc // import "git.sr.ht/~gbmor/getwtxt/svc" import ( "bytes" diff --git a/svc/help.go b/svc/help.go index 3cc60ff..6eb5be7 100644 --- a/svc/help.go +++ b/svc/help.go @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with Getwtxt. If not, see <https://www.gnu.org/licenses/>. */ -package svc // import "github.com/getwtxt/getwtxt/svc" +package svc // import "git.sr.ht/~gbmor/getwtxt/svc" import "fmt" @@ -31,7 +31,7 @@ func titleScreen() { \__, |\___|\__| \_/\_/ \__/_/\_\\__| |___/ version ` + Vers + ` - github.com/getwtxt/getwtxt + git.sr.ht/~gbmor/getwtxt GPL v3 `) @@ -284,7 +284,7 @@ func manualScreen() { Adding a user: curl -X POST 'http://localhost:9001/api/plain/users\ - ?url=https://gbmor.dev/twtxt.txt&nickname=gbmor' + ?url=https://example.org/twtxt.txt&nickname=somebody' Retrieve user list: curl 'http://localhost:9001/api/plain/users' diff --git a/svc/http.go b/svc/http.go index 6350e9e..c137d08 100644 --- a/svc/http.go +++ b/svc/http.go @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with Getwtxt. If not, see <https://www.gnu.org/licenses/>. */ -package svc // import "github.com/getwtxt/getwtxt/svc" +package svc // import "git.sr.ht/~gbmor/getwtxt/svc" import ( "context" diff --git a/svc/init.go b/svc/init.go index 00e107b..3f9d505 100644 --- a/svc/init.go +++ b/svc/init.go @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with Getwtxt. If not, see <https://www.gnu.org/licenses/>. */ -package svc // import "github.com/getwtxt/getwtxt/svc" +package svc // import "git.sr.ht/~gbmor/getwtxt/svc" import ( "html/template" @@ -26,7 +26,7 @@ import ( "os/signal" "time" - "github.com/getwtxt/registry" + "git.sr.ht/~gbmor/getwtxt/registry" "github.com/spf13/pflag" ) diff --git a/svc/init_test.go b/svc/init_test.go index 2fbfd5e..0c255c5 100644 --- a/svc/init_test.go +++ b/svc/init_test.go @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with Getwtxt. If not, see <https://www.gnu.org/licenses/>. */ -package svc // import "github.com/getwtxt/getwtxt/svc" +package svc // import "git.sr.ht/~gbmor/getwtxt/svc" import ( "bytes" @@ -29,10 +29,12 @@ import ( "sync" "testing" - "github.com/getwtxt/registry" + "git.sr.ht/~gbmor/getwtxt/registry" "github.com/spf13/viper" ) +const testTwtxtURL = "https://git.sr.ht/~gbmor/getwtxt/blob/master/testdata/twtxt.txt" + var ( testport string initTestOnce sync.Once @@ -111,21 +113,21 @@ func testConfig() { // user and their statuses, for testing. func mockRegistry() { twtxtCache = registry.New(nil) - statuses, _, _ := registry.GetTwtxt("https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt", nil) - parsed, _ := registry.ParseUserTwtxt(statuses, "getwtxttest", "https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt") - _ = twtxtCache.AddUser("getwtxttest", "https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt", net.ParseIP("127.0.0.1"), parsed) + statuses, _, _ := registry.GetTwtxt(testTwtxtURL, nil) + parsed, _ := registry.ParseUserTwtxt(statuses, "getwtxttest", testTwtxtURL) + _ = twtxtCache.AddUser("getwtxttest", testTwtxtURL, net.ParseIP("127.0.0.1"), parsed) } // Empties the mock registry's user of statuses // for functions that test status modifications func killStatuses() { twtxtCache.Mu.Lock() - user := twtxtCache.Users["https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt"] + user := twtxtCache.Users[testTwtxtURL] user.Mu.Lock() user.Status = registry.NewTimeMap() user.LastModified = "0" - twtxtCache.Users["https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt"] = user + twtxtCache.Users[testTwtxtURL] = user user.Mu.Unlock() twtxtCache.Mu.Unlock() diff --git a/svc/leveldb.go b/svc/leveldb.go index 16751ce..5fb4a45 100644 --- a/svc/leveldb.go +++ b/svc/leveldb.go @@ -17,14 +17,14 @@ You should have received a copy of the GNU General Public License along with Getwtxt. If not, see <https://www.gnu.org/licenses/>. */ -package svc // import "github.com/getwtxt/getwtxt/svc" +package svc // import "git.sr.ht/~gbmor/getwtxt/svc" import ( "net" "strings" "time" - "github.com/getwtxt/registry" + "git.sr.ht/~gbmor/getwtxt/registry" "github.com/syndtr/goleveldb/leveldb" ) diff --git a/svc/periodic.go b/svc/periodic.go index c8b918b..c6818a6 100644 --- a/svc/periodic.go +++ b/svc/periodic.go @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with Getwtxt. If not, see <https://www.gnu.org/licenses/>. */ -package svc // import "github.com/getwtxt/getwtxt/svc" +package svc // import "git.sr.ht/~gbmor/getwtxt/svc" import ( "log" diff --git a/svc/post.go b/svc/post.go index db4afd7..bba8136 100644 --- a/svc/post.go +++ b/svc/post.go @@ -17,14 +17,14 @@ You should have received a copy of the GNU General Public License along with Getwtxt. If not, see <https://www.gnu.org/licenses/>. */ -package svc // import "github.com/getwtxt/getwtxt/svc" +package svc // import "git.sr.ht/~gbmor/getwtxt/svc" import ( "fmt" "net/http" "strings" - "github.com/getwtxt/registry" + "git.sr.ht/~gbmor/getwtxt/registry" ) // Requests to apiEndpointPOSTHandler are passed off to this diff --git a/svc/post_test.go b/svc/post_test.go index cd15565..6f68a66 100644 --- a/svc/post_test.go +++ b/svc/post_test.go @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with Getwtxt. If not, see <https://www.gnu.org/licenses/>. */ -package svc // import "github.com/getwtxt/getwtxt/svc" +package svc // import "git.sr.ht/~gbmor/getwtxt/svc" import ( "fmt" @@ -27,7 +27,7 @@ import ( "strings" "testing" - "github.com/getwtxt/registry" + "git.sr.ht/~gbmor/getwtxt/registry" ) var apiPostUserCases = []struct { @@ -39,7 +39,7 @@ var apiPostUserCases = []struct { { name: "Known Good User", nick: "getwtxttest", - uri: "https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt", + uri: testTwtxtURL, wantErr: false, }, { @@ -100,7 +100,7 @@ func Benchmark_apiPostUser(b *testing.B) { twtxtCache = registry.New(nil) params := url.Values{} - params.Set("url", "https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt") + params.Set("url", testTwtxtURL) params.Set("nickname", "gbmor") req, _ := http.NewRequest("POST", "https://localhost"+portnum+"/api/plain/users", strings.NewReader(params.Encode())) req.Header.Add("Content-Type", "application/x-www-form-urlencoded") diff --git a/svc/query.go b/svc/query.go index 8f22838..6da3601 100644 --- a/svc/query.go +++ b/svc/query.go @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with Getwtxt. If not, see <https://www.gnu.org/licenses/>. */ -package svc // import "github.com/getwtxt/getwtxt/svc" +package svc // import "git.sr.ht/~gbmor/getwtxt/svc" import ( "crypto/sha256" @@ -28,7 +28,7 @@ import ( "strings" "sync" - "github.com/getwtxt/registry" + "git.sr.ht/~gbmor/getwtxt/registry" "github.com/gorilla/mux" ) diff --git a/svc/query_test.go b/svc/query_test.go index d2d7304..939592e 100644 --- a/svc/query_test.go +++ b/svc/query_test.go @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with Getwtxt. If not, see <https://www.gnu.org/licenses/>. */ -package svc // import "github.com/getwtxt/getwtxt/svc" +package svc // import "git.sr.ht/~gbmor/getwtxt/svc" import ( "net" @@ -25,7 +25,7 @@ import ( "strings" "testing" - "github.com/getwtxt/registry" + "git.sr.ht/~gbmor/getwtxt/registry" ) func Test_dedupe(t *testing.T) { @@ -61,7 +61,7 @@ func Benchmark_dedupe(b *testing.B) { func Test_parseQueryOut(t *testing.T) { initTestConf() - urls := "https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt" + urls := testTwtxtURL nick := "getwtxttest" out, _, err := registry.GetTwtxt(urls, nil) @@ -95,7 +95,7 @@ func Test_parseQueryOut(t *testing.T) { func Benchmark_parseQueryOut(b *testing.B) { initTestConf() - urls := "https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt" + urls := testTwtxtURL nick := "getwtxttest" out, _, err := registry.GetTwtxt(urls, nil) @@ -203,9 +203,9 @@ func Test_compositeStatusQuery(t *testing.T) { func Benchmark_compositeStatusQuery(b *testing.B) { initTestConf() - statuses, _, _ := registry.GetTwtxt("https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt", nil) - parsed, _ := registry.ParseUserTwtxt(statuses, "getwtxttest", "https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt") - _ = twtxtCache.AddUser("getwtxttest", "https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt", net.ParseIP("127.0.0.1"), parsed) + statuses, _, _ := registry.GetTwtxt(testTwtxtURL, nil) + parsed, _ := registry.ParseUserTwtxt(statuses, "getwtxttest", testTwtxtURL) + _ = twtxtCache.AddUser("getwtxttest", testTwtxtURL, net.ParseIP("127.0.0.1"), parsed) b.ResetTimer() for i := 0; i < b.N; i++ { diff --git a/svc/sqlite.go b/svc/sqlite.go index 58a804d..128aed3 100644 --- a/svc/sqlite.go +++ b/svc/sqlite.go @@ -17,14 +17,14 @@ You should have received a copy of the GNU General Public License along with Getwtxt. If not, see <https://www.gnu.org/licenses/>. */ -package svc // import "github.com/getwtxt/getwtxt/svc" +package svc // import "git.sr.ht/~gbmor/getwtxt/svc" import ( "database/sql" "net" "time" - "github.com/getwtxt/registry" + "git.sr.ht/~gbmor/getwtxt/registry" _ "github.com/mattn/go-sqlite3" // for the sqlite3 driver ) diff --git a/svc/svc.go b/svc/svc.go index 8c1c63a..72ccdc3 100644 --- a/svc/svc.go +++ b/svc/svc.go @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with Getwtxt. If not, see <https://www.gnu.org/licenses/>. */ -package svc // import "github.com/getwtxt/getwtxt/svc" +package svc // import "git.sr.ht/~gbmor/getwtxt/svc" import ( "fmt" diff --git a/testdata/twtxt.txt b/testdata/twtxt.txt index ab28c66..1365751 100644 --- a/testdata/twtxt.txt +++ b/testdata/twtxt.txt @@ -18,7 +18,7 @@ # == Metadata == # # nick = getwtxttest -# url = https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt +# url = https://git.sr.ht/~gbmor/getwtxt/blob/master/testdata/twtxt.txt # # == Content == # |