diff options
| -rw-r--r-- | assets/tmpl/index.html | 8 | ||||
| -rw-r--r-- | go.mod | 2 | ||||
| -rw-r--r-- | go.sum | 4 | ||||
| -rw-r--r-- | handlers.go | 40 | ||||
| -rw-r--r-- | main.go | 40 | ||||
| -rw-r--r-- | query.go | 58 |
6 files changed, 117 insertions, 35 deletions
diff --git a/assets/tmpl/index.html b/assets/tmpl/index.html index 9827646..15856f7 100644 --- a/assets/tmpl/index.html +++ b/assets/tmpl/index.html @@ -34,13 +34,15 @@ /api/plain/mentions /api/plain/tweets /api/plain/tags</code></pre> + <p>All queries accept <code>?page=N</code> as a parameter, returning sets of 20 results. + This may be omitted for the first page of results.</p> <p>Query by user:</p> <pre><code>$ curl '{{.URL}}/api/plain/users?q=foo' foo https://example.com/twtxt.txt 2019-05-09T08:42:23.000Z foobar https://example2.com/twtxt.txt 2019-03-14T19:23:00.000Z foo_barrington https://example3.com/twtxt.txt 2019-05-01T15:59:39.000Z</code></pre> <p>Query by tweet content:</p> - <pre><code>$ curl '{{.URL}}/api/plain/tweets?q=getwtxt' + <pre><code>$ curl '{{.URL}}/api/plain/tweets?q=getwtxt&page=2' foo_barrington https://example3.com/twtxt.txt 2019-04-30T06:00:09.000Z I just built getwtxt, time to set it up!</code></pre> <p>Query by mention:</p> <pre><code>$ curl '{{.URL}}/api/plain/mentions?url=https://foobarrington.co.uk/twtxt.txt' @@ -48,8 +50,8 @@ foo https://example.com/twtxt.txt 2019-02-26T11:06:44.000Z @<foo_bar <p>Query by tag:</p> <pre><code>$ curl '{{.URL}}/api/plain/tags/programming' foo https://example.com/twtxt.txt 2019-03-01T09:31:02.000Z I love #programming!</code></pre> - <p>Get latest 20 tweets:</p> - <pre><code>$ curl '{{.URL}}/api/plain/tweets' + <p>Get 20 tweets:</p> + <pre><code>$ curl '{{.URL}}/api/plain/tweets?page=2' foobar https://example2.com/twtxt.txt 2019-05-13T12:46:20.000Z It's been a busy day at work! ...</code></pre> <p>Get all users:</p> diff --git a/go.mod b/go.mod index ec32001..4bb494a 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.11 require ( github.com/BurntSushi/toml v0.3.1 // indirect github.com/fsnotify/fsnotify v1.4.7 - github.com/getwtxt/registry v0.0.0-20190531085138-9a0f6b50332c + github.com/getwtxt/registry v0.2.2 github.com/gorilla/handlers v1.4.0 github.com/gorilla/mux v1.7.1 github.com/spf13/pflag v1.0.3 diff --git a/go.sum b/go.sum index d9c4404..ac90169 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/getwtxt/registry v0.0.0-20190531085138-9a0f6b50332c h1:qALsi03YJHPb5CXBxYyuGREh0v8OLravaNMwkkHu944= -github.com/getwtxt/registry v0.0.0-20190531085138-9a0f6b50332c/go.mod h1:BGSIALOFqIRj+ACLB8etWGUOgFAKN8oFDpCsw6YOdYQ= +github.com/getwtxt/registry v0.2.2 h1:vaDgmyojGcgzBVArFbO4RwpF3zLTIGHOALmHsYZpjLQ= +github.com/getwtxt/registry v0.2.2/go.mod h1:BGSIALOFqIRj+ACLB8etWGUOgFAKN8oFDpCsw6YOdYQ= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= diff --git a/handlers.go b/handlers.go index edfb333..63b8c26 100644 --- a/handlers.go +++ b/handlers.go @@ -4,8 +4,10 @@ import ( "crypto/sha256" "fmt" "net/http" + "strconv" "strings" + "github.com/getwtxt/registry" "github.com/gorilla/mux" ) @@ -42,6 +44,30 @@ func apiFormatHandler(w http.ResponseWriter, r *http.Request) { indexHandler(w, r) } +func apiAllTweetsHandler(w http.ResponseWriter, r *http.Request) { + out, err := twtxtCache.QueryAllStatuses() + if err != nil { + log500(w, r, err) + } + + data := parseQueryOut(out) + if err != nil { + data = []byte("") + } + + etag := fmt.Sprintf("%x", sha256.Sum256(data)) + w.Header().Set("ETag", etag) + w.Header().Set("Content-Type", txtutf8) + + _, err = w.Write(data) + if err != nil { + log500(w, r, err) + return + } + + log200(r) +} + // handles "/api/plain/(users|mentions|tweets)" func apiEndpointHandler(w http.ResponseWriter, r *http.Request) { @@ -61,18 +87,30 @@ func apiEndpointHandler(w http.ResponseWriter, r *http.Request) { return } + page := 1 + pageVal := r.FormValue("page") + if pageVal != "" { + page, err = strconv.Atoi(pageVal) + if err != nil || page == 0 { + page = 1 + } + } + // if there's no query, return everything in // registry for a given endpoint var out []string switch r.URL.Path { case "/api/plain/users": out, err = twtxtCache.QueryUser("") + out = registry.ReduceToPage(page, out) case "/api/plain/mentions": out, err = twtxtCache.QueryInStatus("@<") + out = registry.ReduceToPage(page, out) default: out, err = twtxtCache.QueryAllStatuses() + out = registry.ReduceToPage(page, out) } data := parseQueryOut(out) @@ -107,6 +145,7 @@ func apiTagsBaseHandler(w http.ResponseWriter, r *http.Request) { return } + out = registry.ReduceToPage(1, out) data := parseQueryOut(out) etag := fmt.Sprintf("%x", sha256.Sum256(data)) @@ -151,6 +190,7 @@ func apiTagsHandler(w http.ResponseWriter, r *http.Request) { out = append(out, out3...) out = uniq(out) + out = registry.ReduceToPage(1, out) data := parseQueryOut(out) etag := fmt.Sprintf("%x", sha256.Sum256(data)) diff --git a/main.go b/main.go index 6f50f68..a3f0133 100644 --- a/main.go +++ b/main.go @@ -20,34 +20,38 @@ func main() { api := index.PathPrefix("/api").Subrouter() index.Path("/"). - Methods("GET"). + Methods("GET", "HEAD"). HandlerFunc(indexHandler) index.Path("/css"). - Methods("GET"). + Methods("GET", "HEAD"). HandlerFunc(cssHandler) index.Path("/api"). - Methods("GET"). + Methods("GET", "HEAD"). HandlerFunc(apiBaseHandler) // twtxt will add support for other formats later. // Maybe json? Making this future-proof. api.Path("/{format:(?:plain)}"). - Methods("GET"). + Methods("GET", "HEAD"). HandlerFunc(apiFormatHandler) + // Non-standard API call to list *all* tweets + // in a single request. + api.Path("/{format:(?:plain)}/tweets/all"). + Methods("GET", "HEAD"). + HandlerFunc(apiAllTweetsHandler) + // Specifying the endpoint with and without query information. // Will return 404 on empty queries otherwise. api.Path("/{format:(?:plain)}/{endpoint:(?:mentions|users|tweets)}"). - Methods("GET"). + Methods("GET", "HEAD"). HandlerFunc(apiEndpointHandler) - // Using stdlib net/url to validate the input URLs rather than regex. - // Validating a URL with regex is unwieldy api.Path("/{format:(?:plain)}/{endpoint:(?:mentions|users|tweets)}"). - Queries("url", "{url}", "q", "{query}"). - Methods("GET"). + Queries("url", "{url}", "q", "{query}", "page", "{[0-9]+}"). + Methods("GET", "HEAD"). HandlerFunc(apiEndpointHandler) // This is for submitting new users. Both query variables must exist @@ -64,7 +68,7 @@ func main() { Methods("POST"). HandlerFunc(apiEndpointPOSTHandler) - // This is for submitting new users incorrectly + // This is also for submitting new users incorrectly // and letting the requester know about their error. api.Path("/{format:(?:plain)}/{endpoint:users}"). Queries("nickname", "{nickname:[a-zA-Z0-9_-]+}"). @@ -73,12 +77,24 @@ func main() { // Show all observed tags api.Path("/{format:(?:plain)}/tags"). - Methods("GET"). + Methods("GET", "HEAD"). + HandlerFunc(apiTagsBaseHandler) + + // Show Nth page of all observed tags + api.Path("/{format:(?:plain)}/tags"). + Queries("page", "{[0-9]+}"). + Methods("GET", "HEAD"). HandlerFunc(apiTagsBaseHandler) // Requests statuses with a specific tag api.Path("/{format:(?:plain)}/tags/{tags:[a-zA-Z0-9_-]+}"). - Methods("GET"). + Methods("GET", "HEAD"). + HandlerFunc(apiTagsHandler) + + // Requests Nth page of statuses with a specific tag + api.Path("/{format:(?:plain)}/tags/{tags:[a-zA-Z0-9_-]+}"). + Queries("page", "{[0-9]+}"). + Methods("GET", "HEAD"). HandlerFunc(apiTagsHandler) confObj.Mu.RLock() diff --git a/query.go b/query.go index 554a53c..dbfb6ee 100644 --- a/query.go +++ b/query.go @@ -5,8 +5,10 @@ import ( "fmt" "log" "net/http" + "strconv" "strings" + "github.com/getwtxt/registry" "github.com/gorilla/mux" ) @@ -55,9 +57,16 @@ func uniq(str []string) []string { func apiEndpointQuery(w http.ResponseWriter, r *http.Request) error { query := r.FormValue("q") urls := r.FormValue("url") + pageVal := r.FormValue("page") var out []string var err error + pageVal = strings.TrimSpace(pageVal) + page, err := strconv.Atoi(pageVal) + if err != nil { + log.Printf("%v\n", err.Error()) + } + vars := mux.Vars(r) endpoint := vars["endpoint"] @@ -78,8 +87,9 @@ func apiEndpointQuery(w http.ResponseWriter, r *http.Request) error { apiErrCheck(err, r) } - out = append(out, out2...) - out = uniq(out) + if query != "" && urls != "" { + out = joinQueryOuts(out2) + } case "mentions": if urls == "" { @@ -90,26 +100,13 @@ func apiEndpointQuery(w http.ResponseWriter, r *http.Request) error { apiErrCheck(err, r) case "tweets": - query = strings.ToLower(query) - out, err = twtxtCache.QueryInStatus(query) - apiErrCheck(err, r) - - query = strings.Title(query) - out2, err := twtxtCache.QueryInStatus(query) - apiErrCheck(err, r) - - query = strings.ToUpper(query) - out3, err := twtxtCache.QueryInStatus(query) - apiErrCheck(err, r) - - out = append(out, out2...) - out = append(out, out3...) - out = uniq(out) + out = compositeStatusQuery(query, r) default: return fmt.Errorf("endpoint query, no cases match") } + out = registry.ReduceToPage(page, out) data := parseQueryOut(out) etag := fmt.Sprintf("%x", sha256.Sum256(data)) @@ -120,3 +117,30 @@ func apiEndpointQuery(w http.ResponseWriter, r *http.Request) error { return err } + +func joinQueryOuts(data ...[]string) []string { + single := []string{} + for _, e := range data { + single = append(single, e...) + } + single = uniq(single) + + return single +} + +func compositeStatusQuery(query string, r *http.Request) []string { + query = strings.ToLower(query) + out, err := twtxtCache.QueryInStatus(query) + apiErrCheck(err, r) + + query = strings.Title(query) + out2, err := twtxtCache.QueryInStatus(query) + apiErrCheck(err, r) + + query = strings.ToUpper(query) + out3, err := twtxtCache.QueryInStatus(query) + apiErrCheck(err, r) + + final := joinQueryOuts(out, out2, out3) + return final +} |