Merge branch 'dev' into 'master' for v0.2.0

6 files changed, 117 insertions, 35 deletions

diff --git a/assets/tmpl/index.html b/assets/tmpl/index.html
index 9827646..15856f7 100644
--- a/assets/tmpl/index.html
+++ b/assets/tmpl/index.html
@@ -34,13 +34,15 @@
 /api/plain/mentions
 /api/plain/tweets
 /api/plain/tags</code></pre>
+      <p>All queries accept <code>?page=N</code> as a parameter, returning sets of 20 results. 
+      This may be omitted for the first page of results.</p>
       <p>Query by user:</p>
         <pre><code>$ curl '{{.URL}}/api/plain/users?q=foo'
 foo               https://example.com/twtxt.txt     2019-05-09T08:42:23.000Z
 foobar            https://example2.com/twtxt.txt    2019-03-14T19:23:00.000Z
 foo_barrington    https://example3.com/twtxt.txt    2019-05-01T15:59:39.000Z</code></pre>
       <p>Query by tweet content:</p>
-        <pre><code>$ curl '{{.URL}}/api/plain/tweets?q=getwtxt'
+        <pre><code>$ curl '{{.URL}}/api/plain/tweets?q=getwtxt&amp;page=2'
 foo_barrington    https://example3.com/twtxt.txt    2019-04-30T06:00:09.000Z    I just built getwtxt, time to set it up!</code></pre>
       <p>Query by mention:</p>
         <pre><code>$ curl '{{.URL}}/api/plain/mentions?url=https://foobarrington.co.uk/twtxt.txt'
@@ -48,8 +50,8 @@ foo    https://example.com/twtxt.txt    2019-02-26T11:06:44.000Z    @&lt;foo_bar
       <p>Query by tag:</p>
         <pre><code>$ curl '{{.URL}}/api/plain/tags/programming'
 foo    https://example.com/twtxt.txt    2019-03-01T09:31:02.000Z    I love #programming!</code></pre>
-      <p>Get latest 20 tweets:</p>
-        <pre><code>$ curl '{{.URL}}/api/plain/tweets'
+      <p>Get 20 tweets:</p>
+        <pre><code>$ curl '{{.URL}}/api/plain/tweets?page=2'
 foobar    https://example2.com/twtxt.txt    2019-05-13T12:46:20.000Z    It's been a busy day at work!
 ...</code></pre>
       <p>Get all users:</p>
diff --git a/go.mod b/go.mod
index ec32001..4bb494a 100644
--- a/go.mod
+++ b/go.mod
@@ -5,7 +5,7 @@ go 1.11
 require (
 	github.com/BurntSushi/toml v0.3.1 // indirect
 	github.com/fsnotify/fsnotify v1.4.7
-	github.com/getwtxt/registry v0.0.0-20190531085138-9a0f6b50332c
+	github.com/getwtxt/registry v0.2.2
 	github.com/gorilla/handlers v1.4.0
 	github.com/gorilla/mux v1.7.1
 	github.com/spf13/pflag v1.0.3
diff --git a/go.sum b/go.sum
index d9c4404..ac90169 100644
--- a/go.sum
+++ b/go.sum
@@ -8,8 +8,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/getwtxt/registry v0.0.0-20190531085138-9a0f6b50332c h1:qALsi03YJHPb5CXBxYyuGREh0v8OLravaNMwkkHu944=
-github.com/getwtxt/registry v0.0.0-20190531085138-9a0f6b50332c/go.mod h1:BGSIALOFqIRj+ACLB8etWGUOgFAKN8oFDpCsw6YOdYQ=
+github.com/getwtxt/registry v0.2.2 h1:vaDgmyojGcgzBVArFbO4RwpF3zLTIGHOALmHsYZpjLQ=
+github.com/getwtxt/registry v0.2.2/go.mod h1:BGSIALOFqIRj+ACLB8etWGUOgFAKN8oFDpCsw6YOdYQ=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w=
diff --git a/handlers.go b/handlers.go
index edfb333..63b8c26 100644
--- a/handlers.go
+++ b/handlers.go
@@ -4,8 +4,10 @@ import (
 	"crypto/sha256"
 	"fmt"
 	"net/http"
+	"strconv"
 	"strings"
 
+	"github.com/getwtxt/registry"
 	"github.com/gorilla/mux"
 )
 
@@ -42,6 +44,30 @@ func apiFormatHandler(w http.ResponseWriter, r *http.Request) {
 	indexHandler(w, r)
 }
 
+func apiAllTweetsHandler(w http.ResponseWriter, r *http.Request) {
+	out, err := twtxtCache.QueryAllStatuses()
+	if err != nil {
+		log500(w, r, err)
+	}
+
+	data := parseQueryOut(out)
+	if err != nil {
+		data = []byte("")
+	}
+
+	etag := fmt.Sprintf("%x", sha256.Sum256(data))
+	w.Header().Set("ETag", etag)
+	w.Header().Set("Content-Type", txtutf8)
+
+	_, err = w.Write(data)
+	if err != nil {
+		log500(w, r, err)
+		return
+	}
+
+	log200(r)
+}
+
 // handles "/api/plain/(users|mentions|tweets)"
 func apiEndpointHandler(w http.ResponseWriter, r *http.Request) {
 
@@ -61,18 +87,30 @@ func apiEndpointHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	page := 1
+	pageVal := r.FormValue("page")
+	if pageVal != "" {
+		page, err = strconv.Atoi(pageVal)
+		if err != nil || page == 0 {
+			page = 1
+		}
+	}
+
 	// if there's no query, return everything in
 	// registry for a given endpoint
 	var out []string
 	switch r.URL.Path {
 	case "/api/plain/users":
 		out, err = twtxtCache.QueryUser("")
+		out = registry.ReduceToPage(page, out)
 
 	case "/api/plain/mentions":
 		out, err = twtxtCache.QueryInStatus("@<")
+		out = registry.ReduceToPage(page, out)
 
 	default:
 		out, err = twtxtCache.QueryAllStatuses()
+		out = registry.ReduceToPage(page, out)
 	}
 
 	data := parseQueryOut(out)
@@ -107,6 +145,7 @@ func apiTagsBaseHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	out = registry.ReduceToPage(1, out)
 	data := parseQueryOut(out)
 
 	etag := fmt.Sprintf("%x", sha256.Sum256(data))
@@ -151,6 +190,7 @@ func apiTagsHandler(w http.ResponseWriter, r *http.Request) {
 	out = append(out, out3...)
 	out = uniq(out)
 
+	out = registry.ReduceToPage(1, out)
 	data := parseQueryOut(out)
 
 	etag := fmt.Sprintf("%x", sha256.Sum256(data))
diff --git a/main.go b/main.go
index 6f50f68..a3f0133 100644
--- a/main.go
+++ b/main.go
@@ -20,34 +20,38 @@ func main() {
 	api := index.PathPrefix("/api").Subrouter()
 
 	index.Path("/").
-		Methods("GET").
+		Methods("GET", "HEAD").
 		HandlerFunc(indexHandler)
 
 	index.Path("/css").
-		Methods("GET").
+		Methods("GET", "HEAD").
 		HandlerFunc(cssHandler)
 
 	index.Path("/api").
-		Methods("GET").
+		Methods("GET", "HEAD").
 		HandlerFunc(apiBaseHandler)
 
 	// twtxt will add support for other formats later.
 	// Maybe json? Making this future-proof.
 	api.Path("/{format:(?:plain)}").
-		Methods("GET").
+		Methods("GET", "HEAD").
 		HandlerFunc(apiFormatHandler)
 
+	// Non-standard API call to list *all* tweets
+	// in a single request.
+	api.Path("/{format:(?:plain)}/tweets/all").
+		Methods("GET", "HEAD").
+		HandlerFunc(apiAllTweetsHandler)
+
 	// Specifying the endpoint with and without query information.
 	// Will return 404 on empty queries otherwise.
 	api.Path("/{format:(?:plain)}/{endpoint:(?:mentions|users|tweets)}").
-		Methods("GET").
+		Methods("GET", "HEAD").
 		HandlerFunc(apiEndpointHandler)
 
-	// Using stdlib net/url to validate the input URLs rather than regex.
-	// Validating a URL with regex is unwieldy
 	api.Path("/{format:(?:plain)}/{endpoint:(?:mentions|users|tweets)}").
-		Queries("url", "{url}", "q", "{query}").
-		Methods("GET").
+		Queries("url", "{url}", "q", "{query}", "page", "{[0-9]+}").
+		Methods("GET", "HEAD").
 		HandlerFunc(apiEndpointHandler)
 
 	// This is for submitting new users. Both query variables must exist
@@ -64,7 +68,7 @@ func main() {
 		Methods("POST").
 		HandlerFunc(apiEndpointPOSTHandler)
 
-	// This is for submitting new users incorrectly
+	// This is also for submitting new users incorrectly
 	// and letting the requester know about their error.
 	api.Path("/{format:(?:plain)}/{endpoint:users}").
 		Queries("nickname", "{nickname:[a-zA-Z0-9_-]+}").
@@ -73,12 +77,24 @@ func main() {
 
 	// Show all observed tags
 	api.Path("/{format:(?:plain)}/tags").
-		Methods("GET").
+		Methods("GET", "HEAD").
+		HandlerFunc(apiTagsBaseHandler)
+
+	// Show Nth page of all observed tags
+	api.Path("/{format:(?:plain)}/tags").
+		Queries("page", "{[0-9]+}").
+		Methods("GET", "HEAD").
 		HandlerFunc(apiTagsBaseHandler)
 
 	// Requests statuses with a specific tag
 	api.Path("/{format:(?:plain)}/tags/{tags:[a-zA-Z0-9_-]+}").
-		Methods("GET").
+		Methods("GET", "HEAD").
+		HandlerFunc(apiTagsHandler)
+
+	// Requests Nth page of statuses with a specific tag
+	api.Path("/{format:(?:plain)}/tags/{tags:[a-zA-Z0-9_-]+}").
+		Queries("page", "{[0-9]+}").
+		Methods("GET", "HEAD").
 		HandlerFunc(apiTagsHandler)
 
 	confObj.Mu.RLock()
diff --git a/query.go b/query.go
index 554a53c..dbfb6ee 100644
--- a/query.go
+++ b/query.go
@@ -5,8 +5,10 @@ import (
 	"fmt"
 	"log"
 	"net/http"
+	"strconv"
 	"strings"
 
+	"github.com/getwtxt/registry"
 	"github.com/gorilla/mux"
 )
 
@@ -55,9 +57,16 @@ func uniq(str []string) []string {
 func apiEndpointQuery(w http.ResponseWriter, r *http.Request) error {
 	query := r.FormValue("q")
 	urls := r.FormValue("url")
+	pageVal := r.FormValue("page")
 	var out []string
 	var err error
 
+	pageVal = strings.TrimSpace(pageVal)
+	page, err := strconv.Atoi(pageVal)
+	if err != nil {
+		log.Printf("%v\n", err.Error())
+	}
+
 	vars := mux.Vars(r)
 	endpoint := vars["endpoint"]
 
@@ -78,8 +87,9 @@ func apiEndpointQuery(w http.ResponseWriter, r *http.Request) error {
 			apiErrCheck(err, r)
 		}
 
-		out = append(out, out2...)
-		out = uniq(out)
+		if query != "" && urls != "" {
+			out = joinQueryOuts(out2)
+		}
 
 	case "mentions":
 		if urls == "" {
@@ -90,26 +100,13 @@ func apiEndpointQuery(w http.ResponseWriter, r *http.Request) error {
 		apiErrCheck(err, r)
 
 	case "tweets":
-		query = strings.ToLower(query)
-		out, err = twtxtCache.QueryInStatus(query)
-		apiErrCheck(err, r)
-
-		query = strings.Title(query)
-		out2, err := twtxtCache.QueryInStatus(query)
-		apiErrCheck(err, r)
-
-		query = strings.ToUpper(query)
-		out3, err := twtxtCache.QueryInStatus(query)
-		apiErrCheck(err, r)
-
-		out = append(out, out2...)
-		out = append(out, out3...)
-		out = uniq(out)
+		out = compositeStatusQuery(query, r)
 
 	default:
 		return fmt.Errorf("endpoint query, no cases match")
 	}
 
+	out = registry.ReduceToPage(page, out)
 	data := parseQueryOut(out)
 
 	etag := fmt.Sprintf("%x", sha256.Sum256(data))
@@ -120,3 +117,30 @@ func apiEndpointQuery(w http.ResponseWriter, r *http.Request) error {
 
 	return err
 }
+
+func joinQueryOuts(data ...[]string) []string {
+	single := []string{}
+	for _, e := range data {
+		single = append(single, e...)
+	}
+	single = uniq(single)
+
+	return single
+}
+
+func compositeStatusQuery(query string, r *http.Request) []string {
+	query = strings.ToLower(query)
+	out, err := twtxtCache.QueryInStatus(query)
+	apiErrCheck(err, r)
+
+	query = strings.Title(query)
+	out2, err := twtxtCache.QueryInStatus(query)
+	apiErrCheck(err, r)
+
+	query = strings.ToUpper(query)
+	out3, err := twtxtCache.QueryInStatus(query)
+	apiErrCheck(err, r)
+
+	final := joinQueryOuts(out, out2, out3)
+	return final
+}