path: root/registry/fetch.go

/*
Copyright (c) 2019 Ben Morrison (gbmor)

This file is part of Registry.

Registry is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

Registry is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with Registry.  If not, see <https://www.gnu.org/licenses/>.
*/

package registry // import "git.sr.ht/~gbmor/getwtxt/registry"

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"sync"
	"time"
)

const rfc3339WithoutSeconds = "2006-01-02T15:04Z07:00"

// GetTwtxt fetches the raw twtxt file data from the user's
// provided URL, after validating the URL. If the returned
// boolean value is false, the fetched URL is a single user's
// twtxt file. If true, the fetched URL is the output of
// another registry's /api/plain/tweets. The output of
// GetTwtxt should be passed to either ParseUserTwtxt or
// ParseRegistryTwtxt, respectively.
// Generally, the *http.Client inside a given Registry instance should
// be passed to GetTwtxt. If the *http.Client passed is nil,
// Registry will use a preconstructed client with a
// timeout of 10s and all other values set to default.
func GetTwtxt(urlKey string, client *http.Client) ([]byte, bool, error) {
	if !strings.HasPrefix(urlKey, "http://") && !strings.HasPrefix(urlKey, "https://") {
		return nil, false, fmt.Errorf("invalid URL: %v", urlKey)
	}

	res, err := doReq(urlKey, "GET", "", client)
	if err != nil {
		return nil, false, err
	}
	defer res.Body.Close()

	var textPlain bool
	for _, v := range res.Header["Content-Type"] {
		if strings.Contains(v, "text/plain") {
			textPlain = true
			break
		}
	}
	if !textPlain {
		return nil, false, fmt.Errorf("received non-text/plain response body from %v", urlKey)
	}

	if res.StatusCode != http.StatusOK {
		return nil, false, fmt.Errorf("didn't get 200 from remote server, received %v: %v", res.StatusCode, urlKey)
	}

	twtxt, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, false, fmt.Errorf("error reading response body from %v: %v", urlKey, err)
	}

	// Signal that we're adding another twtxt registry as a "user"
	if strings.HasSuffix(urlKey, "/api/plain/tweets") || strings.HasSuffix(urlKey, "/api/plain/tweets/all") {
		return twtxt, true, nil
	}

	return twtxt, false, nil
}

// DiffTwtxt issues a HEAD request on the user's
// remote twtxt data. It then checks the Content-Length
// header. If it's different from the stored result of
// the previous Content-Length header, update the stored
// value for a given user and return true.
// Otherwise, return false. In some error conditions,
// such as the user not being in the registry, it returns true.
// In other error conditions considered "unrecoverable,"
// such as the supplied URL being invalid, it returns false.
func (registry *Registry) DiffTwtxt(urlKey string) (bool, error) {
	if !strings.HasPrefix(urlKey, "http://") && !strings.HasPrefix(urlKey, "https://") {
		return false, fmt.Errorf("invalid URL: %v", urlKey)
	}

	registry.Mu.Lock()
	user, ok := registry.Users[urlKey]
	if !ok {
		return true, fmt.Errorf("user not in registry")
	}

	user.Mu.Lock()

	defer func() {
		registry.Users[urlKey] = user
		user.Mu.Unlock()
		registry.Mu.Unlock()
	}()

	res, err := doReq(urlKey, "HEAD", user.LastModified, registry.HTTPClient)
	if err != nil {
		return false, err
	}

	switch res.StatusCode {
	case http.StatusOK:
		for _, e := range res.Header["Last-Modified"] {
			if e != "" {
				user.LastModified = e
				break
			}
		}
		return true, nil

	case http.StatusNotModified:
		return false, nil
	}

	return false, nil
}

// internal function. boilerplate for http requests.
func doReq(urlKey, method, modTime string, client *http.Client) (*http.Response, error) {
	if client == nil {
		client = &http.Client{
			Transport:     nil,
			CheckRedirect: nil,
			Jar:           nil,
			Timeout:       10 * time.Second,
		}
	}

	var b []byte
	buf := bytes.NewBuffer(b)
	req, err := http.NewRequest(method, urlKey, buf)
	if err != nil {
		return nil, err
	}

	if modTime != "" {
		req.Header.Set("If-Modified-Since", modTime)
	}

	res, err := client.Do(req)
	if err != nil {
		return nil, fmt.Errorf("couldn't %v %v: %v", method, urlKey, err)
	}

	return res, nil
}

// ParseUserTwtxt takes a fetched twtxt file in the form of
// a slice of bytes, parses it, and returns it as a
// TimeMap. The output may then be passed to Index.AddUser()
func ParseUserTwtxt(twtxt []byte, nickname, urlKey string) (TimeMap, error) {
	var erz []byte
	if len(twtxt) == 0 {
		return nil, fmt.Errorf("no data to parse in twtxt file")
	}

	reader := bytes.NewReader(twtxt)
	scanner := bufio.NewScanner(reader)
	timemap := NewTimeMap()

	for scanner.Scan() {
		nopadding := strings.TrimSpace(scanner.Text())
		if strings.HasPrefix(nopadding, "#") || nopadding == "" {
			continue
		}

		columns := strings.Split(nopadding, "\t")
		if len(columns) != 2 {
			return nil, fmt.Errorf("improperly formatted data in twtxt file")
		}

		noSeconds := false
		count := strings.Count(columns[0], ":")

		if strings.Contains(columns[0], "Z") {
			split := strings.Split(columns[0], "Z")
			if len(split[1]) > 0 && count == 2 {
				noSeconds = true
			}
		} else if count == 2 {
			noSeconds = true
		}

		var thetime time.Time
		var err error
		if strings.Contains(columns[0], ".") {
			thetime, err = time.Parse(time.RFC3339Nano, columns[0])
		} else if noSeconds {
			// this means they're probably not including seconds into the datetime
			thetime, err = time.Parse(rfc3339WithoutSeconds, columns[0])
		} else {
			thetime, err = time.Parse(time.RFC3339, columns[0])
		}
		if err != nil {
			erz = append(erz, []byte(fmt.Sprintf("unable to retrieve date: %v\n", err))...)
		}

		timemap[thetime] = nickname + "\t" + urlKey + "\t" + nopadding
	}

	if len(erz) == 0 {
		return timemap, nil
	}
	return timemap, fmt.Errorf("%v", string(erz))
}

// ParseRegistryTwtxt takes output from a remote registry and outputs
// the accessible user data via a slice of Users.
func ParseRegistryTwtxt(twtxt []byte) ([]*User, error) {
	var erz []byte
	if len(twtxt) == 0 {
		return nil, fmt.Errorf("received no data")
	}

	reader := bytes.NewReader(twtxt)
	scanner := bufio.NewScanner(reader)
	userdata := []*User{}

	for scanner.Scan() {

		nopadding := strings.TrimSpace(scanner.Text())

		if strings.HasPrefix(nopadding, "#") || nopadding == "" {
			continue
		}

		columns := strings.Split(nopadding, "\t")
		if len(columns) != 4 {
			return nil, fmt.Errorf("improperly formatted data")
		}

		thetime, err := time.Parse(time.RFC3339, columns[2])
		if err != nil {
			erz = append(erz, []byte(fmt.Sprintf("%v\n", err))...)
			continue
		}

		parsednickname := columns[0]
		dataIndex := 0
		parsedurl := columns[1]
		inIndex := false

		for i, e := range userdata {
			if e.Nick == parsednickname || e.URL == parsedurl {
				dataIndex = i
				inIndex = true
				break
			}
		}

		if inIndex {
			tmp := userdata[dataIndex]
			tmp.Status[thetime] = nopadding
			userdata[dataIndex] = tmp
		} else {
			timeNowRFC := time.Now().Format(time.RFC3339)
			if err != nil {
				erz = append(erz, []byte(fmt.Sprintf("%v\n", err))...)
			}

			tmp := &User{
				Mu:   sync.RWMutex{},
				Nick: parsednickname,
				URL:  parsedurl,
				Date: timeNowRFC,
				Status: TimeMap{
					thetime: nopadding,
				},
			}

			userdata = append(userdata, tmp)
		}
	}

	return userdata, fmt.Errorf("%v", erz)
}