moved bulk of code to its own package to clean up source tree

1 files changed, 600 insertions, 0 deletions

diff --git a/svc/init.go b/svc/init.go
new file mode 100644
index 0000000..18fe0b6
--- /dev/null
+++ b/svc/init.go
@@ -0,0 +1,600 @@
+package svc // import "github.com/getwtxt/getwtxt/svc"
+
+import (
+	"database/sql"
+	"fmt"
+	"html/template"
+	"log"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/fsnotify/fsnotify"
+	"github.com/getwtxt/registry"
+	"github.com/spf13/pflag"
+	"github.com/spf13/viper"
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+const getwtxt = "0.2.2"
+
+var (
+	flagVersion  *bool   = pflag.BoolP("version", "v", false, "Display version information, then exit.")
+	flagHelp     *bool   = pflag.BoolP("help", "h", false, "Display the quick-help screen.")
+	flagMan      *bool   = pflag.BoolP("manual", "m", false, "Display the configuration manual.")
+	flagConfFile *string = pflag.StringP("config", "c", "", "The name/path of the configuration file you wish to use.")
+	flagAssets   *string = pflag.StringP("assets", "a", "", "The location of the getwtxt assets directory")
+	flagDBPath   *string = pflag.StringP("db", "d", "", "Path to the getwtxt database")
+	flagDBType   *string = pflag.StringP("dbtype", "t", "", "Type of database being used")
+)
+
+var confObj = &Configuration{}
+
+// signals to close the log file
+var closeLog = make(chan bool, 1)
+
+// used to transmit database pointer after
+// initialization
+var dbChan = make(chan dbase, 1)
+
+var tmpls *template.Template
+
+var twtxtCache = registry.NewIndex()
+
+var remoteRegistries = &RemoteRegistries{}
+
+var staticCache = &struct {
+	index    []byte
+	indexMod time.Time
+	css      []byte
+	cssMod   time.Time
+}{
+	index:    nil,
+	indexMod: time.Time{},
+	css:      nil,
+	cssMod:   time.Time{},
+}
+
+func init() {
+	checkFlags()
+	titleScreen()
+	initConfig()
+	initLogging()
+	tmpls = initTemplates()
+	initDatabase()
+	go cacheAndPush()
+	watchForInterrupt()
+}
+
+func checkFlags() {
+	pflag.Parse()
+	if *flagVersion {
+		titleScreen()
+		os.Exit(0)
+	}
+	if *flagHelp {
+		titleScreen()
+		helpScreen()
+		os.Exit(0)
+	}
+	if *flagMan {
+		titleScreen()
+		helpScreen()
+		manualScreen()
+		os.Exit(0)
+	}
+}
+
+func initConfig() {
+
+	if *flagConfFile == "" {
+		viper.SetConfigName("getwtxt")
+		viper.SetConfigType("yml")
+		viper.AddConfigPath(".")
+		viper.AddConfigPath("/usr/local/getwtxt")
+		viper.AddConfigPath("/etc")
+		viper.AddConfigPath("/usr/local/etc")
+
+	} else {
+		path, file := filepath.Split(*flagConfFile)
+		if path == "" {
+			path = "."
+		}
+		if file == "" {
+			file = *flagConfFile
+		}
+		filename := strings.Split(file, ".")
+		viper.SetConfigName(filename[0])
+		viper.SetConfigType(filename[1])
+		viper.AddConfigPath(path)
+	}
+
+	log.Printf("Loading configuration ...\n")
+	if err := viper.ReadInConfig(); err != nil {
+		log.Printf("%v\n", err.Error())
+		log.Printf("Using defaults ...\n")
+	} else {
+		viper.WatchConfig()
+		viper.OnConfigChange(func(e fsnotify.Event) {
+			log.Printf("Config file change detected. Reloading...\n")
+			rebindConfig()
+		})
+	}
+
+	viper.SetDefault("ListenPort", 9001)
+	viper.SetDefault("LogFile", "getwtxt.log")
+	viper.SetDefault("DatabasePath", "getwtxt.db")
+	viper.SetDefault("AssetsDirectory", "assets")
+	viper.SetDefault("DatabaseType", "leveldb")
+	viper.SetDefault("StdoutLogging", false)
+	viper.SetDefault("ReCacheInterval", "1h")
+	viper.SetDefault("DatabasePushInterval", "5m")
+
+	viper.SetDefault("Instance.SiteName", "getwtxt")
+	viper.SetDefault("Instance.OwnerName", "Anonymous Microblogger")
+	viper.SetDefault("Instance.Email", "nobody@knows")
+	viper.SetDefault("Instance.URL", "https://twtxt.example.com")
+	viper.SetDefault("Instance.Description", "A fast, resilient twtxt registry server written in Go!")
+
+	confObj.Mu.Lock()
+
+	confObj.Port = viper.GetInt("ListenPort")
+	confObj.LogFile = viper.GetString("LogFile")
+
+	if *flagDBType == "" {
+		confObj.DBType = strings.ToLower(viper.GetString("DatabaseType"))
+	} else {
+		confObj.DBType = *flagDBType
+	}
+
+	if *flagDBPath == "" {
+		confObj.DBPath = viper.GetString("DatabasePath")
+	} else {
+		confObj.DBPath = *flagDBPath
+	}
+	log.Printf("Using %v database: %v\n", confObj.DBType, confObj.DBPath)
+
+	if *flagAssets == "" {
+		confObj.AssetsDir = viper.GetString("AssetsDirectory")
+	} else {
+		confObj.AssetsDir = *flagAssets
+	}
+
+	confObj.StdoutLogging = viper.GetBool("StdoutLogging")
+	if confObj.StdoutLogging {
+		log.Printf("Logging to stdout\n")
+	} else {
+		log.Printf("Logging to %v\n", confObj.LogFile)
+	}
+
+	confObj.CacheInterval = viper.GetDuration("StatusFetchInterval")
+	log.Printf("User status fetch interval: %v\n", confObj.CacheInterval)
+
+	confObj.DBInterval = viper.GetDuration("DatabasePushInterval")
+	log.Printf("Database push interval: %v\n", confObj.DBInterval)
+
+	confObj.LastCache = time.Now()
+	confObj.LastPush = time.Now()
+	confObj.Version = getwtxt
+
+	confObj.Instance.Vers = getwtxt
+	confObj.Instance.Name = viper.GetString("Instance.SiteName")
+	confObj.Instance.URL = viper.GetString("Instance.URL")
+	confObj.Instance.Owner = viper.GetString("Instance.OwnerName")
+	confObj.Instance.Mail = viper.GetString("Instance.Email")
+	confObj.Instance.Desc = viper.GetString("Instance.Description")
+
+	confObj.Mu.Unlock()
+
+}
+
+func initLogging() {
+
+	confObj.Mu.RLock()
+
+	if confObj.StdoutLogging {
+		log.SetOutput(os.Stdout)
+
+	} else {
+
+		logfile, err := os.OpenFile(confObj.LogFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
+		if err != nil {
+			log.Printf("Could not open log file: %v\n", err.Error())
+		}
+
+		// Listen for the signal to close the log file
+		// in a separate thread. Passing it as an argument
+		// to prevent race conditions when the config is
+		// reloaded.
+		go func(logfile *os.File) {
+
+			<-closeLog
+			log.Printf("Closing log file ...\n")
+
+			err = logfile.Close()
+			if err != nil {
+				log.Printf("Couldn't close log file: %v\n", err.Error())
+			}
+		}(logfile)
+
+		log.SetOutput(logfile)
+	}
+
+	confObj.Mu.RUnlock()
+}
+
+func rebindConfig() {
+
+	confObj.Mu.RLock()
+	if !confObj.StdoutLogging {
+		closeLog <- true
+	}
+	confObj.Mu.RUnlock()
+
+	confObj.Mu.Lock()
+
+	confObj.LogFile = viper.GetString("LogFile")
+	confObj.DBType = strings.ToLower(viper.GetString("DatabaseType"))
+	confObj.DBPath = viper.GetString("DatabasePath")
+	confObj.StdoutLogging = viper.GetBool("StdoutLogging")
+	confObj.CacheInterval = viper.GetDuration("StatusFetchInterval")
+	confObj.DBInterval = viper.GetDuration("DatabasePushInterval")
+
+	confObj.Instance.Name = viper.GetString("Instance.SiteName")
+	confObj.Instance.URL = viper.GetString("Instance.URL")
+	confObj.Instance.Owner = viper.GetString("Instance.OwnerName")
+	confObj.Instance.Mail = viper.GetString("Instance.Email")
+	confObj.Instance.Desc = viper.GetString("Instance.Description")
+
+	confObj.Mu.Unlock()
+
+	initLogging()
+}
+
+func initTemplates() *template.Template {
+	confObj.Mu.RLock()
+	assetsDir := confObj.AssetsDir
+	confObj.Mu.RUnlock()
+
+	return template.Must(template.ParseFiles(assetsDir + "/tmpl/index.html"))
+}
+
+// Pull DB data into cache, if available.
+func initDatabase() {
+	var db dbase
+	var err error
+
+	confObj.Mu.RLock()
+	switch confObj.DBType {
+
+	case "leveldb":
+		var lvl *leveldb.DB
+		lvl, err = leveldb.OpenFile(confObj.DBPath, nil)
+		db = &dbLevel{db: lvl}
+
+	case "sqlite":
+		var lite *sql.DB
+		db = &dbSqlite{db: lite}
+
+	}
+	confObj.Mu.RUnlock()
+
+	if err != nil {
+		log.Fatalf("%v\n", err.Error())
+	}
+
+	dbChan <- db
+
+	pullDatabase()
+}
+
+// Watch for SIGINT aka ^C
+// Close the log file then exit
+func watchForInterrupt() {
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, os.Interrupt)
+
+	go func() {
+		for sigint := range c {
+
+			log.Printf("\n\nCaught %v. Cleaning up ...\n", sigint)
+			confObj.Mu.RLock()
+			log.Printf("Closing database connection to %v...\n", confObj.DBPath)
+
+			db := <-dbChan
+
+			switch dbType := db.(type) {
+
+			case *dbLevel:
+				lvl := dbType
+				if err := lvl.db.Close(); err != nil {
+					log.Printf("%v\n", err.Error())
+				}
+
+			}
+
+			if !confObj.StdoutLogging {
+				closeLog <- true
+			}
+
+			confObj.Mu.RUnlock()
+			close(dbChan)
+			close(closeLog)
+
+			// Let everything catch up
+			time.Sleep(100 * time.Millisecond)
+			os.Exit(0)
+		}
+	}()
+}
+
+func titleScreen() {
+	fmt.Printf(`
+    
+                       _            _        _
+             __ _  ___| |___      _| |___  _| |_
+            / _  |/ _ \ __\ \ /\ / / __\ \/ / __|
+           | (_| |  __/ |_ \ V  V /| |_ >  <| |_
+            \__, |\___|\__| \_/\_/  \__/_/\_\\__|
+            |___/
+                       version ` + getwtxt + `
+                 github.com/getwtxt/getwtxt
+                          GPL  v3  
+
+`)
+}
+
+func helpScreen() {
+	fmt.Printf(`
+                        getwtxt Help
+
+
+                 :: Command Line Options ::
+
+    Command line options are used to explicitly override defaults,
+ or what has been specified in the configuration file.
+
+    -h [--help]      Print this help screen.
+    -m [--manual]    Print the manual.
+    -v [--version]   Print the version information and quit.
+    -c [--config]    Path to an alternate configuration file
+                       to use. May be relative or absolute.
+    -a [--assets]    Path to the assets directory, containing
+                       style.css and tmpl/index.html
+    -d [--db]        Path getwtxt should use for the database.
+    -t [--dbtype]    Type of database to use.
+                       Options: leveldb
+
+`)
+}
+func manualScreen() {
+	fmt.Printf(`
+                       :: Sections ::
+
+    >> Configuration File
+        Covers syntax and location of default configuration,
+        passing a specific configuration file to getwtxt, 
+        and acceptable formats for configuration files.
+
+    >> Customizing the Landing Page
+        Covers the location of the landing page template,
+        format of the template, and optional preprocessor
+        tags available to use when creating a new landing
+        page template.
+
+    >> Interacting With the Registry
+        Explains all API endpoints, their parameters,
+        and expected output.
+
+
+                  :: Configuration File ::
+
+    The default configuration file is in YAML format, chosen for
+ its clarity and its support of commenting (unlike JSON). It may
+ be placed in any of the following locations by default:
+
+    The same directory as the getwtxt executable
+    /usr/local/getwtxt/
+    /etc/
+    /usr/local/etc/
+
+    The paths are searched in that order. The first configuration
+ file found is used by getwtxt, while the locations further down
+ are ignored.
+    
+    Multiple configuration files may be used, however, with the
+ '-c' command line flag. The path passed to getwtxt via '-c' may
+ be relative or absolute. For example, both of the following are
+ allowed:
+
+    ./getwtxt -c myconfig.json
+    ./getwtxt -c /etc/ExtraConfigsDir/mysecondconfig.toml
+
+ The supported configuration types are:
+    YAML, TOML, JSON, HCL
+
+    The configuration file contains several options used to
+ customize your instance of getwtxt. None are required, they will 
+ simply use their default value unless otherwise specified.
+
+    ListenPort: Defines the port getwtxt should bind to.
+        Default: 9001
+
+    DatabaseType: The type of back-end getwtxt should use
+        to store registry data. Currently, only leveldb
+        is available, with more options in development.
+        Default: leveldb
+
+    DatabasePath: The location of the LevelDB structure
+        used by getwtxt to back up registry data. This
+        can be a relative or absolute path.
+        Default: getwtxt.db
+
+    AssetsDirectory: This is the directory where getwtxt
+        can find style.css and tmpl/index.html -- the
+        stylesheet for the landing page and the landing
+        page template, respectively.
+        Default: assets
+
+    StdoutLogging: Boolean used to determine whether
+        getwtxt should send logging output to stdout.
+        This is useful for debugging, but you should
+        probably save your logs once your instance 
+        is running.
+        Default: false
+
+    LogFile: The location of getwtxt's log file. This,
+        like DatabasePath, can be relative or absolute.
+        Default: getwtxt.log
+
+    DatabasePushInterval: The interval on which getwtxt
+        will push registry data from the in-memory cache
+        to the on-disk LevelDB database. The following
+        time suffixes may be used:
+            ns, us, ms, s, m, h
+        Default: 5m
+
+    StatusFetchInterval: The interval on which getwtxt
+        will crawl all users' twtxt files to retrieve
+        new statuses. The same time suffixes as
+        DatabasePushInterval may be used.
+        Default: 1h
+
+    Instance: Signifies the start of instance-specific
+        meta information. The following are used only
+        for the summary and use information displayed
+        by the default web page for getwtxt. If desired,
+        the assets/tmpl/index.html file may be
+        customized. Keep in mind that in YAML, the
+        following options must be preceded by two spaces
+        so that they are interpreted as sub-options.
+
+    SiteName: The name of your getwtxt instance.
+        Default: getwtxt
+
+    URL: The publicly-accessible URL of your 
+        getwtxt instance.
+        Default: https://twtxt.example.com
+
+    OwnerName: Your name.
+        Default: Anonymous Microblogger 
+
+    Email: Your email address.
+        Default: nobody@knows
+
+    Description: A short description of your getwtxt
+        instance or your site. As this likely includes
+        whitespace, it should be in double-quotes.
+        This can include XHTML or HTML line breaks if 
+        desired: 
+            <br />
+            <br>
+        Default: "A fast, resilient twtxt registry
+            server written in Go!"
+
+
+             :: Customizing the Landing Page ::
+
+    If you like, feel free to customize the landing page
+ template provided at 
+
+        assets/tmpl/index.html
+
+    It must be standard HTML or XHTML. There are a few special 
+ tags available to use that will be replaced with specific values 
+ when the template is parsed by getwtxt.
+
+    Values are derived from the "Instance" section of the 
+ configuration file, except for the version of getwtxt used. The 
+ following will be in the form of:
+    
+    {{.TemplateTag}} What it will be replaced with when
+        the template is processed and the landing page is
+        served to a visitor.
+
+    The surrounding double braces and prefixed period are required 
+ if you choose to use these tags in your modified landing page. The
+ tags themselves are not required; access to them is provided simply
+ for convenience.
+
+    {{.Vers}} The version of getwtxt used. Does not include
+        the preceding 'v'. Ex: 0.2.0
+
+    {{.Name}} The name of the instance.
+
+    {{.Owner}} The instance owner's name.
+
+    {{.Mail}} Email address used for contacting the instance
+        owner if the need arises.
+
+    {{.Desc}} Short description placed in the configuration
+        file. This is why HTML tags are allowed.
+
+    {{.URL}} The publicly-accessible URL of your instance. In
+        the default landing page, example API calls are shown
+        using this URL for the convenience of the user.
+
+
+              :: Interacting with the Registry ::
+
+    The registry API is rather simple, and can be interacted with
+ via the command line using cURL. Example output of the calls will
+ not be provided. 
+
+    Pseudo line-breaks will be represented with a backslash. 
+ Examples with line-breaks are not syntactically correct and will
+ be rejected by cURL. Please concatenate the example calls without 
+ the backslash. This is only present to maintain consistent 
+ formatting for this manual text.
+
+    Ex: 
+        /api/plain/users\
+        ?q=FOO
+    Should be: 
+        /api/plain/users?q=FOO
+
+    All queries (every call except adding users) accept the
+ ?page=N parameter, where N > 0. The output is provided in groups 
+ of 20 results. For example, indexed at 1, ?page=2 (or &page=2 if 
+ it is not the first parameter) appended to any query will return 
+ results 21 through 40. If the page requested will exceed the 
+ bounds of the query output, the last 20 query results are returned.
+
+ Adding a user:
+    curl -X POST 'http://localhost:9001/api/plain/users\
+        ?url=https://gbmor.dev/twtxt.txt&nickname=gbmor'
+
+ Retrieve user list:
+    curl 'http://localhost:9001/api/plain/users'
+
+ Retrieve all statuses:
+    curl 'http://localhost:9001/api/plain/tweets'
+
+ Retrieve all statuses with mentions:
+    curl 'http://localhost:9001/api/plain/mentions'
+
+ Retrieve all statuses with tags:
+    curl 'http://localhost:9001/api/plain/tags'
+
+ Query for users by keyword:
+    curl 'http://localhost:9001/api/plain/users?q=FOO'
+
+ Query for users by URL:
+    curl 'http://localhost:9001/api/plain/users\
+        ?url=https://gbmor.dev/twtxt.txt'
+
+ Query for statuses by substring:
+    curl 'http://localhost:9001/api/plain/tweets\
+        ?q=SUBSTRING'
+
+ Query for statuses mentioning a user:
+    curl 'http://localhost:9001/api/plain/mentions\
+        ?url=https://gbmor.dev/twtxt.txt'
+
+ Query for statuses with a given tag:
+    curl 'http://localhost:9001/api/plain/tags/myTagHere'
+
+`)
+}