www - Unnamed repository; edit this file 'description' to name the repository.

diff options

Diffstat (limited to 'index.html')

-rw-r--r--

1 files changed, 2 insertions, 0 deletions

9-05-21 04:13:21 -0400 database push/pull functions' href='/gbmor/getwtxt/commit/cache.go?h=v0.3.2&id=c896e6b85ff32c3c0c45336de8b75752a31574ca'>c896e6b ^

b29e1c1 ^

c896e6b ^

b29e1c1 ^

c896e6b ^

b29e1c1 ^

c896e6b ^

b29e1c1 ^

c896e6b ^

b29e1c1 ^

c896e6b ^

b29e1c1 ^

c896e6b ^

b29e1c1 ^

c896e6b ^

b29e1c1 ^

c896e6b ^

b29e1c1 ^

c896e6b ^

b29e1c1 ^

c896e6b ^

b29e1c1 ^

c896e6b ^

b29e1c1 ^

c896e6b ^

b29e1c1 ^

c896e6b ^

b29e1c1 ^

c896e6b ^

b29e1c1 ^

c896e6b ^

b29e1c1 ^

c896e6b ^

b29e1c1 ^

c896e6b ^

b29e1c1 ^

c896e6b ^

b29e1c1 ^

c896e6b ^

b29e1c1 ^

c896e6b ^

c896e6b ^

b29e1c1 ^

c896e6b ^

c896e6b ^

b29e1c1 ^

c896e6b ^

b29e1c1 ^

c896e6b ^

b29e1c1 ^

c896e6b ^

b29e1c1 ^

c896e6b ^

b29e1c1 ^

c896e6b ^

c896e6b ^

13
14

19
20

25
26

41
42

43
44

53
54
55

67
68

79
80

81
82

88
89

90
91

97
98

104
105
106

114
115

123
124
125

132
133

144
145

161
162
163

175
176
177

179
180

181
182

package main

import (
	"log"
	"reflect"
	"strings"
	"time"

	"github.com/getwtxt/registry"
	"github.com/syndtr/goleveldb/leveldb"
)

// Checks whether it's time to refresh
// the cache.
func checkCacheTime() bool {
	return time.Since(confObj.lastCache) > confObj.cacheInterval
}

// Checks whether it's time to push
// the cache to the database
func checkDBtime() bool {
	return time.Since(confObj.lastPush) > confObj.dbInterval
}

// Launched by init as a goroutine to constantly watch
// for the update interval to pass.
func cacheAndPush() {
	for {
		if checkCacheTime() {
			refreshCache()
		}
		if checkDBtime() {
			if err := pushDatabase(); err != nil {
				log.Printf("Error pushing cache to database: %v\n", err)
			}
		}
	}
}

// Refreshes the cache.
func refreshCache() {

	// Iterate over the registry and
	// update each individual user.
	for k := range twtxtCache.Reg {
		err := twtxtCache.UpdateUser(k)
		if err != nil {
			log.Printf("%v\n", err)
			continue
		}
	}

	// Re-scrape all the remote registries
	// to see if they have any new users
	// to add locally.
	for _, v := range remoteRegistries.List {
		err := twtxtCache.ScrapeRemoteRegistry(v)
		if err != nil {
			log.Printf("Error while refreshing local copy of remote registry user data: %v\n", err)
		}
	}
	confObj.mu.Lock()
	confObj.lastCache = time.Now()
	confObj.mu.Unlock()
}

// Pushes the registry's cache data to a local
// database for safe keeping.
func pushDatabase() error {
	// Acquire the database from the aether.
	// goleveldb is concurrency-safe, so we
	// can immediately push it back into the
	// channel for other functions to use.
	db := <-dbChan
	dbChan <- db

	// Create a batch write job so it can
	// be done at one time rather than
	// per entry.
	twtxtCache.Mu.RLock()
	var dbBasket *leveldb.Batch
	for k, v := range twtxtCache.Reg {
		dbBasket.Put([]byte(k+"*Nick"), []byte(v.Nick))
		dbBasket.Put([]byte(k+"*URL"), []byte(v.URL))
		dbBasket.Put([]byte(k+"*IP"), []byte(v.IP))
		dbBasket.Put([]byte(k+"*Date"), []byte(v.Date))
		for i, e := range v.Status {
			rfc := i.Format(time.RFC3339)
			dbBasket.Put([]byte(k+"*Status*"+rfc), []byte(e))
		}
	}
	twtxtCache.Mu.RUnlock()

	// Save our list of remote registries to scrape.
	for k, v := range remoteRegistries.List {
		dbBasket.Put([]byte("remote*"+string(k)), []byte(v))
	}

	// Execute the batch job.
	if err := db.Write(dbBasket, nil); err != nil {
		return err
	}

	// Update the last push time for
	// our timer/watch function to
	// reference.
	confObj.mu.Lock()
	confObj.lastPush = time.Now()
	confObj.mu.Unlock()

	return nil
}

// Pulls registry data from the DB on startup.
// Iterates over the database one entry at a time.
func pullDatabase() {
	// Acquire the database from the aether.
	// goleveldb is concurrency-safe, so we
	// can immediately push it back into the
	// channel for other functions to use.
	db := <-dbChan
	dbChan <- db

	iter := db.NewIterator(nil, nil)

	// Read the database key-by-key
	for iter.Next() {
		key := iter.Key()
		val := iter.Value()

		split := strings.Split(string(key), "*")
		urls := string(split[0])
		field := string(split[1])

		// Start with an empty Data struct. If
		// there's already one in the cache, pull
		// it and use it instead.
		data := registry.NewUserData()
		twtxtCache.Mu.RLock()
		if _, ok := twtxtCache.Reg[urls]; ok {
			data = twtxtCache.Reg[urls]
		}
		twtxtCache.Mu.RUnlock()
		ref := reflect.ValueOf(data).Elem()

		// Use reflection to find the right field
		// in the Data struct. Once found, assign
		// the value and break so the DB iteration
		// can continue.
		if field != "Status" && urls != "remote" {
			for i := 0; i < ref.NumField(); i++ {

				f := ref.Field(i)
				if f.String() == field {
					f.Set(reflect.ValueOf(val))
					break
				}
			}
		} else if field == "Status" && urls != "remote" {

			// If we're looking at a Status entry in the DB,
			// parse the time then add it to the TimeMap under
			// data.Status
			thetime, err := time.Parse("RFC3339", split[2])
			if err != nil {
				log.Printf("%v\n", err)
			}
			data.Status[thetime] = string(val)

		} else {
			// The third and final possibility is
			// if we've come across an entry for
			// a remote twtxt registry to scrape.
			// If so, add it to our list.
			remoteRegistries.Mu.Lock()
			remoteRegistries.List = append(remoteRegistries.List, string(val))
			remoteRegistries.Mu.Unlock()
			continue
		}

		// Push the data struct (back) into
		// the cache.
		twtxtCache.Mu.Lock()
		twtxtCache.Reg[urls] = data
		twtxtCache.Mu.Unlock()
	}

	iter.Release()
	err := iter.Error()
	if err != nil {
		log.Printf("Error while pulling DB into registry cache: %v\n", err)
	}
}


context:
space:
mode: