Bitcoin Transaction Monitor (#45)

This PR adds a scatterplot (entry time x feerate) of recent mempool entries.

Author: 0xB10C

README.md

@@ -8,23 +8,18 @@
 Seemingly stuck and longtime-unconfirmed transactions can be quite annoying for users transacting on the Bitcoin network.
 The idea of mempool.observer is to provide users with information about unconfirmed transactions and transaction fees.
 
-> This is v2-master of memo. 
-> A full project refresh.
-
 ## Project Structure
 
 Folder Structure
 ```
 memo/
-├── api/          # Go module functioning as an API returning JSON
-├── database/     # Database creation scripts
-├── memod/        # Go module functioning as a worker deamon wirting data to database
+├── api/          # Go code that compiles to a binary API returning JSON from a Redis instance
+├── memod/        # Go code that compiles to a binary worker daemon writing data to a Redis instance
 └── www/          # Statically served HTML, JS and CSS files
 ```
 
 There exists a overview of my [infrastructure setup](https://github.com/0xB10C/memo/wiki/Infrastructure-memo-v2) for mempool.observer.
 
-
 ## Project History
 
 I've started building the first version of mempool.observer mid 2017 as my first Bitcoin related project.
@@ -36,14 +31,19 @@ Later this year the bitcoin transaction fees rose and I had quite some traffic.
 The high fees were caused by a huge transaction flood as the price rose to $20k.
 I regularly had problems with long running scripts due to querying and processing the huge mempool on a low end VPS.
 Due to time constrains I wasn't able to improve the performance.
-This resulted in mempool.observer v1 dieing the not-maintained-death sometime in 2018.
+This resulted in mempool.observer version 1 dieing the not-maintained-death sometime in 2018.
 
-I've focused full time on Bitcoin in spring 2019 and spend a part of that time to work on v2.
-V2 is a full rewrite of mempool.observer - only the idea, license and the quote from Maxwell remained.
-The goal is to offer way more than v1 did, but build on a foundation with performance and maintainability in mind.
+I've focused full time on Bitcoin in spring 2019 and spend a part of that time to work on version 2.
+Version 2 is a full rewrite of mempool.observer - only the idea, license and the quote from Maxwell remained.
+The goal is to offer way more than version 1 did, but build on a foundation with performance and maintainability in mind.
 I'm open for ideas and feedback.
 
+## Self-Hosting
+
+Self-hosting memo is possible, but there is no detailed setup and maintenance documentation written yet.
+You might need to do some exploration on your own to get everything working.
+Keep in mind, that you need a customized Bitcoin Core version to run the Bitcoin Transaction Monitor.
 
-## Licencse
+## Licence
 This project and all it's files are licensed under a GNU Affero General Public License.
 

api/api.go

@@ -6,9 +6,11 @@ import (
 	"net/http"
 	"strconv"
 
+	"github.com/0xb10c/memo/api/cache"
 	"github.com/0xb10c/memo/api/config"
 	"github.com/0xb10c/memo/api/database"
 	"github.com/gin-contrib/cors"
+	"github.com/gin-contrib/gzip"
 	"github.com/gin-gonic/gin"
 )
 
@@ -25,20 +27,25 @@ func main() {
 	} else {
 		corsConfig.AllowOrigins = []string{"*"}
 	}
+
 	router.Use(cors.New(corsConfig))
+	router.Use(gzip.Gzip(gzip.DefaultCompression))
 
 	err := database.SetupDatabase()
 	if err != nil {
 		panic(fmt.Errorf("Failed to setup database: %v", err))
 	}
 
+	go cache.SetupCache()
+
 	api := router.Group("/api")
 	{
 		api.GET("/mempool", getMempool)
 		api.GET("/recentBlocks", getRecentBlocks)
 		api.GET("/historicalMempool/:timeframe/:by", getHistoricalMempool)
-		api.GET("/timeInMempool", getTimeInMempool)
 		api.GET("/transactionStats", getTransactionStats)
+		api.GET("/getMempoolEntries", getCachedMempoolEntries)
+		api.GET("/getRecentFeerateAPIData", getRecentFeerateAPIEntries)
 	}
 
 	portString := ":" + config.GetString("api.port")
@@ -117,9 +124,21 @@ func getHistoricalMempool(c *gin.Context) {
 	c.JSON(http.StatusOK, mempoolStates)
 }
 
-func getTimeInMempool(c *gin.Context) {
+func getTransactionStats(c *gin.Context) {
+	tss, err := database.GetTransactionStats()
+	if err != nil {
+		fmt.Println(err.Error())
+		c.JSON(http.StatusInternalServerError, gin.H{
+			"error": "Database error",
+		})
+		return
+	}
 
-	timestamp, timeAxis, feerateAxis, err := database.GetTimeInMempool()
+	c.JSON(http.StatusOK, tss)
+}
+
+func getMempoolEntries(c *gin.Context) {
+	mes, err := database.GetMempoolEntries()
 	if err != nil {
 		fmt.Println(err.Error())
 		c.JSON(http.StatusInternalServerError, gin.H{
@@ -128,15 +147,12 @@ func getTimeInMempool(c *gin.Context) {
 		return
 	}
 
-	c.JSON(http.StatusOK, gin.H{
-		"timestamp":   timestamp,
-		"feerateAxis": feerateAxis,
-		"timeAxis":    timeAxis,
-	})
+	c.JSON(http.StatusOK, mes)
 }
 
-func getTransactionStats(c *gin.Context) {
-	tss, err := database.GetTransactionStats()
+func getRecentFeerateAPIEntries(c *gin.Context) {
+
+	entries, err := database.GetRecentFeerateAPIEntries()
 	if err != nil {
 		fmt.Println(err.Error())
 		c.JSON(http.StatusInternalServerError, gin.H{
@@ -145,5 +161,18 @@ func getTransactionStats(c *gin.Context) {
 		return
 	}
 
-	c.JSON(http.StatusOK, tss)
+	c.JSON(http.StatusOK, entries)
+}
+
+func getCachedMempoolEntries(c *gin.Context) {
+	entries, err := database.GetMempoolEntriesCache()
+	if err != nil {
+		fmt.Println(err.Error())
+		c.JSON(http.StatusInternalServerError, gin.H{
+			"error": "Database error",
+		})
+		return
+	}
+	c.Header("Content-Type", "application/json; charset=utf-8")
+	c.String(http.StatusOK, entries)
 }

api/cache/cache.go

@@ -0,0 +1,45 @@
+package cache
+
+import (
+	"encoding/json"
+	"log"
+
+	"github.com/0xb10c/memo/api/database"
+	"github.com/jasonlvhit/gocron"
+)
+
+// SetupCache sets up caching scripts
+func SetupCache() {
+	SetupMempoolEntriesCacher()
+}
+
+// SetupMempoolEntriesCacher sets up a periodic GetMempoolEntries() fetch job
+func SetupMempoolEntriesCacher() {
+	cacheMempoolEntries()
+
+	fetchInterval := uint64(30)
+	s := gocron.NewScheduler()
+	s.Every(fetchInterval).Seconds().Do(cacheMempoolEntries)
+	log.Printf("Setup GetMempoolEntries() cacher to run every %d seconds.\n", fetchInterval)
+	<-s.Start()
+	defer s.Clear()
+}
+
+func cacheMempoolEntries() {
+	log.Printf("Caching mempool entries.\n")
+	mes, err := database.GetMempoolEntries()
+	if err != nil {
+		log.Printf("Error getting mempool entries %v.\n", err)
+	}
+
+	mesJSON, err := json.Marshal(mes)
+	if err != nil {
+		log.Printf("Error marshalling mempool entries %v.\n", err)
+	}
+
+	err = database.SetMempoolEntriesCache(string(mesJSON))
+	if err != nil {
+		log.Printf("Could not cache mempool entries %v.\n", err)
+	}
+
+}

api/database/database.go

@@ -1,18 +1,18 @@
 package database
 
 import (
-	"encoding/json"
 	"errors"
 	"strconv"
 	"time"
 
 	"github.com/0xb10c/memo/api/config"
+	"github.com/0xb10c/memo/api/types"
 	"github.com/gomodule/redigo/redis"
+	jsoniter "github.com/json-iterator/go"
 )
 
-var (
-	Pool *redis.Pool
-)
+var Pool *redis.Pool
+var json = jsoniter.ConfigCompatibleWithStandardLibrary
 
 func newPool() *redis.Pool {
 	dbUser := config.GetString("redis.user")
@@ -81,7 +81,7 @@ func GetMempool() (timestamp time.Time, feerateMapJSON string, megabyteMarkersJS
 	return
 }
 
-func GetRecentBlocks() (blocks []RecentBlock, err error) {
+func GetRecentBlocks() (blocks []types.RecentBlock, err error) {
 	c := Pool.Get()
 	defer c.Close()
 
@@ -90,9 +90,9 @@ func GetRecentBlocks() (blocks []RecentBlock, err error) {
 		return
 	}
 
-	blocks = make([]RecentBlock, 0)
+	blocks = make([]types.RecentBlock, 0)
 	for index := range reJSON {
-		block := RecentBlock{}
+		block := types.RecentBlock{}
 		err = json.Unmarshal([]byte(reJSON[index]), &block)
 		if err != nil {
 			return
@@ -110,7 +110,7 @@ type MempoolState struct {
 	DataInBuckets     []float64 `json:"dataInBuckets"`
 }
 
-func GetHistorical(timeframe int, by string) (hmds []HistoricalMempoolData, err error) {
+func GetHistorical(timeframe int, by string) (hmds []types.HistoricalMempoolData, err error) {
 	c := Pool.Get()
 	defer c.Close()
 
@@ -149,9 +149,9 @@ func GetHistorical(timeframe int, by string) (hmds []HistoricalMempoolData, err
 		return
 	}
 
-	hmds = make([]HistoricalMempoolData, 0)
+	hmds = make([]types.HistoricalMempoolData, 0)
 	for index := range hmdsJSON {
-		hmd := HistoricalMempoolData{}
+		hmd := types.HistoricalMempoolData{}
 		err = json.Unmarshal([]byte(hmdsJSON[index]), &hmd)
 		if err != nil {
 			return
@@ -162,61 +162,102 @@ func GetHistorical(timeframe int, by string) (hmds []HistoricalMempoolData, err
 	return
 }
 
-// GetTimeInMempool gets the TimeInMempool data from the database
-func GetTimeInMempool() (timestamp int64, timeAxis []int, feerateAxis []float64, err error) {
+// GetTransactionStats gets the Transaction Stats data from the database
+func GetTransactionStats() (tss []types.TransactionStat, err error) {
 	c := Pool.Get()
 	defer c.Close()
 
-	prefix := "timeInMempool"
-
-	response, err := redis.Strings(c.Do("MGET", prefix+":timeAxis", prefix+":feerateAxis", prefix+":utcTimestamp"))
+	tssJSON, err := redis.Strings(c.Do("LRANGE", "transactionStats", 0, 180))
 	if err != nil {
 		return
 	}
 
-	err = json.Unmarshal([]byte(response[0]), &timeAxis)
-	if err != nil {
-		return
+	type transactionStat struct {
+		SegwitCount int   `json:"segwitCount"`
+		RbfCount    int   `json:"rbfCount"`
+		TxCount     int   `json:"txCount"`
+		Timestamp   int64 `json:"timestamp"`
 	}
 
-	err = json.Unmarshal([]byte(response[1]), &feerateAxis)
-	if err != nil {
-		return
+	tss = make([]types.TransactionStat, 0)
+	for index := range tssJSON {
+		ts := types.TransactionStat{}
+		err = json.Unmarshal([]byte(tssJSON[index]), &ts)
+		if err != nil {
+			return
+		}
+		tss = append(tss, ts)
 	}
 
-	err = json.Unmarshal([]byte(response[2]), &timestamp)
+	return
+}
+
+// GetMempoolEntries gets the last x mempool Entries from the database
+func GetMempoolEntries() (mes []types.MempoolEntry, err error) {
+	c := Pool.Get()
+	defer c.Close()
+
+	// gets recent entries from 0 to 19999 (20k)
+	mesJSON, err := redis.Strings(c.Do("ZREVRANGE", "mempoolEntries", 0, 29999))
 	if err != nil {
 		return
 	}
 
+	mes = make([]types.MempoolEntry, 0)
+	for index := range mesJSON {
+		me := types.MempoolEntry{}
+		err = json.Unmarshal([]byte(mesJSON[index]), &me)
+		if err != nil {
+			return
+		}
+		mes = append(mes, me)
+	}
+
 	return
 }
 
-// GetTransactionStats gets the Transaction Stats data from the database
-func GetTransactionStats() (tss []TransactionStat, err error) {
+// SetMempoolEntriesCache SETs the response of a recent GetMempoolEntries() as a cache
+func SetMempoolEntriesCache(mesJSON string) (err error) {
 	c := Pool.Get()
 	defer c.Close()
 
-	tssJSON, err := redis.Strings(c.Do("LRANGE", "transactionStats", 0, 180))
+	_, err = c.Do("SET", "cache:mempoolEntries", mesJSON)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// GetMempoolEntriesCache GET the cached response of a recent GetMempoolEntries() call
+func GetMempoolEntriesCache() (mesJSON string, err error) {
+	c := Pool.Get()
+	defer c.Close()
+
+	mesJSON, err = redis.String(c.Do("GET", "cache:mempoolEntries"))
 	if err != nil {
 		return
 	}
+	return
+}
 
-	type transactionStat struct {
-		SegwitCount int   `json:"segwitCount"`
-		RbfCount    int   `json:"rbfCount"`
-		TxCount     int   `json:"txCount"`
-		Timestamp   int64 `json:"timestamp"`
+// GetRecentFeerateAPIEntries returns the recent feeRate API entrys from Redis
+func GetRecentFeerateAPIEntries() (entries []types.FeeRateAPIEntry, err error) {
+	c := Pool.Get()
+	defer c.Close()
+
+	entrysJSON, err := redis.Strings(c.Do("LRANGE", "feerateAPIEntries", 0, 100))
+	if err != nil {
+		return
 	}
 
-	tss = make([]TransactionStat, 0)
-	for index := range tssJSON {
-		ts := TransactionStat{}
-		err = json.Unmarshal([]byte(tssJSON[index]), &ts)
+	entries = make([]types.FeeRateAPIEntry, 0)
+	for index := range entrysJSON {
+		entry := types.FeeRateAPIEntry{}
+		err = json.Unmarshal([]byte(entrysJSON[index]), &entry)
 		if err != nil {
 			return
 		}
-		tss = append(tss, ts)
+		entries = append(entries, entry)
 	}
 
 	return