caching support

internal/influencer/influencer.go

@@ -1,6 +1,7 @@
 package influencer
 
 import (
+	"github.com/missionMeteora/iodb"
 	"github.com/swayops/sway/config"
 	"github.com/swayops/sway/internal/common"
 	"github.com/swayops/sway/platforms/facebook"
@@ -24,11 +25,14 @@ type Influencer struct {
 
 	Active   []*common.Deal // Accepted pending deals to be completed
 	Historic []*common.Deal // Contains historic deals completed
+
+	db *iodb.DB
 }
 
-func New(twitterId, instaId, fbId, ytId, tumblrId string, cfg *config.Config) (*Influencer, error) {
+func New(db *iodb.DB, twitterId, instaId, fbId, ytId, tumblrId string, cfg *config.Config) (*Influencer, error) {
 	inf := &Influencer{
 		Id: pseudoUUID(), // Possible change to standard numbering?
+		db: db,
 	}
 
 	err := inf.NewInsta(instaId, cfg)
@@ -55,7 +59,6 @@ func New(twitterId, instaId, fbId, ytId, tumblrId string, cfg *config.Config) (*
 		return inf, err
 	}
 
-	// Saving to db functionality TBD.. iodb?
 	return inf, nil
 }
 

misc/iodb_cache.go

@@ -0,0 +1,41 @@
+package misc
+
+import (
+	"bytes"
+	"io"
+	"time"
+
+	"github.com/missionMeteora/iodb"
+)
+
+const DefaultCacheDuration = 1 * time.Hour
+
+type Platform string
+
+const (
+	PlatformTwitter   Platform = "twitter"
+	PlatformInstagram          = "instagram"
+	PlatformFacebook           = "facebook"
+	PlatformYoutube            = "youtube"
+	PlatformTumblr             = "tumblr"
+)
+
+func GetPlatformCache(db *iodb.DB, platform Platform, pid string) (rc io.ReadCloser) {
+	b := db.Bucket(string(platform))
+	if b == nil {
+		return nil
+	}
+	rc, _ = b.Get(pid)
+	return
+}
+
+func PutPlatformCache(db *iodb.DB, platform Platform, pid string, data []byte, dur time.Duration) error {
+	b, err := db.CreateBucket(string(platform))
+	if err != nil {
+		return nil
+	}
+	if dur > 0 {
+		return b.PutTimed(pid, bytes.NewReader(data), dur)
+	}
+	return b.Put(pid, bytes.NewReader(data))
+}

platforms/facebook/post.go

@@ -1,6 +1,12 @@
 package facebook
 
-import "github.com/swayops/sway/config"
+import (
+	"encoding/json"
+
+	"github.com/missionMeteora/iodb"
+	"github.com/swayops/sway/config"
+	"github.com/swayops/sway/misc"
+)
 
 type Post struct {
 	Id        string
@@ -16,7 +22,16 @@ type Post struct {
 	Type string // "video", "photo", "shared_story", "link"
 }
 
-func (pt *Post) UpdateData(cfg *config.Config) error {
+func (pt *Post) UpdateData(db *iodb.DB, cfg *config.Config) (err error) {
+	if rc := misc.GetPlatformCache(db, misc.PlatformFacebook, pt.Id); rc != nil {
+		defer rc.Close()
+		var post postData
+		if err = json.NewDecoder(rc).Decode(&post); err != nil {
+			return
+		}
+		pt.Likes, pt.Comments, pt.Shares = post.Likes, post.Comments, post.Shares
+		return
+	}
 	if lk, err := getLikes(pt.Id, cfg); err == nil {
 		pt.Likes = lk
 	} else {
@@ -35,5 +50,10 @@ func (pt *Post) UpdateData(cfg *config.Config) error {
 		return err
 	}
 
-	return nil
+	j, _ := json.Marshal(postData{pt.Likes, pt.Comments, pt.Shares})
+	return misc.PutPlatformCache(db, misc.PlatformFacebook, pt.Id, j, misc.DefaultCacheDuration)
+}
+
+type postData struct {
+	Likes, Comments, Shares float32
 }

platforms/instagram/post.go

@@ -1,9 +1,11 @@
 package instagram
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 
+	"github.com/missionMeteora/iodb"
 	"github.com/swayops/sway/config"
 	"github.com/swayops/sway/misc"
 )
@@ -52,17 +54,26 @@ type PostLikes struct {
 	Count float32 `json:"count"`
 }
 
-func (pt *Post) UpdateData(cfg *config.Config) error {
-	endpoint := fmt.Sprintf(postInfoUrl, cfg.Instagram.Endpoint, pt.Id, cfg.Instagram.ClientId)
-
+func (pt *Post) UpdateData(db *iodb.DB, cfg *config.Config) (err error) {
 	var post DataByPost
-	err := misc.Request("GET", endpoint, "", &post)
-	if err != nil {
-		return err
-	}
-
-	if post.Data == nil {
-		return ErrBadResponse
+	if rc := misc.GetPlatformCache(db, misc.PlatformInstagram, pt.Id); rc != nil {
+		defer rc.Close()
+		if err = json.NewDecoder(rc).Decode(&post); err != nil {
+			return
+		}
+	} else {
+		endpoint := fmt.Sprintf(postInfoUrl, cfg.Instagram.Endpoint, pt.Id, cfg.Instagram.ClientId)
+
+		if err = misc.Request("GET", endpoint, "", &post); err != nil {
+			return err
+		}
+		if post.Data == nil {
+			return ErrBadResponse
+		}
+		j, _ := json.Marshal(&post)
+		if err = misc.PutPlatformCache(db, misc.PlatformInstagram, pt.Id, j, misc.DefaultCacheDuration); err != nil {
+			return
+		}
 	}
 
 	if post.Data.Comments != nil {
@@ -73,5 +84,5 @@ func (pt *Post) UpdateData(cfg *config.Config) error {
 		pt.Likes = post.Data.Likes.Count
 	}
 
-	return nil
+	return
 }

platforms/tumblr/posts.go

@@ -1,12 +1,14 @@
 package tumblr
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"log"
 	"math/big"
 	"time"
 
+	"github.com/missionMeteora/iodb"
 	"github.com/mrjones/oauth"
 	"github.com/swayops/sway/config"
 	"github.com/swayops/sway/misc"
@@ -18,6 +20,8 @@ var (
 		AuthorizeTokenUrl: "https://www.tumblr.com/oauth/authorize",
 		AccessTokenUrl:    "https://www.tumblr.com/oauth/access_token",
 	}
+
+	ErrBadResponse = errors.New(`Empty data response from insta post!`)
 )
 
 type Timestamp int64
@@ -40,7 +44,7 @@ func (posts Posts) Avgs() (reblog, likes, total float32) {
 }
 
 type Post struct {
-	ID        big.Int   `json:"id"`
+	Id        big.Int   `json:"id"`
 	Type      string    `json:"type"`
 	TS        Timestamp `json:"timestamp"`
 	NoteCount uint32    `json:"note_count"`
@@ -69,18 +73,32 @@ func (p *Post) Counts() (reblog, likes, total float32) {
 }
 
 // UpdateData needs the parent tumblr call because it needs the blog id
-func (p *Post) UpdateData(tr *Tumblr, cfg *config.Config) (err error) {
+func (p *Post) UpdateData(db *iodb.DB, tr *Tumblr, cfg *config.Config) (err error) {
+	id := p.Id.String()
+	if rc := misc.GetPlatformCache(db, misc.PlatformTumblr, id); rc != nil {
+		defer rc.Close()
+		var post Post
+		if err = json.NewDecoder(rc).Decode(&post); err != nil {
+			return
+		}
+		*p = post
+		return
+	}
 	var resp apiResponse
-	if err = misc.HttpGetJson(tr.client, fmt.Sprintf(singlePostUrl, cfg.Tumblr.Endpoint, tr.Id, p.ID.String()), &resp); err != nil {
+	if err = misc.HttpGetJson(tr.client, fmt.Sprintf(singlePostUrl, cfg.Tumblr.Endpoint, tr.Id, id), &resp); err != nil {
 		return
 	}
 	if resp.Meta.Status != 200 {
 		return errors.New(resp.Meta.Message)
 	}
-	if len(resp.Response.Posts) == 1 { // should never be more or less than 1
-		*p = *resp.Response.Posts[0]
+	if len(resp.Response.Posts) != 1 { // should never be more or less than 1
+		return ErrBadResponse
 	}
-	return
+
+	*p = *resp.Response.Posts[0]
+
+	j, _ := json.Marshal(p)
+	return misc.PutPlatformCache(db, misc.PlatformTumblr, id, j, misc.DefaultCacheDuration)
 }
 
 type apiResponse struct {

platforms/twitter/tweet.go

@@ -4,8 +4,11 @@ import (
 	"compress/gzip"
 	"encoding/json"
 	"fmt"
+	"io"
+	"io/ioutil"
 	"net/http"
 
+	"github.com/missionMeteora/iodb"
 	"github.com/swayops/sway/config"
 
 	"github.com/swayops/sway/misc"
@@ -95,11 +98,25 @@ func (t *Tweet) Hashtags() (out []string) {
 	return
 }
 
-func (t *Tweet) UpdateData(cfg *config.Config) (err error) {
+func (t *Tweet) UpdateData(db *iodb.DB, cfg *config.Config) (err error) {
+	var (
+		tmp Tweet
+		rc  io.ReadCloser
+	)
+	if rc = misc.GetPlatformCache(db, misc.PlatformTwitter, t.Id); rc != nil {
+		if err = json.NewDecoder(rc).Decode(&tmp); err == nil {
+			*t = tmp
+		}
+		rc.Close()
+		return err
+	}
+
 	var (
 		resp   *http.Response
 		client *http.Client
+		j      []byte
 	)
+
 	if client, err = getClient(cfg); err != nil {
 		return
 	}
@@ -117,9 +134,13 @@ func (t *Tweet) UpdateData(cfg *config.Config) (err error) {
 		defer gr.Close()
 		r = gr
 	}
-
-	var tmp Tweet
-	if err = json.NewDecoder(r).Decode(&tmp); err == nil {
+	if j, err = ioutil.ReadAll(r); err != nil {
+		return
+	}
+	if err = misc.PutPlatformCache(db, misc.PlatformTwitter, t.Id, j, misc.DefaultCacheDuration); err != nil {
+		return
+	}
+	if err = json.Unmarshal(j, &tmp); err == nil {
 		*t = tmp
 	}
 	return

platforms/youtube/post.go

@@ -1,6 +1,12 @@
 package youtube
 
-import "github.com/swayops/sway/config"
+import (
+	"encoding/json"
+
+	"github.com/missionMeteora/iodb"
+	"github.com/swayops/sway/config"
+	"github.com/swayops/sway/misc"
+)
 
 type Post struct {
 	Id          string
@@ -17,7 +23,16 @@ type Post struct {
 	Comments float32
 }
 
-func (pt *Post) UpdateData(cfg *config.Config) error {
+func (pt *Post) UpdateData(db *iodb.DB, cfg *config.Config) (err error) {
+	if rc := misc.GetPlatformCache(db, misc.PlatformYoutube, pt.Id); rc != nil {
+		defer rc.Close()
+		var post Post
+		if err = json.NewDecoder(rc).Decode(&post); err != nil {
+			return
+		}
+		*pt = post
+		return
+	}
 	views, likes, dislikes, comments, err := getVideoStats(pt.Id, cfg)
 	if err != nil {
 		return err
@@ -27,5 +42,7 @@ func (pt *Post) UpdateData(cfg *config.Config) error {
 	pt.Dislikes = dislikes
 	pt.Views = views
 	pt.Comments = comments
-	return nil
+	j, _ := json.Marshal(pt)
+
+	return misc.PutPlatformCache(db, misc.PlatformYoutube, pt.Id, j, misc.DefaultCacheDuration)
 }

server/server.go

@@ -2,30 +2,38 @@ package server
 
 import (
 	"fmt"
+	"path/filepath"
 	"sync"
 
 	"github.com/boltdb/bolt"
 	"github.com/gin-gonic/gin"
+	"github.com/missionMeteora/iodb"
 	"github.com/swayops/sway/config"
 	"github.com/swayops/sway/misc"
 )
 
 var buckets = []string{"test"}
 
 type Server struct {
-	cfg *config.Config
-	r   *gin.Engine
-	db  *bolt.DB
+	cfg  *config.Config
+	r    *gin.Engine
+	db   *bolt.DB
+	iodb *iodb.DB
 	// Db
 }
 
-func New(cfg *config.Config, r *gin.Engine) (*Server, error) {
+func New(cfg *config.Config, r *gin.Engine) (_ *Server, err error) {
+	var idb *iodb.DB
+	if idb, err = iodb.New(filepath.Join(cfg.DBPath, "cache.iodb"), nil); err != nil {
+		return
+	}
 	db := misc.OpenDB(cfg.DBPath, cfg.DBName)
 
 	srv := &Server{
-		cfg: cfg,
-		r:   r,
-		db:  db,
+		cfg:  cfg,
+		r:    r,
+		db:   db,
+		iodb: idb,
 	}
 
 	srv.InitializeDB(cfg.Buckets)