Update to reflect changes to sfomuseum/go-activitypub v0.0.2 (#3)

* snapshot: update to reflect change to sfomuseum/go-activitypub, compiles but untested * update to use activities database, untested * update makefile * update to use sfomuseum/go-activitypub v0.0.2, vendor deps --------- Co-authored-by: sfomuseumbot <sfomuseumbot@localhost>
sfomuseum · Nov 12, 2024 · e79e86b · e79e86b
1 parent 2d3ef35
commit e79e86b
Show file tree

Hide file tree

Showing 605 changed files with 19,255 additions and 3,633 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,6 @@
 *~
 bin
 work
-*.zip
+*.zip
+go.work*
+.DS_Store
diff --git a/Makefile b/Makefile
@@ -4,6 +4,7 @@ LDFLAGS=-s -w
 TABLE_PREFIX=
 
 ACCOUNTS_DB_URI=awsdynamodb://$(TABLE_PREFIX)accounts?partition_key=Id&allow_scans=true&local=true&region=localhost&credentials=anon:
+ACTIVITIES_DB_URI=awsdynamodb://$(TABLE_PREFIX)activities?partition_key=Id&allow_scans=true&local=true&region=localhost&credentials=anon:
 FOLLOWING_DB_URI=awsdynamodb://$(TABLE_PREFIX)following?partition_key=Id&allow_scans=true&local=true&region=localhost&credentials=anon:
 FOLLOWERS_DB_URI=awsdynamodb://$(TABLE_PREFIX)followers?partition_key=Id&allow_scans=true&local=true&region=localhost&credentials=anon:
 BLOCKS_DB_URI=awsdynamodb://$(TABLE_PREFIX)blocks?partition_key=Id&allow_scans=true&local=true&region=localhost&credentials=anon:
@@ -19,6 +20,7 @@ cli:
 publish:
 	go run cmd/publish-feeds/main.go \
 		-accounts-database-uri '$(ACCOUNTS_DB_URI)' \
+		-activities-database-uri '$(ACTIVITIES_DB_URI)' \
 		-followers-database-uri '$(FOLLOWERS_DB_URI)' \
 		-posts-database-uri '$(POSTS_DB_URI)' \
 		-deliveries-database-uri '$(DELIVERIES_DB_URI)' \

diff --git a/app/dynamodb/tables/create/create.go b/app/dynamodb/tables/create/create.go
@@ -12,25 +12,28 @@ import (
 	ap_dynamodb "github.com/sfomuseum/go-activitypub-feeds/schema/dynamodb"
 )
 
-func Run(ctx context.Context, logger *slog.Logger) error {
+func Run(ctx context.Context) error {
 	fs := DefaultFlagSet()
-	return RunWithFlagSet(ctx, fs, logger)
+	return RunWithFlagSet(ctx, fs)
 }
 
-func RunWithFlagSet(ctx context.Context, fs *flag.FlagSet, logger *slog.Logger) error {
+func RunWithFlagSet(ctx context.Context, fs *flag.FlagSet) error {
 
 	opts, err := OptionsFromFlagSet(ctx, fs)
 
 	if err != nil {
 		return fmt.Errorf("Failed to derive options from flagset, %w", err)
 	}
 
-	return RunWithOptions(ctx, opts, logger)
+	return RunWithOptions(ctx, opts)
 }
 
-func RunWithOptions(ctx context.Context, opts *RunOptions, logger *slog.Logger) error {
+func RunWithOptions(ctx context.Context, opts *RunOptions) error {
 
-	slog.SetDefault(logger)
+	if opts.Verbose {
+		slog.SetLogLoggerLevel(slog.LevelDebug)
+		slog.Debug("Verbose logging enabled")
+	}
 
 	cl, err := aa_dynamodb.NewClient(ctx, opts.DynamodbClientURI)
 

diff --git a/app/dynamodb/tables/create/flags.go b/app/dynamodb/tables/create/flags.go
@@ -10,6 +10,7 @@ var refresh bool
 
 var dynamodb_client_uri string
 var table_prefix string
+var verbose bool
 
 func DefaultFlagSet() *flag.FlagSet {
 
@@ -18,6 +19,6 @@ func DefaultFlagSet() *flag.FlagSet {
 	fs.BoolVar(&refresh, "refresh", false, "...")
 	fs.StringVar(&dynamodb_client_uri, "dynamodb-client-uri", "", "...")
 	fs.StringVar(&table_prefix, "table-prefix", "", "...")
-
+	fs.BoolVar(&verbose, "verbose", false, "Enable verbose (debug) logging.")
 	return fs
 }
diff --git a/app/dynamodb/tables/create/options.go b/app/dynamodb/tables/create/options.go
@@ -12,6 +12,7 @@ type RunOptions struct {
 	Refresh           bool
 	DynamodbClientURI string
 	TablePrefix       string
+	Verbose           bool
 }
 
 func OptionsFromFlagSet(ctx context.Context, fs *flag.FlagSet) (*RunOptions, error) {
@@ -28,6 +29,7 @@ func OptionsFromFlagSet(ctx context.Context, fs *flag.FlagSet) (*RunOptions, err
 		Refresh:           refresh,
 		DynamodbClientURI: dynamodb_client_uri,
 		TablePrefix:       table_prefix,
+		Verbose:           verbose,
 	}
 
 	return opts, nil

diff --git a/app/feeds/publish/flags.go b/app/feeds/publish/flags.go
@@ -8,6 +8,7 @@ import (
 )
 
 var accounts_database_uri string
+var activities_database_uri string
 var followers_database_uri string
 var deliveries_database_uri string
 var posts_database_uri string
@@ -29,9 +30,10 @@ var verbose bool
 
 func DefaultFlagSet() *flag.FlagSet {
 
-	fs := flagset.NewFlagSet("follow")
+	fs := flagset.NewFlagSet("publish")
 
 	fs.StringVar(&accounts_database_uri, "accounts-database-uri", "", "...")
+	fs.StringVar(&activities_database_uri, "activities-database-uri", "", "...")
 	fs.StringVar(&posts_database_uri, "posts-database-uri", "", "...")
 	fs.StringVar(&post_tags_database_uri, "post-tags-database-uri", "", "...")
 	fs.StringVar(&deliveries_database_uri, "deliveries-database-uri", "", "...")

diff --git a/app/feeds/publish/options.go b/app/feeds/publish/options.go
@@ -13,18 +13,20 @@ import (
 
 type RunOptions struct {
 	AccountsDatabaseURI             string
-	FeedsPublicationLogsDatabaseURI string
+	ActivitiesDatabaseURI           string
 	PostsDatabaseURI                string
 	PostTagsDatabaseURI             string
 	FollowersDatabaseURI            string
 	DeliveriesDatabaseURI           string
+	FeedsPublicationLogsDatabaseURI string
 	DeliveryQueueURI                string
 	AccountName                     string
 	Mode                            string
 	FeedURIs                        []string
 	URIs                            *uris.URIs
 	Verbose                         bool
 	MaxPostsPerFeed                 int
+	MaxAttempts                     int
 	Templates                       *template.Template
 }
 
@@ -50,11 +52,12 @@ func OptionsFromFlagSet(ctx context.Context, fs *flag.FlagSet) (*RunOptions, err
 
 	opts := &RunOptions{
 		AccountsDatabaseURI:             accounts_database_uri,
-		FeedsPublicationLogsDatabaseURI: feeds_publication_logs_database_uri,
+		ActivitiesDatabaseURI:           activities_database_uri,
 		FollowersDatabaseURI:            followers_database_uri,
 		DeliveriesDatabaseURI:           deliveries_database_uri,
 		PostsDatabaseURI:                posts_database_uri,
 		PostTagsDatabaseURI:             post_tags_database_uri,
+		FeedsPublicationLogsDatabaseURI: feeds_publication_logs_database_uri,
 		DeliveryQueueURI:                delivery_queue_uri,
 		AccountName:                     account_name,
 		Mode:                            mode,

diff --git a/app/feeds/publish/publish.go b/app/feeds/publish/publish.go
@@ -13,67 +13,82 @@ import (
 	"github.com/mmcdole/gofeed"
 	"github.com/sfomuseum/go-activitypub"
 	"github.com/sfomuseum/go-activitypub-feeds"
-	ap_slog "github.com/sfomuseum/go-activitypub/slog"
+	"github.com/sfomuseum/go-activitypub/database"
+	"github.com/sfomuseum/go-activitypub/posts"
+	"github.com/sfomuseum/go-activitypub/queue"
 )
 
 type itemTemplateVars struct {
 	FeedURL string
 	Item    *gofeed.Item
 }
 
-func Run(ctx context.Context, logger *slog.Logger) error {
+func Run(ctx context.Context) error {
 	fs := DefaultFlagSet()
-	return RunWithFlagSet(ctx, fs, logger)
+	return RunWithFlagSet(ctx, fs)
 }
 
-func RunWithFlagSet(ctx context.Context, fs *flag.FlagSet, logger *slog.Logger) error {
+func RunWithFlagSet(ctx context.Context, fs *flag.FlagSet) error {
 
 	opts, err := OptionsFromFlagSet(ctx, fs)
 
 	if err != nil {
 		return fmt.Errorf("Failed to derive options from flagset, %w", err)
 	}
 
-	return RunWithOptions(ctx, opts, logger)
+	return RunWithOptions(ctx, opts)
 }
 
-func RunWithOptions(ctx context.Context, opts *RunOptions, logger *slog.Logger) error {
+func RunWithOptions(ctx context.Context, opts *RunOptions) error {
 
-	ap_slog.ConfigureLogger(logger, opts.Verbose)
+	if opts.Verbose {
+		slog.SetLogLoggerLevel(slog.LevelDebug)
+		slog.Debug("Verbose logging enabled")
+	}
+
+	// logger := slog.Default()
 
-	accounts_db, err := activitypub.NewAccountsDatabase(ctx, opts.AccountsDatabaseURI)
+	accounts_db, err := database.NewAccountsDatabase(ctx, opts.AccountsDatabaseURI)
 
 	if err != nil {
-		return fmt.Errorf("Failed to create new database, %w", err)
+		return fmt.Errorf("Failed to instantiate accounts database, %w", err)
 	}
 
 	defer accounts_db.Close(ctx)
 
-	posts_db, err := activitypub.NewPostsDatabase(ctx, opts.PostsDatabaseURI)
+	activities_db, err := database.NewActivitiesDatabase(ctx, opts.ActivitiesDatabaseURI)
+
+	if err != nil {
+		return fmt.Errorf("Failed to instantiate activities database, %w", err)
+	}
+
+	defer activities_db.Close(ctx)
+
+	posts_db, err := database.NewPostsDatabase(ctx, opts.PostsDatabaseURI)
 
 	if err != nil {
 		return fmt.Errorf("Failed to create instantiate posts database, %w", err)
 	}
 
 	defer posts_db.Close(ctx)
 
-	post_tags_db, err := activitypub.NewPostTagsDatabase(ctx, opts.PostTagsDatabaseURI)
+	post_tags_db, err := database.NewPostTagsDatabase(ctx, opts.PostTagsDatabaseURI)
 
 	if err != nil {
 		return fmt.Errorf("Failed to create instantiate post tags database, %w", err)
 	}
 
 	defer post_tags_db.Close(ctx)
 
-	followers_db, err := activitypub.NewFollowersDatabase(ctx, opts.FollowersDatabaseURI)
+	followers_db, err := database.NewFollowersDatabase(ctx, opts.FollowersDatabaseURI)
 
 	if err != nil {
 		return fmt.Errorf("Failed to create instantiate followers database, %w", err)
 	}
 
 	defer followers_db.Close(ctx)
 
-	deliveries_db, err := activitypub.NewDeliveriesDatabase(ctx, opts.DeliveriesDatabaseURI)
+	deliveries_db, err := database.NewDeliveriesDatabase(ctx, opts.DeliveriesDatabaseURI)
 
 	if err != nil {
 		return fmt.Errorf("Failed to create instantiate deliveries database, %w", err)
@@ -89,7 +104,7 @@ func RunWithOptions(ctx context.Context, opts *RunOptions, logger *slog.Logger)
 
 	defer feeds_db.Close(ctx)
 
-	delivery_q, err := activitypub.NewDeliveryQueue(ctx, opts.DeliveryQueueURI)
+	delivery_q, err := queue.NewDeliveryQueue(ctx, opts.DeliveryQueueURI)
 
 	if err != nil {
 		return fmt.Errorf("Failed to create new delivery queue, %w", err)
@@ -139,14 +154,18 @@ func RunWithOptions(ctx context.Context, opts *RunOptions, logger *slog.Logger)
 
 				guid := item.GUID
 
+				logger := slog.Default()
+				logger = logger.With("feed", feed_url)
+				logger = logger.With("item", guid)
+
 				is_published, err := feeds_db.IsPublished(ctx, acct.Id, feed_url, guid)
 
 				if err != nil {
 					return fmt.Errorf("Failed to determine if %s#%s has been published by %d", feed_url, guid, acct.Id)
 				}
 
 				if is_published {
-					logger.Debug("Already published", "feed", feed_url, "item", guid)
+					logger.Debug("Item already published")
 					continue
 				}
 
@@ -168,31 +187,59 @@ func RunWithOptions(ctx context.Context, opts *RunOptions, logger *slog.Logger)
 
 				body := buf.String()
 
-				slog.Debug("Feed item", "feed", feed_url, "item", guid, "body", body)
+				logger.Debug("Post item", "body", body)
 
-				post_opts := &activitypub.AddPostOptions{
+				post_opts := &posts.AddPostOptions{
 					URIs:             opts.URIs,
 					PostsDatabase:    posts_db,
 					PostTagsDatabase: post_tags_db,
 				}
 
-				post, post_tags, err := activitypub.AddPost(ctx, post_opts, acct, body)
+				post, mentions, err := posts.AddPost(ctx, post_opts, acct, body)
+
+				if err != nil {
+					return fmt.Errorf("Failed to add post, %w", err)
+				}
+
+				logger = logger.With("post id", post.Id)
+
+				ap_activity, err := posts.ActivityFromPost(ctx, opts.URIs, acct, post, mentions)
+
+				if err != nil {
+					return fmt.Errorf("Failed to create new (create) activity, %w", err)
+				}
+
+				activity, err := activitypub.NewActivity(ctx, ap_activity)
+
+				if err != nil {
+					return fmt.Errorf("Failed to create AP wrapper, %w", err)
+				}
+
+				logger = logger.With("activity id", activity.Id)
+
+				activity.AccountId = acct.Id
+				activity.ActivityType = activitypub.PostActivityType
+				activity.ActivityTypeId = post.Id
+
+				err = activities_db.AddActivity(ctx, activity)
 
 				if err != nil {
-					return fmt.Errorf("Failed to add new post, %w", err)
+					return fmt.Errorf("Failed to add new activity, %w", err)
 				}
 
-				deliver_opts := &activitypub.DeliverPostToFollowersOptions{
+				deliver_opts := &queue.DeliverActivityToFollowersOptions{
 					AccountsDatabase:   accounts_db,
 					FollowersDatabase:  followers_db,
 					DeliveriesDatabase: deliveries_db,
 					DeliveryQueue:      delivery_q,
-					Post:               post,
-					PostTags:           post_tags,
+					Activity:           activity,
+					Mentions:           mentions,
 					URIs:               opts.URIs,
 				}
 
-				err = activitypub.DeliverPostToFollowers(ctx, deliver_opts)
+				logger.Debug("Deliver activity")
+
+				err = queue.DeliverActivityToFollowers(ctx, deliver_opts)
 
 				if err != nil {
 					return fmt.Errorf("Failed to deliver post, %w", err)

diff --git a/cmd/create-dynamodb-tables/main.go b/cmd/create-dynamodb-tables/main.go
@@ -2,21 +2,17 @@ package main
 
 import (
 	"context"
-	"os"
+	"log"
 
 	"github.com/sfomuseum/go-activitypub-feeds/app/dynamodb/tables/create"
-	"github.com/sfomuseum/go-activitypub/slog"
 )
 
 func main() {
 
 	ctx := context.Background()
-	logger := slog.Default()
-
-	err := create.Run(ctx, logger)
+	err := create.Run(ctx)
 
 	if err != nil {
-		logger.Error("Failed to create tables", "error", err)
-		os.Exit(1)
+		log.Fatalf("Failed to create table, %v", err)
 	}
 }
diff --git a/cmd/publish-feeds/main.go b/cmd/publish-feeds/main.go
@@ -2,21 +2,17 @@ package main
 
 import (
 	"context"
-	"os"
+	"log"
 
 	"github.com/sfomuseum/go-activitypub-feeds/app/feeds/publish"
-	"github.com/sfomuseum/go-activitypub/slog"
 )
 
 func main() {
 
 	ctx := context.Background()
-	logger := slog.Default()
-
-	err := publish.Run(ctx, logger)
+	err := publish.Run(ctx)
 
 	if err != nil {
-		logger.Error("Failed to publish feeds", "error", err)
-		os.Exit(1)
+		log.Fatalf("Failed to publish feeds, %v", err)
 	}
 }