internal/gaby: sync groups in the sync endpoint
For starters, only golang-nuts group will be synced.
Updates golang/oscar#27
Change-Id: Ie9058c24db74af6e6a79e61151c1be832d02dd06
Reviewed-on: https://go-review.googlesource.com/c/oscar/+/622455
Reviewed-by: Jonathan Amsterdam <jba@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
diff --git a/internal/gaby/main.go b/internal/gaby/main.go
index ee4041c..0bc194b 100644
--- a/internal/gaby/main.go
+++ b/internal/gaby/main.go
@@ -34,6 +34,7 @@
"golang.org/x/oscar/internal/gcp/gemini"
"golang.org/x/oscar/internal/gerrit"
"golang.org/x/oscar/internal/github"
+ "golang.org/x/oscar/internal/googlegroups"
"golang.org/x/oscar/internal/llm"
"golang.org/x/oscar/internal/pebble"
"golang.org/x/oscar/internal/related"
@@ -71,6 +72,7 @@
addr string // address to serve HTTP on
githubProject string // github project to monitor and update
gerritProjects []string // gerrit projects to monitor and update
+ googleGroups []string // google groups to monitor and update
slog *slog.Logger // slog output to use
slogLevel *slog.LevelVar // slog level, for changing as needed
@@ -84,6 +86,7 @@
github *github.Client // github client to use
disc *discussion.Client // github discussion client to use
gerrit *gerrit.Client // gerrit client to use
+ ggroups *googlegroups.Client // google groups client to use
crawler *crawl.Crawler // web crawler to use
meter ometric.Meter // used to create Open Telemetry instruments
report *errorreporting.Client // used to report important gaby errors to Cloud Error Reporting service
@@ -109,6 +112,7 @@
addr: "localhost:4229", // 4229 = gaby on a phone
githubProject: "golang/go",
gerritProjects: []string{"go"},
+ googleGroups: []string{"golang-nuts"},
}
shutdown := g.initGCP()
@@ -123,6 +127,11 @@
_ = g.gerrit.Add(project) // in principle needed only once per g.db lifetime
}
+ g.ggroups = googlegroups.New(g.slog, g.db, g.secret, g.http)
+ for _, group := range g.googleGroups {
+ _ = g.ggroups.Add(group) // in principle needed only once per g.db lifetime
+ }
+
g.docs = docs.New(g.slog, g.db)
ai, err := gemini.NewClient(g.ctx, g.slog, g.secret, g.http, gemini.DefaultEmbeddingModel, gemini.DefaultGenerativeModel)
@@ -418,7 +427,7 @@
// syncEndpoint is called manually to invoke a specific sync job.
// It performs a sync if enablesync is true.
- // Usage: /sync?job={github | crawl | gerrit | discussion}
+ // Usage: /sync?job={github | crawl | gerrit | discussion | groups}
mux.HandleFunc("GET /"+syncEndpoint, func(w http.ResponseWriter, r *http.Request) {
g.slog.Info(syncEndpoint + " start")
defer g.slog.Info(syncEndpoint + " end")
@@ -443,6 +452,8 @@
err = g.syncCrawl(g.ctx)
case "gerrit":
err = g.syncGerrit(g.ctx)
+ case "groups":
+ err = g.syncGerrit(g.ctx)
default:
err = fmt.Errorf("unrecognized sync job %s", job)
}
@@ -543,6 +554,7 @@
gabyGitHubSyncLock = "gabygithubsync"
gabyDiscussionSyncLock = "gabydiscussionsync"
gabyGerritSyncLock = "gabygerritsync"
+ gabyGroupsSyncLock = "gabygroupssync"
gabyEmbedLock = "gabyembedsync"
gabyCrawlLock = "gabycrawlsync"
@@ -592,6 +604,18 @@
return nil
}
+func (g *Gaby) syncGroups(ctx context.Context) error {
+ g.db.Lock(gabyGroupsSyncLock)
+ defer g.db.Unlock(gabyGroupsSyncLock)
+
+ // Download new events from all google groups.
+ if err := g.ggroups.Sync(ctx); err != nil {
+ return err
+ }
+ // TODO: add docs
+ return nil
+}
+
// embedAll store embeddings for all new documents in the vector database.
// This must happen after all other syncs.
func (g *Gaby) embedAll(ctx context.Context) error {
