internal/overview: add overview posting functionality

Add the core functionality to post AI-generated overviews
to GitHub.

[Client.Run] determines which issues need overviews
(those that are less than 1 year old and have 50 or more comments).
For each issue, it posts an new overview comment and
links to it from the issue body. For issues that already have overview
comments, the overview issue is updated if needed.

Don't turn it on yet.

Change-Id: I3153a38c544edc16b1c2685fb91e029f7ede1e92
Reviewed-on: https://go-review.googlesource.com/c/oscar/+/638275
Reviewed-by: Hyang-Ah Hana Kim <[email protected]>
LUCI-TryBot-Result: Go LUCI <[email protected]>
Reviewed-by: Jonathan Amsterdam <[email protected]>

internal/gaby/main.go

@@ -185,7 +185,14 @@ func main() {
 	g.embed = ai
 	g.llm = ai
 	g.llmapp = llmapp.NewWithChecker(g.slog, ai, g.policy, g.db)
-	g.overview = overview.New(g.slog, g.db, g.github, g.llmapp, "overview", "gabyhelp")
+	ov := overview.New(g.slog, g.db, g.github, g.llmapp, "overview", "gabyhelp")
+	for _, proj := range g.githubProjects {
+		ov.EnableProject(proj)
+	}
+	if !slices.Contains(autoApprovePkgs, "overview") {
+		ov.RequireApproval()
+	}
+	g.overview = ov
 
 	cr := crawl.New(g.slog, g.db, g.http)
 	cr.Add("https://go.dev/")
@@ -290,7 +297,7 @@ func main() {
 	select {}
 }
 
-var validApprovalPkgs = []string{"commentfix", "related", "rules", "labels"}
+var validApprovalPkgs = []string{"commentfix", "related", "rules", "labels", "overview"}
 
 // parseApprovalPkgs parses a comma-separated list of package names,
 // checking that the packages are valid.
@@ -666,7 +673,7 @@ func (g *Gaby) newServer(report func(error, *http.Request)) *http.ServeMux {
 			report := actions.RunWithReport(g.ctx, g.slog, g.db)
 			_, _ = w.Write(storage.JSON(report))
 		} else {
-			http.Error(w, fmt.Sprintf("runactions: flag -enablechanges or -testactions not set"), http.StatusInternalServerError)
+			http.Error(w, "runactions: flag -enablechanges or -testactions not set", http.StatusInternalServerError)
 		}
 	})
 
@@ -813,6 +820,9 @@ func (g *Gaby) syncAndRunAll(ctx context.Context) (errs []error) {
 		check(g.labelAll(ctx))
 		check(g.postAllRules(ctx))
 		check(g.postAllBisections(ctx))
+		// TODO(tatianabradley): Uncomment once ready to enable.
+		// check(g.postAllOverviews(ctx))
+
 		// Apply all actions.
 		check(g.runActions())
 	}
@@ -929,6 +939,15 @@ func (g *Gaby) postAllRules(ctx context.Context) error {
 	return g.rulesPoster.Run(ctx)
 }
 
+func (g *Gaby) postAllOverviews(ctx context.Context) error {
+	// Hold the lock for GitHub sync because [overview.Client.Run] can't run
+	// in parallel with a GitHub sync.
+	g.db.Lock(gabyGitHubSyncLock)
+	defer g.db.Unlock(gabyGitHubSyncLock)
+
+	return g.overview.Run(ctx)
+}
+
 func (g *Gaby) labelAll(ctx context.Context) error {
 	g.db.Lock(gabyLabelLock)
 	defer g.db.Unlock(gabyLabelLock)

internal/github/data.go

@@ -25,8 +25,16 @@ import (
 // (for example "https://github.com/golang/go/issues/12345"),
 // only consulting the database (not actual GitHub).
 func (c *Client) LookupIssueURL(url string) (*Issue, error) {
-	bad := func() (*Issue, error) {
-		return nil, fmt.Errorf("not a github URL: %q", url)
+	proj, n, err := parseIssueURL(url)
+	if err != nil {
+		return nil, err
+	}
+	return LookupIssue(c.db, proj, n)
+}
+
+func parseIssueURL(url string) (string, int64, error) {
+	bad := func() (string, int64, error) {
+		return "", 0, fmt.Errorf("not a github URL: %q", url)
 	}
 	proj, ok := strings.CutPrefix(url, "https://github.com/")
 	if !ok {
@@ -45,8 +53,39 @@ func (c *Client) LookupIssueURL(url string) (*Issue, error) {
 	if err != nil || n <= 0 {
 		return bad()
 	}
+	return proj, n, nil
+}
 
-	return LookupIssue(c.db, proj, n)
+// LookupIssueCommentURL looks up an issue comment by HTML URL
+// (for example https://github.com/golang/go/issues/12345#issuecomment-135132324),
+// only consulting the database (not actual GitHub).
+func (c *Client) LookupIssueCommentURL(url string) (*IssueComment, error) {
+	project, issue, comment, err := ParseIssueCommentURL(url)
+	if err != nil {
+		return nil, err
+	}
+	e, err := c.LookupIssueCommentEvent(project, issue, comment)
+	if err != nil {
+		return nil, err
+	}
+	return e.Typed.(*IssueComment), nil
+}
+
+// ParseIssueCommentURL returns the project, issue number and comment
+// number for a URL
+// (for example https://github.com/golang/go/issues/12345#issuecomment-135132324).
+// Note: this is the URL format stored in the [IssueComment.HTMLURL] field.
+func ParseIssueCommentURL(u string) (project string, issue, comment int64, err error) {
+	before, after, _ := strings.Cut(u, "#issuecomment-")
+	proj, i, err := parseIssueURL(before)
+	if err != nil {
+		return "", 0, 0, err
+	}
+	c, err := strconv.ParseInt(after, 10, 64)
+	if err != nil || i <= 0 {
+		return "", 0, 0, err
+	}
+	return proj, i, c, nil
 }
 
 // LookupIssue looks up an issue by project and issue number
@@ -73,6 +112,17 @@ func LookupIssues(db storage.DB, project string, issueMin, issueMax int64) iter.
 	}
 }
 
+// LookupIssueCommentEvent looks up an issue comment event by project, issue, and comment number,
+// (for example "golang/go", 12345, 135132324) only consulting the database
+// (not actual GitHub).
+func (c *Client) LookupIssueCommentEvent(project string, issue, comment int64) (*Event, error) {
+	t, ok := timed.Get(c.db, eventKind, o(project, issue, "/issues/comments", comment))
+	if !ok {
+		return nil, fmt.Errorf("github.LookupIssueCommentEvent: issue %s#%d comment %d not in database", project, issue, comment)
+	}
+	return decodeEvent(c.db, t), nil
+}
+
 // Comments returns an iterator over the comments for the issue in the db.
 func (c *Client) Comments(iss *Issue) iter.Seq[*IssueComment] {
 	return func(yield func(*IssueComment) bool) {
@@ -86,6 +136,12 @@ func (c *Client) Comments(iss *Issue) iter.Seq[*IssueComment] {
 	}
 }
 
+// EventsByAPI returns an iterator over the events for the issue in the
+// Client's db with the given API.
+func (c *Client) EventsByAPI(project string, issue int64, api string) iter.Seq[*Event] {
+	return eventsByAPI(c.db, project, issue, api)
+}
+
 // eventsByAPI returns an iterator over the events for the issue in the db
 // with the given API.
 func eventsByAPI(db storage.DB, project string, issue int64, api string) iter.Seq[*Event] {

internal/github/data_test.go

@@ -24,3 +24,14 @@ func TestMustParseTime(t *testing.T) {
 		}
 	}
 }
+
+func TestParseIssueCommentURL(t *testing.T) {
+	u := "https://github.com/golang/go/issues/12345#issuecomment-135132324"
+	project, issue, comment, err := ParseIssueCommentURL(u)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if project != "golang/go" || issue != 12345 || comment != 135132324 {
+		t.Errorf("ParseIssueCommentURL(%s) = (%s, %d, %d) want (%s, %d, %d)", u, project, issue, comment, "golang/go", 12345, 135132324)
+	}
+}

internal/github/edit.go

@@ -36,7 +36,7 @@ func (c *Client) PostIssueComment(ctx context.Context, issue *Issue, changes *Is
 			Issue:               issue.Number,
 			IssueCommentChanges: changes.clone(),
 		})
-		return "", "", nil
+		return "test-api-url", "test-url", nil
 	}
 
 	body, err := c.post(ctx, issue.URL+"/comments", changes)