278 intermittent 503 from google workspace deletes users and its aliases in aws without retrying (#279)

Description of changes:
Add retries, to compensate for the increased frequency of 503 errors from the directory api.

By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.

Author: ChrisPates

internal/google/client.go

@@ -38,6 +38,11 @@ type client struct {
 	service *admin.Service
 }
 
+const (
+	// Maximum number of retries for admin api
+	MaxRetries = 5
+)
+
 // NewClient creates a new client for Google's Admin API
 func NewClient(ctx context.Context, adminEmail string, serviceAccountKey []byte) (Client, error) {
 	config, err := google.JWTConfigFromJSON(serviceAccountKey, admin.AdminDirectoryGroupReadonlyScope,
@@ -65,6 +70,19 @@ func NewClient(ctx context.Context, adminEmail string, serviceAccountKey []byte)
 
 // GetDeletedUsers will get the deleted users from the Google's Admin API.
 func (c *client) GetDeletedUsers() ([]*admin.User, error) {
+	var err error
+	var members []*admin.User
+
+	for iteration := 1; iteration < MaxRetries; iteration++ {
+		members, err = c.getDeletedUsers()
+		if err == nil {
+			return members, nil
+		}
+	}
+	return nil, err
+}
+
+func (c *client) getDeletedUsers() ([]*admin.User, error) {
 	u := make([]*admin.User, 0)
 	var err error
 
@@ -80,6 +98,19 @@ func (c *client) GetDeletedUsers() ([]*admin.User, error) {
 
 // GetGroupMembers will get the members of the group specified
 func (c *client) GetGroupMembers(g *admin.Group) ([]*admin.Member, error) {
+	var err error
+	var members []*admin.Member
+
+	for iteration := 1; iteration < MaxRetries; iteration++ {
+		members, err = c.getGroupMembers(g)
+		if err == nil {
+			return members, nil
+		}
+	}
+	return nil, err
+}
+
+func (c *client) getGroupMembers(g *admin.Group) ([]*admin.Member, error) {
 	m := make([]*admin.Member, 0)
 	var err error
 
@@ -108,12 +139,25 @@ func (c *client) GetGroupMembers(g *admin.Group) ([]*admin.Member, error) {
 //	orgName=Engineering orgTitle:Manager
 //	EmploymentData.projects:'GeneGnomes'
 func (c *client) GetUsers(query string, filter string) ([]*admin.User, error) {
+	var err error
+	var users []*admin.User
+
+	for iteration := 1; iteration < MaxRetries; iteration++ {
+		users, err = c.getUsers(query, filter)
+		if err == nil {
+			return users, nil
+		}
+	}
+	return nil, err
+}
+
+func (c *client) getUsers(query string, filter string) ([]*admin.User, error) {
 	u := make([]*admin.User, 0)
 	var err error
 
 	// If we have an empty query, return nothing.
 	if query == "" {
-		return u, err
+		return nil, nil
 	}
 
 	// If we have wildcard then fetch all users
@@ -170,6 +214,19 @@ func (c *client) GetUsers(query string, filter string) ([]*admin.User, error) {
 //	name:Admin* email:aws-*
 //	email:aws-*
 func (c *client) GetGroups(query string) ([]*admin.Group, error) {
+	var err error
+	var groups []*admin.Group
+
+	for iteration := 1; iteration < MaxRetries; iteration++ {
+		groups, err = c.getGroups(query)
+		if err == nil {
+			return groups, nil
+		}
+	}
+	return nil, err
+}
+
+func (c *client) getGroups(query string) ([]*admin.Group, error) {
 	g := make([]*admin.Group, 0)
 	var err error
 

internal/sync.go

@@ -762,7 +762,10 @@ func (s *syncGSuite) getGoogleGroupsAndUsers(queryGroups string, queryUsers stri
 		}
 
 		log.WithField("func", funcName).Debug("fetch membership")
-		membersUsers := s.getGoogleUsersInGroup(g, gUserDetailCache, gGroupDetailCache)
+		membersUsers, err := s.getGoogleUsersInGroup(g, gUserDetailCache, gGroupDetailCache)
+		if err != nil {
+			return nil, nil, nil, err
+		}
 
 		// If we've not seen the user email address before add it to the list of unique users
 		// also, we need to deduplicate the list of members.
@@ -1311,7 +1314,7 @@ func (s *syncGSuite) RemoveUserFromGroup(userID *string, groupID *string) error
 	return nil
 }
 
-func (s *syncGSuite) getGoogleUsersInGroup(group *admin.Group, userCache map[string]*admin.User, groupCache map[string]*admin.Group) []*admin.User {
+func (s *syncGSuite) getGoogleUsersInGroup(group *admin.Group, userCache map[string]*admin.User, groupCache map[string]*admin.Group) ([]*admin.User, error) {
 	funcName := "getGoogleUsersInGroup"
 	log.WithFields(log.Fields{
 		"func":  funcName,
@@ -1326,7 +1329,7 @@ func (s *syncGSuite) getGoogleUsersInGroup(group *admin.Group, userCache map[str
 			"GroupId": group.Id,
 			"Error":   err,
 		}).Error("failed retrieving membership")
-		return nil
+		return nil, err
 	}
 	membersUsers := make([]*admin.User, 0)
 
@@ -1429,7 +1432,7 @@ func (s *syncGSuite) getGoogleUsersInGroup(group *admin.Group, userCache map[str
 					"error":        err,
 					"Member.Email": m.Email,
 				}).Error("Fetching user")
-				continue
+				return nil, err
 			}
 			// Add user to the cache
 			for _, u := range googleUsers {
@@ -1479,5 +1482,5 @@ func (s *syncGSuite) getGoogleUsersInGroup(group *admin.Group, userCache map[str
 		"# Members":    len(membersUsers),
 		"membersUsers": membersUsers,
 	}).Debug("Return")
-	return membersUsers
+	return membersUsers, nil
 }