From 9c0ac96b7ed0462b76122326383a4d79ee86e8ac Mon Sep 17 00:00:00 2001 From: "adam.richards" Date: Mon, 21 Jun 2021 13:57:58 -0600 Subject: [PATCH 01/29] fix sonarqube findings --- clients/gitlab/client.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/clients/gitlab/client.go b/clients/gitlab/client.go index 94827ff..48c5c19 100644 --- a/clients/gitlab/client.go +++ b/clients/gitlab/client.go @@ -160,7 +160,7 @@ func (r Client) DeleteUser(userID int) (int, error) { // DeleteUserByUsername - convenience method func (r Client) DeleteUserByUsername(username string) (int, error) { // expect return status code of http.StatusNoContent 204 or http.StatusNotFound 404 - + logPrefix := "Completed call to DeleteUserByUsername." var opts = gogitlab.ListUsersOptions{Username: &username} users, lres, err := r.client.Users.ListUsers(&opts) if err != nil { @@ -168,15 +168,15 @@ func (r Client) DeleteUserByUsername(username string) (int, error) { } if len(users) == 0 { statusCode := http.StatusNotFound - rlog.Debug(fmt.Sprintf("Completed call to DeleteUser. Status Code: %d", statusCode)) + rlog.Debug(fmt.Sprintf("%s User Not Found. Status Code: %d", logPrefix, statusCode)) return statusCode, nil } - + res, err := r.client.Users.DeleteUser(users[0].ID) if err != nil { return res.StatusCode, err } - rlog.Debug(fmt.Sprintf("Completed call to DeleteUser. Status Code: %d", res.StatusCode)) + rlog.Debug(fmt.Sprintf("%s Status Code: %d", logPrefix, res.StatusCode)) return res.StatusCode, nil } -- GitLab From 7afa512c5b2d794f1a78426bff336ba9eb0936c5 Mon Sep 17 00:00:00 2001 From: "adam.richards" Date: Mon, 21 Jun 2021 15:44:43 -0600 Subject: [PATCH 02/29] add wait to delete group in order to insure resouce has been deleted before returning --- clients/gitlab/client.go | 95 +++++++++++++++++-- .../gitlab/api/gitlab_api_adhoc_test.go | 49 +++++----- rlog.conf | 2 +- 3 files changed, 117 insertions(+), 29 deletions(-) diff --git a/clients/gitlab/client.go b/clients/gitlab/client.go index 48c5c19..5103208 100644 --- a/clients/gitlab/client.go +++ b/clients/gitlab/client.go @@ -37,9 +37,11 @@ package gitlab import ( "fmt" + "net/http" + "time" + "github.com/romana/rlog" gogitlab "github.com/xanzy/go-gitlab" - "net/http" ) const itemsPerPage = 50 @@ -83,7 +85,7 @@ func (r Client) GetUser(userID int) (*gogitlab.User, int, error) { return nil, 0, err } - rlog.Debug(fmt.Sprintf("Completed call to GetUser userID: %d", user.ID)) + rlog.Debugf("Completed call to GetUser userID: %d", user.ID) return user, res.StatusCode, nil } @@ -191,6 +193,56 @@ func (r Client) GetGroup(groupID int) (*gogitlab.Group, int, error) { return group, res.StatusCode, nil } +// GetGroupByFullPath - +func (r Client) GetGroupByFullPath(fullPath *string) (*gogitlab.Group, int, error) { + // slow. get all groups and check for exact match to fullPath + listOptions := gogitlab.ListOptions{ + Page: 1, + PerPage: itemsPerPage, + } + + // some valid values path, name + logPrefix := "Completed call to GetGroupByFullPath." + orderBy := "path" + opts := gogitlab.ListGroupsOptions{ + ListOptions: listOptions, + OrderBy: &orderBy, + } + + group := &gogitlab.Group{} + var more = true + var found = false + + // while more and not found + for more && !found { + groups, res, err := r.client.Groups.ListGroups(&opts) + if err != nil { + return nil, 0, err + } + for x := 0; x < len(groups) && !found; x++ { + rlog.Debugf(">>>>>>>> group %s", groups[x].FullPath) + if groups[x].FullPath == *fullPath { + found = true + rlog.Debugf(">>>>>>>> FOUND group %s", groups[x].FullPath) + group = groups[x] + } + } + + if res.NextPage > 0 { + opts.ListOptions.Page = opts.ListOptions.Page + 1 + } else { + more = false + } + + } + if !found { + rlog.Debug(fmt.Sprintf("%s NOT FOUND %s", logPrefix, *fullPath)) + return nil, http.StatusNotFound, nil + } + rlog.Debug(fmt.Sprintf("%s FOUND %s", logPrefix, group.FullPath)) + return group, http.StatusFound, nil +} + // GetGroups - func (r Client) GetGroups(search *string) ([]*gogitlab.Group, error) { listOptions := gogitlab.ListOptions{ @@ -253,11 +305,33 @@ func (r Client) AddGroup(group *gogitlab.Group, parentID *int) (*gogitlab.Group, } // DeleteGroup - -func (r Client) DeleteGroup(groupID int) (int, error) { +func (r Client) DeleteGroup(groupID int, wait bool) (int, error) { + // recommended to use wait = true + // setting wait will use a loop to wait until resource has completed deletion res, err := r.client.Groups.DeleteGroup(groupID) if err != nil { - return res.StatusCode, err + return 0, err + } + rlog.Debugf("DeleteGroup status: %d", res.StatusCode) + // wait for resource to be deleted + if wait { + gone := false + retryCount := 50 + for !gone && retryCount >= 0 { + _, res, err = r.client.Groups.GetGroup(groupID) + if err != nil && res == nil { + rlog.Debugf("DeleteGroup hard error: %v", err) + return 0, err + } + if res.StatusCode == http.StatusNotFound { + gone = true + } + retryCount = retryCount -1 + rlog.Debugf("DeleteGroup wait status: %d", res.StatusCode) + time.Sleep((10 * time.Millisecond)) + } } + rlog.Debugf("Completed call to DeleteGroup status: %d", res.StatusCode) return res.StatusCode, nil } @@ -269,7 +343,7 @@ func (r Client) GetProject(projectID int) (*gogitlab.Project, error) { return nil, err } - rlog.Debug(fmt.Sprintf("Completed call to GetProject. status Code: %d", res.StatusCode)) + rlog.Debugf("Completed call to GetProject. status Code: %d", res.StatusCode) return project, nil } @@ -309,7 +383,7 @@ func (r Client) GetProjects(search *string) ([]*gogitlab.Project, error) { more = false } } - rlog.Debug(fmt.Sprintf("Completed call to GetProjectss. Records returned: %d", len(projectList))) + rlog.Debugf("Completed call to GetProjectss. Records returned: %d", len(projectList)) return projectList, nil } @@ -333,3 +407,12 @@ func (r Client) AddProject(project *gogitlab.Project, groupID int) (*gogitlab.Pr } return newProject, res.StatusCode, nil } + +// DeleteProject - +func (r Client) DeleteProject(projectID int) (int, error) { + res, err := r.client.Projects.DeleteProject(projectID) + if err != nil { + return res.StatusCode, err + } + return res.StatusCode, nil +} diff --git a/integration-tests/gitlab/api/gitlab_api_adhoc_test.go b/integration-tests/gitlab/api/gitlab_api_adhoc_test.go index 0f27134..57f3cc3 100644 --- a/integration-tests/gitlab/api/gitlab_api_adhoc_test.go +++ b/integration-tests/gitlab/api/gitlab_api_adhoc_test.go @@ -3,15 +3,19 @@ package integration import ( - "testing" "errors" + "net/http" "os" + "testing" + + gitlab "valkyrie.dso.mil/valkyrie-api/clients/gitlab" + "github.com/romana/rlog" gogitlab "github.com/xanzy/go-gitlab" ) -// getClient_AdHoc - +// getClient_AdHoc - func getClient_AdHoc() (gitlab.Client, error) { gitlabAPIURL, ok := os.LookupEnv("GITLAB_API_URL") if !ok { @@ -21,7 +25,7 @@ func getClient_AdHoc() (gitlab.Client, error) { if !ok { return gitlab.Client{}, errors.New("env variable GITLAB_API_TOKEN undefinded") } - var client = gitlab.NewClient(gitlabAPIURL, gitlabAPIToken,nil) + var client = gitlab.NewClient(gitlabAPIURL, gitlabAPIToken, nil) return client, nil } func TestClient_AdHoc_getUsers_All(t *testing.T) { @@ -37,7 +41,7 @@ func TestClient_AdHoc_getUsers_All(t *testing.T) { return } t.Logf("GetUsers %d", len(got)) - for x:=0;x>>>>>>> delete group %d", statusCode) + + } - groupName := "adam root" - groupPath := "adam-root" groupObj := gogitlab.Group{Name: groupName, Path: groupPath} - group, statusCode, err := c.AddGroup(&groupObj, nil) + group, statusCode, err = c.AddGroup(&groupObj, nil) if err != nil { t.Errorf("AddGroup error = %v", err) return @@ -173,12 +180,10 @@ func TestClient_AdHoc_addGroups_SubGroup(t *testing.T) { return } t.Logf("AddGroup: %s %d", subgroup.FullPath, statusCode) - } }) } - func TestClient_AdHoc_getProjects_ALL(t *testing.T) { t.Run("test", func(t *testing.T) { c, err := getClient_AdHoc() @@ -192,7 +197,7 @@ func TestClient_AdHoc_getProjects_ALL(t *testing.T) { return } t.Logf("GetProjects %d", len(got)) - for x:=0;x Date: Mon, 21 Jun 2021 17:31:19 -0600 Subject: [PATCH 03/29] add wait for delete user and delete group --- clients/gitlab/client.go | 61 +++++++++++++------ clients/gitlab/client_test.go | 8 +-- .../gitlab/api/gitlab_api_adhoc_test.go | 49 ++++++++++++--- 3 files changed, 87 insertions(+), 31 deletions(-) diff --git a/clients/gitlab/client.go b/clients/gitlab/client.go index 5103208..620bf0a 100644 --- a/clients/gitlab/client.go +++ b/clients/gitlab/client.go @@ -150,17 +150,40 @@ func (r Client) AddUser(user *gogitlab.User, password string) (*gogitlab.User, i } // DeleteUser - -func (r Client) DeleteUser(userID int) (int, error) { +func (r Client) DeleteUser(userID int, waitInterval int, waitCount int) (int, error) { + // waiting will be skilled if waitCount is 0 + // setting wait will use a loop to wait until resource has completed deletion res, err := r.client.Users.DeleteUser(userID) if err != nil { return res.StatusCode, err } - rlog.Debug(fmt.Sprintf("Completed call to DeleteUser. Status Code: %d", res.StatusCode)) + rlog.Debugf("DeleteUser status: %d", res.StatusCode) + // wait for resource to be deleted + if waitCount > 0 { + done := false + retryCount := waitCount + for !done && retryCount > 0 { + _, res, err = r.client.Groups.GetGroup(userID) + if err != nil && res == nil { + rlog.Warnf("DeleteUser hard error: %v", err) + return 0, err + } + if res.StatusCode == http.StatusNotFound { + done = true + } + retryCount = retryCount - 1 + rlog.Debugf("DeleteUser wait status: %d", res.StatusCode) + time.Sleep((time.Duration(waitInterval) * time.Millisecond)) + } + } + rlog.Debugf("Completed call to DeleteUser status: %d", res.StatusCode) return res.StatusCode, nil } // DeleteUserByUsername - convenience method -func (r Client) DeleteUserByUsername(username string) (int, error) { +func (r Client) DeleteUserByUsername(username string, waitInterval int, waitCount int) (int, error) { + // waiting will be skilled if waitCount is 0 + // setting wait will use a loop to wait until resource has completed deletion // expect return status code of http.StatusNoContent 204 or http.StatusNotFound 404 logPrefix := "Completed call to DeleteUserByUsername." var opts = gogitlab.ListUsersOptions{Username: &username} @@ -174,12 +197,12 @@ func (r Client) DeleteUserByUsername(username string) (int, error) { return statusCode, nil } - res, err := r.client.Users.DeleteUser(users[0].ID) + statusCode, err := r.DeleteUser(users[0].ID, waitInterval, waitCount) if err != nil { - return res.StatusCode, err + return 0, err } - rlog.Debug(fmt.Sprintf("%s Status Code: %d", logPrefix, res.StatusCode)) - return res.StatusCode, nil + rlog.Debug(fmt.Sprintf("%s Status Code: %d", logPrefix, statusCode)) + return statusCode, nil } // GetGroup - @@ -220,10 +243,10 @@ func (r Client) GetGroupByFullPath(fullPath *string) (*gogitlab.Group, int, erro return nil, 0, err } for x := 0; x < len(groups) && !found; x++ { - rlog.Debugf(">>>>>>>> group %s", groups[x].FullPath) + rlog.Debugf("GetGroupByFullPath checking group %s", groups[x].FullPath) if groups[x].FullPath == *fullPath { found = true - rlog.Debugf(">>>>>>>> FOUND group %s", groups[x].FullPath) + rlog.Debugf("GetGroupByFullPath FOUND group %s", groups[x].FullPath) group = groups[x] } } @@ -305,8 +328,8 @@ func (r Client) AddGroup(group *gogitlab.Group, parentID *int) (*gogitlab.Group, } // DeleteGroup - -func (r Client) DeleteGroup(groupID int, wait bool) (int, error) { - // recommended to use wait = true +func (r Client) DeleteGroup(groupID int, waitInterval int, waitCount int) (int, error) { + // waiting will be skilled if waitCount is 0 // setting wait will use a loop to wait until resource has completed deletion res, err := r.client.Groups.DeleteGroup(groupID) if err != nil { @@ -314,21 +337,21 @@ func (r Client) DeleteGroup(groupID int, wait bool) (int, error) { } rlog.Debugf("DeleteGroup status: %d", res.StatusCode) // wait for resource to be deleted - if wait { - gone := false - retryCount := 50 - for !gone && retryCount >= 0 { + if waitCount > 0 { + done := false + retryCount := waitCount + for !done && retryCount > 0 { _, res, err = r.client.Groups.GetGroup(groupID) if err != nil && res == nil { - rlog.Debugf("DeleteGroup hard error: %v", err) + rlog.Warnf("DeleteGroup hard error: %v", err) return 0, err } if res.StatusCode == http.StatusNotFound { - gone = true + done = true } - retryCount = retryCount -1 + retryCount = retryCount - 1 rlog.Debugf("DeleteGroup wait status: %d", res.StatusCode) - time.Sleep((10 * time.Millisecond)) + time.Sleep((time.Duration(waitInterval) * time.Millisecond)) } } rlog.Debugf("Completed call to DeleteGroup status: %d", res.StatusCode) diff --git a/clients/gitlab/client_test.go b/clients/gitlab/client_test.go index 5927ed3..9e890b6 100644 --- a/clients/gitlab/client_test.go +++ b/clients/gitlab/client_test.go @@ -282,8 +282,6 @@ func TestClient_DeleteUser(t *testing.T) { testUserID := 1 testUsername := "testusername" testUser := gogitlab.User{ID: testUserID, Username: testUsername} - testUserArray := []*gogitlab.User{} - testUserArray = append(testUserArray, &testUser) httpmock.RegisterResponder("DELETE", `=~^https://test/api/v4/users.*`, @@ -326,7 +324,7 @@ func TestClient_DeleteUser(t *testing.T) { token: tt.fields.token, apiURL: tt.fields.apiURL, } - got, err := r.DeleteUser(tt.args.userID) + got, err := r.DeleteUser(tt.args.userID, 0, 0) if (err != nil) != tt.wantErr { t.Errorf("Client.DeleteUser() error = %v, wantErr %v", err, tt.wantErr) return @@ -396,7 +394,7 @@ func TestClient_DeleteUserByUsername(t *testing.T) { token: tt.fields.token, apiURL: tt.fields.apiURL, } - got, err := r.DeleteUserByUsername(tt.args.username) + got, err := r.DeleteUserByUsername(tt.args.username, 0, 0) if (err != nil) != tt.wantErr { t.Errorf("Client.DeleteUserByUsername() error = %v, wantErr %v", err, tt.wantErr) return @@ -678,7 +676,7 @@ func TestClient_DeleteGroup(t *testing.T) { token: tt.fields.token, apiURL: tt.fields.apiURL, } - got, err := r.DeleteGroup(tt.args.groupID) + got, err := r.DeleteGroup(tt.args.groupID, 0, 0) if (err != nil) != tt.wantErr { t.Errorf("Client.DeleteGroup() error = %v, wantErr %v", err, tt.wantErr) return diff --git a/integration-tests/gitlab/api/gitlab_api_adhoc_test.go b/integration-tests/gitlab/api/gitlab_api_adhoc_test.go index 57f3cc3..f826ae0 100644 --- a/integration-tests/gitlab/api/gitlab_api_adhoc_test.go +++ b/integration-tests/gitlab/api/gitlab_api_adhoc_test.go @@ -4,11 +4,11 @@ package integration import ( "errors" + "fmt" "net/http" "os" "testing" - gitlab "valkyrie.dso.mil/valkyrie-api/clients/gitlab" "github.com/romana/rlog" @@ -93,7 +93,7 @@ func TestClient_AdHoc_deleteUser_None(t *testing.T) { return } username := "notfound" - statusCode, err := c.DeleteUserByUsername(username) + statusCode, err := c.DeleteUserByUsername(username, 1000, 20) if err != nil { t.Errorf("Client.DeleteUser() error = %v", err) return @@ -158,7 +158,8 @@ func TestClient_AdHoc_addGroups_SubGroup(t *testing.T) { if (err == nil) && (statusCode == http.StatusFound) { - statusCode, err = c.DeleteGroup(group.ID, true) + // delete and wait for resource to clear.. up to 120 seconds + statusCode, err = c.DeleteGroup(group.ID, 1000, 120) rlog.Debugf(">>>>>>>> delete group %d", statusCode) } @@ -171,16 +172,50 @@ func TestClient_AdHoc_addGroups_SubGroup(t *testing.T) { } t.Logf("AddGroup: %s %d", group.FullPath, statusCode) - groupName = "adam subgroup" - groupPath = "adam-subgroup" + groupName = "adam subgroup1" + groupPath = "adam-subgroup1" + groupObj = gogitlab.Group{Name: groupName, Path: groupPath} + subgroup1, statusCode, err := c.AddGroup(&groupObj, &group.ID) + if err != nil { + t.Errorf("AddGroup error = %v", err) + return + } + t.Logf("AddGroup: %s %d", subgroup1.FullPath, statusCode) + + groupName = "adam subgroup2" + groupPath = "adam-subgroup2" groupObj = gogitlab.Group{Name: groupName, Path: groupPath} - subgroup, statusCode, err := c.AddGroup(&groupObj, &group.ID) + subgroup2, statusCode, err := c.AddGroup(&groupObj, &group.ID) if err != nil { t.Errorf("AddGroup error = %v", err) return } - t.Logf("AddGroup: %s %d", subgroup.FullPath, statusCode) + t.Logf("AddGroup: %s %d", subgroup2.FullPath, statusCode) + for i := 1; i <= 250; i++ { + projectName := "test project " + fmt.Sprintf("%d", i) + + projectObj := gogitlab.Project{Name: projectName} + got, statusCode, err := c.AddProject(&projectObj, group.ID) + + if err != nil { + t.Errorf("Client.AddProject() error = %v %d", err, statusCode) + } else { + t.Logf("AddProject: [%s, %s, %d]", got.NameWithNamespace, got.HTTPURLToRepo, statusCode) + } + } + for i := 1; i <= 250; i++ { + projectName := "test project " + fmt.Sprintf("%d", i) + + projectObj := gogitlab.Project{Name: projectName} + got, statusCode, err := c.AddProject(&projectObj, subgroup1.ID) + + if err != nil { + t.Errorf("Client.AddProject() error = %v %d", err, statusCode) + } else { + t.Logf("AddProject: [%s, %s, %d]", got.NameWithNamespace, got.HTTPURLToRepo, statusCode) + } + } }) } -- GitLab From 15aa4645e41d829e27a93e0316f92193b06d9bcd Mon Sep 17 00:00:00 2001 From: "adam.richards" Date: Mon, 21 Jun 2021 18:48:04 -0600 Subject: [PATCH 04/29] update tests to wait for delte resources --- clients/gitlab/client.go | 6 +- clients/gitlab/client_test.go | 55 ++++++++++++++++--- .../gitlab/api/gitlab_api_adhoc_test.go | 10 ++-- .../gitlab/api/gitlab_api_test.go | 4 +- rlog.conf | 2 +- 5 files changed, 61 insertions(+), 16 deletions(-) diff --git a/clients/gitlab/client.go b/clients/gitlab/client.go index 620bf0a..4c73f70 100644 --- a/clients/gitlab/client.go +++ b/clients/gitlab/client.go @@ -162,8 +162,12 @@ func (r Client) DeleteUser(userID int, waitInterval int, waitCount int) (int, er if waitCount > 0 { done := false retryCount := waitCount + getCustomeAttributes := true + opts := gogitlab.GetUsersOptions{ + WithCustomAttributes: &getCustomeAttributes, + } for !done && retryCount > 0 { - _, res, err = r.client.Groups.GetGroup(userID) + _, res, err = r.client.Users.GetUser(userID, opts) if err != nil && res == nil { rlog.Warnf("DeleteUser hard error: %v", err) return 0, err diff --git a/clients/gitlab/client_test.go b/clients/gitlab/client_test.go index 9e890b6..d8b8bda 100644 --- a/clients/gitlab/client_test.go +++ b/clients/gitlab/client_test.go @@ -283,9 +283,29 @@ func TestClient_DeleteUser(t *testing.T) { testUsername := "testusername" testUser := gogitlab.User{ID: testUserID, Username: testUsername} + // httpmock.RegisterResponder("DELETE", + // `=~^https://test/api/v4/users.*`, + // httpmock.NewJsonResponderOrPanic(202, testUser), + // ) + + // mock to list out the articles + counter := 0 + httpmock.RegisterResponder("DELETE", `=~^https://test/api/v4/users.*`, - httpmock.NewJsonResponderOrPanic(202, testUser), + func(req *http.Request) (*http.Response, error) { + return httpmock.NewJsonResponse(201, testUser) + }, + ) + httpmock.RegisterResponder("GET", + `=~^https://test/api/v4/users.*`, + func(req *http.Request) (*http.Response, error) { + counter = counter + 1 + if counter%4 == 0 { + return httpmock.NewJsonResponse(404, testUser) + } + return httpmock.NewJsonResponse(202, testUser) + }, ) // test objects @@ -313,7 +333,7 @@ func TestClient_DeleteUser(t *testing.T) { name: "DeleteUser 1", fields: fields{client: testGitlabClient, token: testToken, apiURL: testAPIUrl}, args: args{userID: testUserID}, - want: 202, + want: 404, wantErr: false, }, } @@ -324,7 +344,7 @@ func TestClient_DeleteUser(t *testing.T) { token: tt.fields.token, apiURL: tt.fields.apiURL, } - got, err := r.DeleteUser(tt.args.userID, 0, 0) + got, err := r.DeleteUser(tt.args.userID, 1000, 10) if (err != nil) != tt.wantErr { t.Errorf("Client.DeleteUser() error = %v, wantErr %v", err, tt.wantErr) return @@ -630,13 +650,34 @@ func TestClient_DeleteGroup(t *testing.T) { testGroup2 := gogitlab.Group{ID: 2, Name: "group spongebob"} testGroupArray = append(testGroupArray, &testGroup1, &testGroup2) + // httpmock.RegisterResponder("DELETE", + // `=~^https://test/api/v4/groups.*`, + // httpmock.NewJsonResponderOrPanic(202, testGroup1), + // ) + // httpmock.RegisterResponder("GET", + // `=~^https://test/api/v4/groups.*`, + // httpmock.NewJsonResponderOrPanic(200, testGroupArray), + // ) +var counter = 0; httpmock.RegisterResponder("DELETE", `=~^https://test/api/v4/groups.*`, - httpmock.NewJsonResponderOrPanic(202, testGroup1), + func(req *http.Request) (*http.Response, error) { + // counter = counter + 1 + // if counter%4 == 0 { + // return httpmock.NewJsonResponse(404, testGroup1) + // } + return httpmock.NewJsonResponse(201, testGroup1) + }, ) httpmock.RegisterResponder("GET", `=~^https://test/api/v4/groups.*`, - httpmock.NewJsonResponderOrPanic(200, testGroupArray), + func(req *http.Request) (*http.Response, error) { + counter = counter + 1 + if counter%4 == 0 { + return httpmock.NewJsonResponse(404, testGroup1) + } + return httpmock.NewJsonResponse(202, testGroup1) + }, ) // test objects @@ -665,7 +706,7 @@ func TestClient_DeleteGroup(t *testing.T) { name: "DeleteGroup 1", fields: fields{client: testGitlabClient, token: testToken, apiURL: testAPIUrl}, args: args{groupID: testGroupID}, - want: 202, + want: 404, wantErr: false, }, } @@ -676,7 +717,7 @@ func TestClient_DeleteGroup(t *testing.T) { token: tt.fields.token, apiURL: tt.fields.apiURL, } - got, err := r.DeleteGroup(tt.args.groupID, 0, 0) + got, err := r.DeleteGroup(tt.args.groupID, 1000, 10) if (err != nil) != tt.wantErr { t.Errorf("Client.DeleteGroup() error = %v, wantErr %v", err, tt.wantErr) return diff --git a/integration-tests/gitlab/api/gitlab_api_adhoc_test.go b/integration-tests/gitlab/api/gitlab_api_adhoc_test.go index f826ae0..04cff12 100644 --- a/integration-tests/gitlab/api/gitlab_api_adhoc_test.go +++ b/integration-tests/gitlab/api/gitlab_api_adhoc_test.go @@ -93,7 +93,7 @@ func TestClient_AdHoc_deleteUser_None(t *testing.T) { return } username := "notfound" - statusCode, err := c.DeleteUserByUsername(username, 1000, 20) + statusCode, err := c.DeleteUserByUsername(username, 100, 200) if err != nil { t.Errorf("Client.DeleteUser() error = %v", err) return @@ -157,9 +157,9 @@ func TestClient_AdHoc_addGroups_SubGroup(t *testing.T) { group, statusCode, err := c.GetGroupByFullPath(&groupFullPath) if (err == nil) && (statusCode == http.StatusFound) { - + t.Logf("DeleteGroup %s. This can take minutes, set log level to DEBUG to see detailed progress.", groupFullPath) // delete and wait for resource to clear.. up to 120 seconds - statusCode, err = c.DeleteGroup(group.ID, 1000, 120) + statusCode, err = c.DeleteGroup(group.ID, 100, 1200) rlog.Debugf(">>>>>>>> delete group %d", statusCode) } @@ -192,7 +192,7 @@ func TestClient_AdHoc_addGroups_SubGroup(t *testing.T) { } t.Logf("AddGroup: %s %d", subgroup2.FullPath, statusCode) - for i := 1; i <= 250; i++ { + for i := 1; i <= 125; i++ { projectName := "test project " + fmt.Sprintf("%d", i) projectObj := gogitlab.Project{Name: projectName} @@ -204,7 +204,7 @@ func TestClient_AdHoc_addGroups_SubGroup(t *testing.T) { t.Logf("AddProject: [%s, %s, %d]", got.NameWithNamespace, got.HTTPURLToRepo, statusCode) } } - for i := 1; i <= 250; i++ { + for i := 1; i <= 125; i++ { projectName := "test project " + fmt.Sprintf("%d", i) projectObj := gogitlab.Project{Name: projectName} diff --git a/integration-tests/gitlab/api/gitlab_api_test.go b/integration-tests/gitlab/api/gitlab_api_test.go index a0a3cfd..9b0c6f1 100644 --- a/integration-tests/gitlab/api/gitlab_api_test.go +++ b/integration-tests/gitlab/api/gitlab_api_test.go @@ -96,7 +96,7 @@ func TestClient_deleteUsers(t *testing.T) { for i := 1; i <= testItemCount; i++ { username := "testuser" + fmt.Sprintf("%d", i) - statusCode, err := c.DeleteUserByUsername(username) + statusCode, err := c.DeleteUserByUsername(username, 100, 200) if err != nil { t.Errorf("Client.DeleteUser() error = %v %d", err, statusCode) @@ -178,7 +178,7 @@ func TestClient_deleteGroups(t *testing.T) { for x := 0; x < len(groupList); x++ { group := groupList[x] - statusCode, err := c.DeleteGroup(group.ID) + statusCode, err := c.DeleteGroup(group.ID, 100, 1200) if err != nil { t.Errorf("Client.DeleteGroup() error = %v %d", err, statusCode) diff --git a/rlog.conf b/rlog.conf index 23fd5be..d50c2ce 100644 --- a/rlog.conf +++ b/rlog.conf @@ -1,6 +1,6 @@ # Comment lines start with a '#' # export RLOG_CONF_FILE= -RLOG_LOG_LEVEL = DEBUG +RLOG_LOG_LEVEL = WARN RLOG_LOG_STREAM = stdout # time format - use ! to prevent override by environment variable RLOG_TIME_FORMAT="2006/01/06 15:04:05.00" -- GitLab From 5c49f6f2a9dbe913714360fd93d28fa3fa59aed3 Mon Sep 17 00:00:00 2001 From: "adam.richards" Date: Tue, 22 Jun 2021 18:26:08 -0600 Subject: [PATCH 05/29] format --- clients/gitlab/client_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clients/gitlab/client_test.go b/clients/gitlab/client_test.go index d8b8bda..4361ff2 100644 --- a/clients/gitlab/client_test.go +++ b/clients/gitlab/client_test.go @@ -290,7 +290,7 @@ func TestClient_DeleteUser(t *testing.T) { // mock to list out the articles counter := 0 - + httpmock.RegisterResponder("DELETE", `=~^https://test/api/v4/users.*`, func(req *http.Request) (*http.Response, error) { @@ -658,7 +658,7 @@ func TestClient_DeleteGroup(t *testing.T) { // `=~^https://test/api/v4/groups.*`, // httpmock.NewJsonResponderOrPanic(200, testGroupArray), // ) -var counter = 0; + var counter = 0 httpmock.RegisterResponder("DELETE", `=~^https://test/api/v4/groups.*`, func(req *http.Request) (*http.Response, error) { -- GitLab From 3183d4ceebd009dd3bd89f45bd4798ce8717d53b Mon Sep 17 00:00:00 2001 From: "adam.richards" Date: Tue, 22 Jun 2021 18:46:49 -0600 Subject: [PATCH 06/29] update docker image --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b3cd263..bedb426 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # NOTE: USING THIS IMAGE UNTIL registry1 credentials are added to IL2 runners # Build the manager binary -FROM registry.il2.dso.mil/platform-one/devops/pipeline-templates/pipeline-job/golang-builder-1.6:1.0 as builder +FROM registry.il2.dso.mil/platform-one/devops/pipeline-templates/valkyrie/golang-builder-1.6:1.0 as builder WORKDIR /workspace # Copy the Go Modules manifests -- GitLab From 3ff0aa0f28967802268da8de3b539215c29df9b0 Mon Sep 17 00:00:00 2001 From: "adam.richards" Date: Wed, 23 Jun 2021 11:15:29 -0600 Subject: [PATCH 07/29] add updates --- clients/gitlab/client.go | 122 +++++++++++++++++++++++++++++++++- clients/gitlab/client_test.go | 76 +++++++++++++++++---- 2 files changed, 183 insertions(+), 15 deletions(-) diff --git a/clients/gitlab/client.go b/clients/gitlab/client.go index 4c73f70..c834939 100644 --- a/clients/gitlab/client.go +++ b/clients/gitlab/client.go @@ -149,6 +149,22 @@ func (r Client) AddUser(user *gogitlab.User, password string) (*gogitlab.User, i return newUser, res.StatusCode, nil } +// UpdateUser - create a User object and update specific values +func (r Client) UpdateUser(user *gogitlab.User) (*gogitlab.User, int, error) { + var userID = user.ID + var opts = gogitlab.ModifyUserOptions{ + Username: &user.Username, + Email: &user.Email, + Name: &user.Name, + } + user, res, err := r.client.Users.ModifyUser(userID, &opts) + if err != nil { + return nil, 0, err + } + rlog.Debug(fmt.Sprintf("Completed call to UpdateUser. Status Code: %d", res.StatusCode)) + return user, res.StatusCode, nil +} + // DeleteUser - func (r Client) DeleteUser(userID int, waitInterval int, waitCount int) (int, error) { // waiting will be skilled if waitCount is 0 @@ -374,6 +390,56 @@ func (r Client) GetProject(projectID int) (*gogitlab.Project, error) { return project, nil } +// GetProjectByFullPath - +func (r Client) GetProjectByFullPath(fullPath *string) (*gogitlab.Project, int, error) { + // slow. get all groups and check for exact match to fullPath + listOptions := gogitlab.ListOptions{ + Page: 1, + PerPage: itemsPerPage, + } + + // some valid values path, name + logPrefix := "Completed call to GetProjectByFullPath." + orderBy := "path" + opts := gogitlab.ListProjectsOptions{ + ListOptions: listOptions, + OrderBy: &orderBy, + } + + project := &gogitlab.Project{} + var more = true + var found = false + + // while more and not found + for more && !found { + projects, res, err := r.client.Projects.ListProjects(&opts) + if err != nil { + return nil, 0, err + } + for x := 0; x < len(projects) && !found; x++ { + rlog.Debugf("GetProjectByFullPath checking project %s", projects[x].PathWithNamespace) + if projects[x].PathWithNamespace == *fullPath { + found = true + rlog.Debugf("GetProjectByFullPath FOUND project %s", projects[x].PathWithNamespace) + project = projects[x] + } + } + + if res.NextPage > 0 { + opts.ListOptions.Page = opts.ListOptions.Page + 1 + } else { + more = false + } + + } + if !found { + rlog.Debug(fmt.Sprintf("%s NOT FOUND %s", logPrefix, *fullPath)) + return nil, http.StatusNotFound, nil + } + rlog.Debug(fmt.Sprintf("%s FOUND %s", logPrefix, project.PathWithNamespace)) + return project, http.StatusFound, nil +} + // GetProjects - func (r Client) GetProjects(search *string) ([]*gogitlab.Project, error) { listOptions := gogitlab.ListOptions{ @@ -435,11 +501,61 @@ func (r Client) AddProject(project *gogitlab.Project, groupID int) (*gogitlab.Pr return newProject, res.StatusCode, nil } +// AddProject - +func (r Client) AddProject(project *gogitlab.Project, groupID int) (*gogitlab.Project, int, error) { + // force visibility to private + var visibility = gogitlab.PrivateVisibility + // copy customizable settings from argument, hard code other options as desired + var opts = gogitlab.CreateProjectOptions{ + Name: &project.Name, + Path: &project.Path, + Description: &project.Description, + NamespaceID: &groupID, + // hard coded values + Visibility: &visibility, + } + + newProject, res, err := r.client.Projects.CreateProject(&opts) + if err != nil { + return nil, res.StatusCode, err + } + return newProject, res.StatusCode, nil +} + // DeleteProject - -func (r Client) DeleteProject(projectID int) (int, error) { +func (r Client) DeleteProject(projectID int, waitInterval int, waitCount int) (int, error) { + logPrefix := "DeleteProject" + // waiting will be skipped if waitCount is 0 + // setting wait will use a loop to wait until resource has completed deletion res, err := r.client.Projects.DeleteProject(projectID) if err != nil { - return res.StatusCode, err + return 0, err + } + rlog.Debugf("% status: %d", logPrefix, res.StatusCode) + + // wait for resource to be deleted + withCustomAttributes := false + opts := gogitlab.GetProjectOptions { + WithCustomAttributes: &withCustomAttributes, + } + + if waitCount > 0 { + done := false + retryCount := waitCount + for !done && retryCount > 0 { + _, res, err = r.client.Projects.GetProject(projectID,&opts) + if err != nil && res == nil { + rlog.Warnf("%s hard error: %v", logPrefix, err) + return 0, err + } + if res.StatusCode == http.StatusNotFound { + done = true + } + retryCount = retryCount - 1 + rlog.Debugf("%s wait status: %d", logPrefix, res.StatusCode) + time.Sleep((time.Duration(waitInterval) * time.Millisecond)) + } } + rlog.Debugf("Completed %s, status: %d", logPrefix, res.StatusCode) return res.StatusCode, nil -} +} \ No newline at end of file diff --git a/clients/gitlab/client_test.go b/clients/gitlab/client_test.go index 4361ff2..276e800 100644 --- a/clients/gitlab/client_test.go +++ b/clients/gitlab/client_test.go @@ -273,6 +273,70 @@ func TestClient_AddUser(t *testing.T) { } } +func TestClient_UpdateUser(t *testing.T) { + // setup a http client for use in mocking + testHTTPClient := &http.Client{} + httpmock.ActivateNonDefault(testHTTPClient) + defer httpmock.DeactivateAndReset() + + testUser1 := gogitlab.User{ID: 1, Username: "joedirt"} + + httpmock.RegisterResponder("PUT", + `=~^https://test/api/v4/users.*`, + httpmock.NewJsonResponderOrPanic(200, testUser1), + ) + + // test objects + testAPIUrl := "https://test/api/v4/" + testToken := "token" + // create a gitlab Client object, inject http client to allow for mocking using httpmock + testGitlabClient, _ := gogitlab.NewClient(testToken, gogitlab.WithBaseURL(testAPIUrl), gogitlab.WithHTTPClient(testHTTPClient)) + + type fields struct { + client *gogitlab.Client + token string + apiURL string + } + type args struct { + user *gogitlab.User + } + tests := []struct { + name string + fields fields + args args + want *gogitlab.User + want1 int + wantErr bool + }{ + { + name: "UpdateUser 1", + fields: fields{client: testGitlabClient, token: testToken, apiURL: testAPIUrl}, + args: args{user: &testUser1}, + want: &testUser1, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := Client{ + client: tt.fields.client, + token: tt.fields.token, + apiURL: tt.fields.apiURL, + } + got, _, err := r.UpdateUser(tt.args.user) + if (err != nil) != tt.wantErr { + t.Errorf("Client.UpdateUser() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got.ID != tt.want.ID { + t.Errorf("Client.UpdateUser() got = %d, want %d", got.ID, tt.want.ID) + return + } + t.Logf("Client.UpdateUser() got = %d, want %d", got.ID, tt.want.ID) + }) + } +} + func TestClient_DeleteUser(t *testing.T) { // setup a http client for use in mocking testHTTPClient := &http.Client{} @@ -644,20 +708,8 @@ func TestClient_DeleteGroup(t *testing.T) { testHTTPClient := &http.Client{} httpmock.ActivateNonDefault(testHTTPClient) defer httpmock.DeactivateAndReset() - // empty gitlab User array - testGroupArray := []*gogitlab.Group{} testGroup1 := gogitlab.Group{ID: 1, Name: "group joedirt"} - testGroup2 := gogitlab.Group{ID: 2, Name: "group spongebob"} - testGroupArray = append(testGroupArray, &testGroup1, &testGroup2) - // httpmock.RegisterResponder("DELETE", - // `=~^https://test/api/v4/groups.*`, - // httpmock.NewJsonResponderOrPanic(202, testGroup1), - // ) - // httpmock.RegisterResponder("GET", - // `=~^https://test/api/v4/groups.*`, - // httpmock.NewJsonResponderOrPanic(200, testGroupArray), - // ) var counter = 0 httpmock.RegisterResponder("DELETE", `=~^https://test/api/v4/groups.*`, -- GitLab From 342c79e359489209bdaf6a3a8b324f73775d4bd6 Mon Sep 17 00:00:00 2001 From: "adam.richards" Date: Wed, 23 Jun 2021 13:14:33 -0600 Subject: [PATCH 08/29] snapshot work --- clients/gitlab/client.go | 161 +++++++++++++----- .../gitlab/api/gitlab_api_groups_test.go | 112 ++++++++++++ .../gitlab/api/gitlab_api_users_test.go | 156 +++++++++++++++++ 3 files changed, 385 insertions(+), 44 deletions(-) create mode 100644 integration-tests/gitlab/api/gitlab_api_groups_test.go create mode 100644 integration-tests/gitlab/api/gitlab_api_users_test.go diff --git a/clients/gitlab/client.go b/clients/gitlab/client.go index c834939..cd2adce 100644 --- a/clients/gitlab/client.go +++ b/clients/gitlab/client.go @@ -76,21 +76,24 @@ func NewClient(apiURL string, token string, httpClient *http.Client) Client { // GetUser - func (r Client) GetUser(userID int) (*gogitlab.User, int, error) { + logPrefix := "GetUser" getCustomeAttributes := true opts := gogitlab.GetUsersOptions{ WithCustomAttributes: &getCustomeAttributes, } user, res, err := r.client.Users.GetUser(userID, opts) if err != nil { + rlog.Warnf("%s error: %v", logPrefix, err) return nil, 0, err } - rlog.Debugf("Completed call to GetUser userID: %d", user.ID) + rlog.Debugf("Completed %s userID: %d", logPrefix, user.ID) return user, res.StatusCode, nil } // GetUsers - func (r Client) GetUsers(search *string) ([]*gogitlab.User, error) { + logPrefix := "GetUsers" getCustomeAttributes := true listOptions := gogitlab.ListOptions{ @@ -105,8 +108,8 @@ func (r Client) GetUsers(search *string) ([]*gogitlab.User, error) { OrderBy: &orderBy, WithCustomAttributes: &getCustomeAttributes, } - // if search defined add it to opts + // if search defined add it to opts if search != nil { opts.Search = search } @@ -116,6 +119,7 @@ func (r Client) GetUsers(search *string) ([]*gogitlab.User, error) { for more { users, res, err := r.client.Users.ListUsers(&opts) if err != nil { + rlog.Warnf("%s error: %v", logPrefix, err) return nil, err } for x := 0; x < len(users); x++ { @@ -129,12 +133,39 @@ func (r Client) GetUsers(search *string) ([]*gogitlab.User, error) { } } - rlog.Debug(fmt.Sprintf("Completed call to GetUsers. Records returned: %d", len(userList))) + rlog.Debug(fmt.Sprintf("Completed %s. Records returned: %d", logPrefix, len(userList))) return userList, nil } +// GetUsers - +func (r Client) GetUserByUsername(username *string) (*gogitlab.User, int, error) { + logPrefix := "GetUserByUsername" + getCustomeAttributes := true + var opts = gogitlab.ListUsersOptions{ + Username: username, + WithCustomAttributes: &getCustomeAttributes, + } + + users, _, err := r.client.Users.ListUsers(&opts) + if err != nil { + rlog.Warnf("%s error: %v", logPrefix, err) + return nil, 0, err + } + + if len(users) == 0 { + statusCode := http.StatusNotFound + rlog.Debugf("%s User Not Found. Status Code: %d", logPrefix, statusCode) + return nil, statusCode, nil + } + + statusCode := http.StatusFound + rlog.Debugf("Completed %s. statusCode: %d", logPrefix, statusCode) + return users[0], statusCode, nil +} + // AddUser - func (r Client) AddUser(user *gogitlab.User, password string) (*gogitlab.User, int, error) { + logPrefix := "AddUser" var opts = gogitlab.CreateUserOptions{ Username: &user.Username, Email: &user.Email, @@ -143,14 +174,16 @@ func (r Client) AddUser(user *gogitlab.User, password string) (*gogitlab.User, i } newUser, res, err := r.client.Users.CreateUser(&opts) if err != nil { - return nil, res.StatusCode, err + rlog.Warnf("%s error: %v", logPrefix, err) + return nil, 0, err } - rlog.Debug(fmt.Sprintf("Completed call to AddUser. Status Code: %d", res.StatusCode)) + rlog.Debug(fmt.Sprintf("Completed %s. Status Code: %d", logPrefix, res.StatusCode)) return newUser, res.StatusCode, nil } // UpdateUser - create a User object and update specific values func (r Client) UpdateUser(user *gogitlab.User) (*gogitlab.User, int, error) { + logPrefix := "UpdateUser" var userID = user.ID var opts = gogitlab.ModifyUserOptions{ Username: &user.Username, @@ -159,9 +192,10 @@ func (r Client) UpdateUser(user *gogitlab.User) (*gogitlab.User, int, error) { } user, res, err := r.client.Users.ModifyUser(userID, &opts) if err != nil { + rlog.Warnf("%s error: %v", logPrefix, err) return nil, 0, err } - rlog.Debug(fmt.Sprintf("Completed call to UpdateUser. Status Code: %d", res.StatusCode)) + rlog.Debug(fmt.Sprintf("Completed %s. Status Code: %d", logPrefix, res.StatusCode)) return user, res.StatusCode, nil } @@ -169,11 +203,13 @@ func (r Client) UpdateUser(user *gogitlab.User) (*gogitlab.User, int, error) { func (r Client) DeleteUser(userID int, waitInterval int, waitCount int) (int, error) { // waiting will be skilled if waitCount is 0 // setting wait will use a loop to wait until resource has completed deletion + logPrefix := "DeleteUser" res, err := r.client.Users.DeleteUser(userID) if err != nil { - return res.StatusCode, err + rlog.Warnf("%s error: %v", logPrefix, err) + return 0, err } - rlog.Debugf("DeleteUser status: %d", res.StatusCode) + rlog.Debugf("%s status: %d", logPrefix, res.StatusCode) // wait for resource to be deleted if waitCount > 0 { done := false @@ -185,18 +221,18 @@ func (r Client) DeleteUser(userID int, waitInterval int, waitCount int) (int, er for !done && retryCount > 0 { _, res, err = r.client.Users.GetUser(userID, opts) if err != nil && res == nil { - rlog.Warnf("DeleteUser hard error: %v", err) + rlog.Warnf("%s error: %v", logPrefix, err) return 0, err } if res.StatusCode == http.StatusNotFound { done = true } retryCount = retryCount - 1 - rlog.Debugf("DeleteUser wait status: %d", res.StatusCode) + rlog.Debugf("%s wait status: %d", logPrefix, res.StatusCode) time.Sleep((time.Duration(waitInterval) * time.Millisecond)) } } - rlog.Debugf("Completed call to DeleteUser status: %d", res.StatusCode) + rlog.Debugf("Completed %s status: %d", logPrefix, res.StatusCode) return res.StatusCode, nil } @@ -207,9 +243,10 @@ func (r Client) DeleteUserByUsername(username string, waitInterval int, waitCoun // expect return status code of http.StatusNoContent 204 or http.StatusNotFound 404 logPrefix := "Completed call to DeleteUserByUsername." var opts = gogitlab.ListUsersOptions{Username: &username} - users, lres, err := r.client.Users.ListUsers(&opts) + users, _, err := r.client.Users.ListUsers(&opts) if err != nil { - return lres.StatusCode, err + rlog.Warnf("%s error: %v", logPrefix, err) + return 0, err } if len(users) == 0 { statusCode := http.StatusNotFound @@ -219,6 +256,7 @@ func (r Client) DeleteUserByUsername(username string, waitInterval int, waitCoun statusCode, err := r.DeleteUser(users[0].ID, waitInterval, waitCount) if err != nil { + rlog.Warnf("%s error: %v", logPrefix, err) return 0, err } rlog.Debug(fmt.Sprintf("%s Status Code: %d", logPrefix, statusCode)) @@ -227,25 +265,26 @@ func (r Client) DeleteUserByUsername(username string, waitInterval int, waitCoun // GetGroup - func (r Client) GetGroup(groupID int) (*gogitlab.Group, int, error) { + logPrefix := "GetGroup" group, res, err := r.client.Groups.GetGroup(groupID) if err != nil { + rlog.Warnf("%s error: %v", logPrefix, err) return nil, 0, err } - rlog.Debug(fmt.Sprintf("Completed call to GetGroup. groupID: %d", group.ID)) + rlog.Debug(fmt.Sprintf("Completed %s. groupID: %d", logPrefix, group.ID)) return group, res.StatusCode, nil } // GetGroupByFullPath - func (r Client) GetGroupByFullPath(fullPath *string) (*gogitlab.Group, int, error) { // slow. get all groups and check for exact match to fullPath + logPrefix := "GetGroupByFullPath" listOptions := gogitlab.ListOptions{ Page: 1, PerPage: itemsPerPage, } - // some valid values path, name - logPrefix := "Completed call to GetGroupByFullPath." orderBy := "path" opts := gogitlab.ListGroupsOptions{ ListOptions: listOptions, @@ -260,6 +299,7 @@ func (r Client) GetGroupByFullPath(fullPath *string) (*gogitlab.Group, int, erro for more && !found { groups, res, err := r.client.Groups.ListGroups(&opts) if err != nil { + rlog.Warnf("%s error: %v", logPrefix, err) return nil, 0, err } for x := 0; x < len(groups) && !found; x++ { @@ -288,6 +328,7 @@ func (r Client) GetGroupByFullPath(fullPath *string) (*gogitlab.Group, int, erro // GetGroups - func (r Client) GetGroups(search *string) ([]*gogitlab.Group, error) { + logPrefix := "GetGroups" listOptions := gogitlab.ListOptions{ Page: 1, PerPage: itemsPerPage, @@ -309,6 +350,7 @@ func (r Client) GetGroups(search *string) ([]*gogitlab.Group, error) { for more { groups, res, err := r.client.Groups.ListGroups(&opts) if err != nil { + rlog.Warnf("%s error: %v", logPrefix, err) return nil, err } for x := 0; x < len(groups); x++ { @@ -322,12 +364,36 @@ func (r Client) GetGroups(search *string) ([]*gogitlab.Group, error) { } } - rlog.Debug(fmt.Sprintf("Completed call to GetGroups. Records returned: %d", len(groupList))) + rlog.Debugf("Completed %s. Records returned: %d", logPrefix, len(groupList)) return groupList, nil } // AddGroup - func (r Client) AddGroup(group *gogitlab.Group, parentID *int) (*gogitlab.Group, int, error) { + // https://stackoverflow.com/questions/65081356/how-do-you-create-a-project-in-a-specific-group-via-gitlab-api + // SubGroup will be created if a parentID is supplied + logPrefix := "AddGroup" + // force visibility to private + var visibility = gogitlab.PrivateVisibility + var opts = gogitlab.CreateGroupOptions{ + Name: &group.Name, + Path: &group.Path, + Description: &group.Description, + ParentID: parentID, + Visibility: &visibility, + } + + newGroup, res, err := r.client.Groups.CreateGroup(&opts) + if err != nil { + rlog.Warnf("%s error: %v", logPrefix, err) + return nil, 0, err + } + rlog.Debugf("Completed %s. statusCode: %d", logPrefix, res.StatusCode) + return newGroup, res.StatusCode, nil +} + +// AddGroup - +func (r Client) UpdateGroup(group *gogitlab.Group, parentID *int) (*gogitlab.Group, int, error) { // https://stackoverflow.com/questions/65081356/how-do-you-create-a-project-in-a-specific-group-via-gitlab-api // SubGroup will be created if a parentID is supplied // force visibility to private @@ -342,7 +408,7 @@ func (r Client) AddGroup(group *gogitlab.Group, parentID *int) (*gogitlab.Group, newGroup, res, err := r.client.Groups.CreateGroup(&opts) if err != nil { - return nil, res.StatusCode, err + return nil, 0, err } return newGroup, res.StatusCode, nil } @@ -351,11 +417,14 @@ func (r Client) AddGroup(group *gogitlab.Group, parentID *int) (*gogitlab.Group, func (r Client) DeleteGroup(groupID int, waitInterval int, waitCount int) (int, error) { // waiting will be skilled if waitCount is 0 // setting wait will use a loop to wait until resource has completed deletion + logPrefix := "DeleteGroup" res, err := r.client.Groups.DeleteGroup(groupID) if err != nil { + rlog.Warnf("%s error: %v", logPrefix, err) return 0, err } - rlog.Debugf("DeleteGroup status: %d", res.StatusCode) + rlog.Debugf("%s status: %d", logPrefix, res.StatusCode) + returnStatusCode := res.StatusCode // wait for resource to be deleted if waitCount > 0 { done := false @@ -363,7 +432,7 @@ func (r Client) DeleteGroup(groupID int, waitInterval int, waitCount int) (int, for !done && retryCount > 0 { _, res, err = r.client.Groups.GetGroup(groupID) if err != nil && res == nil { - rlog.Warnf("DeleteGroup hard error: %v", err) + rlog.Warnf("%s error: %v", logPrefix, err) return 0, err } if res.StatusCode == http.StatusNotFound { @@ -373,33 +442,42 @@ func (r Client) DeleteGroup(groupID int, waitInterval int, waitCount int) (int, rlog.Debugf("DeleteGroup wait status: %d", res.StatusCode) time.Sleep((time.Duration(waitInterval) * time.Millisecond)) } + if done { + returnStatusCode = http.StatusOK + } else { + returnStatusCode = http.StatusRequestTimeout + } } - rlog.Debugf("Completed call to DeleteGroup status: %d", res.StatusCode) - return res.StatusCode, nil + rlog.Debugf("Completed %s statusCode: %d", logPrefix, returnStatusCode) + return returnStatusCode, nil } // GetProject - func (r Client) GetProject(projectID int) (*gogitlab.Project, error) { + logPrefix := "GetProject" opts := gogitlab.GetProjectOptions{} project, res, err := r.client.Projects.GetProject(projectID, &opts) if err != nil { + rlog.Warnf("%s hard error: %v", logPrefix, err) return nil, err } - rlog.Debugf("Completed call to GetProject. status Code: %d", res.StatusCode) + rlog.Debugf("Completed %s. status Code: %d", logPrefix, res.StatusCode) return project, nil } // GetProjectByFullPath - func (r Client) GetProjectByFullPath(fullPath *string) (*gogitlab.Project, int, error) { // slow. get all groups and check for exact match to fullPath + logPrefix := "GetProjectByFullPath" + listOptions := gogitlab.ListOptions{ Page: 1, PerPage: itemsPerPage, } // some valid values path, name - logPrefix := "Completed call to GetProjectByFullPath." + orderBy := "path" opts := gogitlab.ListProjectsOptions{ ListOptions: listOptions, @@ -417,10 +495,10 @@ func (r Client) GetProjectByFullPath(fullPath *string) (*gogitlab.Project, int, return nil, 0, err } for x := 0; x < len(projects) && !found; x++ { - rlog.Debugf("GetProjectByFullPath checking project %s", projects[x].PathWithNamespace) + rlog.Debugf("%s checking %s", logPrefix, projects[x].PathWithNamespace) if projects[x].PathWithNamespace == *fullPath { found = true - rlog.Debugf("GetProjectByFullPath FOUND project %s", projects[x].PathWithNamespace) + rlog.Debugf("%s FOUND %s", logPrefix, projects[x].PathWithNamespace) project = projects[x] } } @@ -476,7 +554,7 @@ func (r Client) GetProjects(search *string) ([]*gogitlab.Project, error) { more = false } } - rlog.Debugf("Completed call to GetProjectss. Records returned: %d", len(projectList)) + rlog.Debugf("Completed call to GetProjects. Records returned: %d", len(projectList)) return projectList, nil } @@ -496,30 +574,25 @@ func (r Client) AddProject(project *gogitlab.Project, groupID int) (*gogitlab.Pr newProject, res, err := r.client.Projects.CreateProject(&opts) if err != nil { - return nil, res.StatusCode, err + return nil, 0, err } return newProject, res.StatusCode, nil } -// AddProject - -func (r Client) AddProject(project *gogitlab.Project, groupID int) (*gogitlab.Project, int, error) { - // force visibility to private - var visibility = gogitlab.PrivateVisibility +// UpdateProject - +func (r Client) UpdateProject(project *gogitlab.Project) (*gogitlab.Project, int, error) { // copy customizable settings from argument, hard code other options as desired - var opts = gogitlab.CreateProjectOptions{ + var opts = gogitlab.EditProjectOptions{ Name: &project.Name, Path: &project.Path, Description: &project.Description, - NamespaceID: &groupID, - // hard coded values - Visibility: &visibility, } - newProject, res, err := r.client.Projects.CreateProject(&opts) + project, res, err := r.client.Projects.EditProject(project.ID, &opts) if err != nil { - return nil, res.StatusCode, err + return nil, 0, err } - return newProject, res.StatusCode, nil + return project, res.StatusCode, nil } // DeleteProject - @@ -535,17 +608,17 @@ func (r Client) DeleteProject(projectID int, waitInterval int, waitCount int) (i // wait for resource to be deleted withCustomAttributes := false - opts := gogitlab.GetProjectOptions { - WithCustomAttributes: &withCustomAttributes, + opts := gogitlab.GetProjectOptions{ + WithCustomAttributes: &withCustomAttributes, } if waitCount > 0 { done := false retryCount := waitCount for !done && retryCount > 0 { - _, res, err = r.client.Projects.GetProject(projectID,&opts) + _, res, err = r.client.Projects.GetProject(projectID, &opts) if err != nil && res == nil { - rlog.Warnf("%s hard error: %v", logPrefix, err) + rlog.Warnf("%s error: %v", logPrefix, err) return 0, err } if res.StatusCode == http.StatusNotFound { @@ -558,4 +631,4 @@ func (r Client) DeleteProject(projectID int, waitInterval int, waitCount int) (i } rlog.Debugf("Completed %s, status: %d", logPrefix, res.StatusCode) return res.StatusCode, nil -} \ No newline at end of file +} diff --git a/integration-tests/gitlab/api/gitlab_api_groups_test.go b/integration-tests/gitlab/api/gitlab_api_groups_test.go new file mode 100644 index 0000000..229ca5b --- /dev/null +++ b/integration-tests/gitlab/api/gitlab_api_groups_test.go @@ -0,0 +1,112 @@ +// +build integration + +package integration + +import ( + "errors" + "fmt" + "os" + "testing" + + gogitlab "github.com/xanzy/go-gitlab" + gitlab "valkyrie.dso.mil/valkyrie-api/clients/gitlab" +) + +const groupsTestItemCount = 65 + +// getClient_Groups - +func getClient_Groups() (gitlab.Client, error) { + gitlabAPIURL, ok := os.LookupEnv("GITLAB_API_URL") + if !ok { + return gitlab.Client{}, errors.New("env variable GITLAB_API_URL undefinded") + } + gitlabAPIToken, ok := os.LookupEnv("GITLAB_API_TOKEN") + if !ok { + return gitlab.Client{}, errors.New("env variable GITLAB_API_TOKEN undefinded") + } + var client = gitlab.NewClient(gitlabAPIURL, gitlabAPIToken, nil) + return client, nil +} + + +func TestClient_GetGroups(t *testing.T) { + t.Run("test", func(t *testing.T) { + c, err := getClient_Groups() + if err != nil { + t.Errorf("Client.getClient() error = %v", err) + return + } + got, err := c.GetGroups(nil) + if err != nil { + t.Errorf("Client.GetGroups() error = %v", err) + return + } + t.Logf("GetGroups - number of groups %d", len(got)) + }) +} + +func TestClient_AddGroups(t *testing.T) { + t.Run("test", func(t *testing.T) { + c, err := getClient_Groups() + if err != nil { + t.Errorf("Client.getClient() error = %v", err) + return + } + + for i := 1; i <= groupsTestItemCount; i++ { + path := "test-group-" + fmt.Sprintf("%d", i) + name := "test group " + fmt.Sprintf("%d", i) + + groupObj := gogitlab.Group{Name: name, Path: path} + got, statusCode, err := c.AddGroup(&groupObj, nil) + + if err != nil { + t.Errorf("Client.AddGroup() error = %v %d", err, statusCode) + } else { + t.Logf("AddGroup %s %s %s %s %d", got.Name, got.Path, got.FullPath, got.WebURL, statusCode) + } + } + + }) +} + +func TestClient_GetGroups2(t *testing.T) { + t.Run("test", func(t *testing.T) { + c, err := getClient_Groups() + if err != nil { + t.Errorf("Client.getClient() error = %v", err) + return + } + got, err := c.GetGroups(nil) + if err != nil { + t.Errorf("Client.GetGroups() error = %v", err) + return + } + t.Logf("GetGroups2 - number of groups %d", len(got)) + }) +} + +func TestClient_deleteGroups(t *testing.T) { + t.Run("test", func(t *testing.T) { + c, err := getClient_Groups() + if err != nil { + t.Errorf("Client.getClient() error = %v", err) + return + } + + search := "test-group" + groupList, err := c.GetGroups(&search) + + for x := 0; x < len(groupList); x++ { + group := groupList[x] + statusCode, err := c.DeleteGroup(group.ID, 100, 1200) + + if err != nil { + t.Errorf("Client.DeleteGroup() error = %v %d", err, statusCode) + } else { + t.Logf("DeleteGroup %s %d", group.Name, statusCode) + } + } + + }) +} diff --git a/integration-tests/gitlab/api/gitlab_api_users_test.go b/integration-tests/gitlab/api/gitlab_api_users_test.go new file mode 100644 index 0000000..3cf5c2c --- /dev/null +++ b/integration-tests/gitlab/api/gitlab_api_users_test.go @@ -0,0 +1,156 @@ +// +build integration + +package integration + +import ( + "errors" + "fmt" + "os" + "testing" + + gogitlab "github.com/xanzy/go-gitlab" + gitlab "valkyrie.dso.mil/valkyrie-api/clients/gitlab" +) + +const testItemCount = 65 + +// getClient - +func getClient_Users() (gitlab.Client, error) { + gitlabAPIURL, ok := os.LookupEnv("GITLAB_API_URL") + if !ok { + return gitlab.Client{}, errors.New("env variable GITLAB_API_URL undefinded") + } + gitlabAPIToken, ok := os.LookupEnv("GITLAB_API_TOKEN") + if !ok { + return gitlab.Client{}, errors.New("env variable GITLAB_API_TOKEN undefinded") + } + var client = gitlab.NewClient(gitlabAPIURL, gitlabAPIToken, nil) + return client, nil +} + +//TestClient_getUsers - +func TestClient_getUsers(t *testing.T) { + t.Run("test", func(t *testing.T) { + c, err := getClient_Users() + if err != nil { + t.Errorf("Client.GetUsers() error = %v", err) + return + } + got, err := c.GetUsers(nil) + if err != nil { + t.Errorf("Client.GetUsers() error = %v", err) + return + } + t.Logf("GetUsers %d", len(got)) + }) +} + +// TestClient_addUsers - +func TestClient_addUsers(t *testing.T) { + t.Run("test", func(t *testing.T) { + c, err := getClient_Users() + if err != nil { + t.Errorf("Client.GetUsers() error = %v", err) + return + } + + for i := 1; i <= testItemCount; i++ { + username := "testuser" + fmt.Sprintf("%d", i) + name := "test user name " + fmt.Sprintf("%d", i) + email := username + "@email.com" + userObj := gogitlab.User{Username: username, Email: email, Name: name} + got, statusCode, err := c.AddUser(&userObj, "password") + + if err != nil { + t.Errorf("Client.AddUser() error = %v %d", err, statusCode) + } else { + t.Logf("AddUser %s %d", got.Username, statusCode) + } + } + + }) +} + +func TestClient_UpdateUser(t *testing.T) { + t.Run("test", func(t *testing.T) { + c, err := getClient_Users() + if err != nil { + t.Errorf("Client.GetUsers() error = %v", err) + return + } + // create a user + username := "testupdateuser01" + name := "test update user name 01" + email := username + "@email.com" + userObj := gogitlab.User{Username: username, Email: email, Name: name} + user, statusCode, err := c.AddUser(&userObj, "password") + t.Logf(" created user %s status code %d", username, statusCode) + + // get user + user, statusCode, err = c.GetUserByUsername(&username) + if err != nil { + t.Errorf("Client.GetUserByUsername() error = %v", err) + return + } + t.Logf("GetUserByUsername: %s", user.Username) + + // update user + testName := "Updated Username 01" + user.Name=testName + _, statusCode, err = c.UpdateUser(user) + + // get user + user, statusCode, err = c.GetUserByUsername(&username) + if err != nil { + t.Errorf("Client.GetUserByUsername() error = %v", err) + return + } + t.Logf("GetUserByUsername success: %s", user.Username) + if ( user.Name != testName) { + t.Errorf("TestClient_UpdateUser expected %s got %s", testName, user.Name) + } else { + t.Logf("TestClient_UpdateUser success. updated Name: %s for username: %s", user.Username, user.Name) + } + + // cleanup + c.DeleteUser(user.ID,0,0) + + }) +} +func TestClient_getUsersCount(t *testing.T) { + t.Run("test", func(t *testing.T) { + c, err := getClient_Users() + if err != nil { + t.Errorf("Client.getClient() error = %v", err) + return + } + got, err := c.GetUsers(nil) + if err != nil { + t.Errorf("Client.GetUsers() error = %v", err) + return + } + t.Logf("GetUsers - user count: %d", len(got)) + }) +} +func TestClient_deleteUsers(t *testing.T) { + t.Run("test", func(t *testing.T) { + c, err := getClient_Users() + if err != nil { + t.Errorf("Client.GetUsers() error = %v", err) + return + } + + for i := 1; i <= testItemCount; i++ { + username := "testuser" + fmt.Sprintf("%d", i) + statusCode, err := c.DeleteUserByUsername(username, 100, 200) + + if err != nil { + t.Errorf("Client.DeleteUser() error = %v %d", err, statusCode) + } else { + t.Logf("Deleted user %s %d", username, statusCode) + } + } + + }) +} + -- GitLab From 15b02de4dcddf293f79541a7080328422d835c0d Mon Sep 17 00:00:00 2001 From: "adam.richards" Date: Wed, 23 Jun 2021 14:25:18 -0600 Subject: [PATCH 09/29] group update --- clients/gitlab/client.go | 53 ++- clients/gitlab/client_test.go | 8 +- .../gitlab/api/gitlab_api_projects_test.go | 319 ++++++++++++++++++ 3 files changed, 362 insertions(+), 18 deletions(-) create mode 100644 integration-tests/gitlab/api/gitlab_api_projects_test.go diff --git a/clients/gitlab/client.go b/clients/gitlab/client.go index cd2adce..eed1b27 100644 --- a/clients/gitlab/client.go +++ b/clients/gitlab/client.go @@ -137,7 +137,7 @@ func (r Client) GetUsers(search *string) ([]*gogitlab.User, error) { return userList, nil } -// GetUsers - +// GetUserByUsername - func (r Client) GetUserByUsername(username *string) (*gogitlab.User, int, error) { logPrefix := "GetUserByUsername" getCustomeAttributes := true @@ -181,14 +181,13 @@ func (r Client) AddUser(user *gogitlab.User, password string) (*gogitlab.User, i return newUser, res.StatusCode, nil } -// UpdateUser - create a User object and update specific values -func (r Client) UpdateUser(user *gogitlab.User) (*gogitlab.User, int, error) { +// UpdateUser - Update existing user. Only allow update of specific values +func (r Client) UpdateUser(userID int, modifyUserOptions *gogitlab.ModifyUserOptions) (*gogitlab.User, int, error) { logPrefix := "UpdateUser" - var userID = user.ID + var opts = gogitlab.ModifyUserOptions{ - Username: &user.Username, - Email: &user.Email, - Name: &user.Name, + Email: modifyUserOptions.Email, + Name: modifyUserOptions.Name, } user, res, err := r.client.Users.ModifyUser(userID, &opts) if err != nil { @@ -210,6 +209,7 @@ func (r Client) DeleteUser(userID int, waitInterval int, waitCount int) (int, er return 0, err } rlog.Debugf("%s status: %d", logPrefix, res.StatusCode) + returnStatusCode := res.StatusCode // wait for resource to be deleted if waitCount > 0 { done := false @@ -231,9 +231,15 @@ func (r Client) DeleteUser(userID int, waitInterval int, waitCount int) (int, er rlog.Debugf("%s wait status: %d", logPrefix, res.StatusCode) time.Sleep((time.Duration(waitInterval) * time.Millisecond)) } + if done { + returnStatusCode = http.StatusOK + } else { + returnStatusCode = http.StatusRequestTimeout + } } + rlog.Debugf("Completed %s status: %d", logPrefix, res.StatusCode) - return res.StatusCode, nil + return returnStatusCode, nil } // DeleteUserByUsername - convenience method @@ -392,25 +398,42 @@ func (r Client) AddGroup(group *gogitlab.Group, parentID *int) (*gogitlab.Group, return newGroup, res.StatusCode, nil } -// AddGroup - -func (r Client) UpdateGroup(group *gogitlab.Group, parentID *int) (*gogitlab.Group, int, error) { +// AddGroupMember - +func (r Client) AddGroupMember(groupID *int, userID *int, accessLevel gogitlab.AccessLevelValue) (*gogitlab.GroupMember, int, error) { + logPrefix := "AddGroupMember" + + var opts = gogitlab.AddGroupMemberOptions{ + UserID: userID, + AccessLevel: &accessLevel, + } + + groupMember, res, err := r.client.GroupMembers.AddGroupMember(groupID, &opts) + if err != nil { + rlog.Warnf("%s error: %v", logPrefix, err) + return nil, 0, err + } + rlog.Debugf("Completed %s. statusCode: %d", logPrefix, res.StatusCode) + return groupMember, res.StatusCode, nil +} + +// UpdateGroup - +func (r Client) UpdateGroup(group *gogitlab.Group) (*gogitlab.Group, int, error) { // https://stackoverflow.com/questions/65081356/how-do-you-create-a-project-in-a-specific-group-via-gitlab-api - // SubGroup will be created if a parentID is supplied // force visibility to private var visibility = gogitlab.PrivateVisibility - var opts = gogitlab.CreateGroupOptions{ + var opts = gogitlab.UpdateGroupOptions{ Name: &group.Name, Path: &group.Path, Description: &group.Description, - ParentID: parentID, Visibility: &visibility, } - newGroup, res, err := r.client.Groups.CreateGroup(&opts) + group, res, err := r.client.Groups.UpdateGroup(group.ID, &opts) + if err != nil { return nil, 0, err } - return newGroup, res.StatusCode, nil + return group, res.StatusCode, nil } // DeleteGroup - diff --git a/clients/gitlab/client_test.go b/clients/gitlab/client_test.go index 276e800..d86a2c9 100644 --- a/clients/gitlab/client_test.go +++ b/clients/gitlab/client_test.go @@ -291,6 +291,7 @@ func TestClient_UpdateUser(t *testing.T) { testToken := "token" // create a gitlab Client object, inject http client to allow for mocking using httpmock testGitlabClient, _ := gogitlab.NewClient(testToken, gogitlab.WithBaseURL(testAPIUrl), gogitlab.WithHTTPClient(testHTTPClient)) + updatedName := "Updated Name" type fields struct { client *gogitlab.Client @@ -298,7 +299,8 @@ func TestClient_UpdateUser(t *testing.T) { apiURL string } type args struct { - user *gogitlab.User + user *gogitlab.User + modifyUserOptions gogitlab.ModifyUserOptions } tests := []struct { name string @@ -311,7 +313,7 @@ func TestClient_UpdateUser(t *testing.T) { { name: "UpdateUser 1", fields: fields{client: testGitlabClient, token: testToken, apiURL: testAPIUrl}, - args: args{user: &testUser1}, + args: args{user: &testUser1, modifyUserOptions: gogitlab.ModifyUserOptions{Name: &updatedName}}, want: &testUser1, wantErr: false, }, @@ -323,7 +325,7 @@ func TestClient_UpdateUser(t *testing.T) { token: tt.fields.token, apiURL: tt.fields.apiURL, } - got, _, err := r.UpdateUser(tt.args.user) + got, _, err := r.UpdateUser(tt.args.user.ID, &tt.args.modifyUserOptions) if (err != nil) != tt.wantErr { t.Errorf("Client.UpdateUser() error = %v, wantErr %v", err, tt.wantErr) return diff --git a/integration-tests/gitlab/api/gitlab_api_projects_test.go b/integration-tests/gitlab/api/gitlab_api_projects_test.go new file mode 100644 index 0000000..9b0c6f1 --- /dev/null +++ b/integration-tests/gitlab/api/gitlab_api_projects_test.go @@ -0,0 +1,319 @@ +// +build integration + +package integration + +import ( + "errors" + "fmt" + "os" + "testing" + + gogitlab "github.com/xanzy/go-gitlab" + gitlab "valkyrie.dso.mil/valkyrie-api/clients/gitlab" +) + +const testItemCount = 65 + +// getClient - +func getClient() (gitlab.Client, error) { + gitlabAPIURL, ok := os.LookupEnv("GITLAB_API_URL") + if !ok { + return gitlab.Client{}, errors.New("env variable GITLAB_API_URL undefinded") + } + gitlabAPIToken, ok := os.LookupEnv("GITLAB_API_TOKEN") + if !ok { + return gitlab.Client{}, errors.New("env variable GITLAB_API_TOKEN undefinded") + } + var client = gitlab.NewClient(gitlabAPIURL, gitlabAPIToken, nil) + return client, nil +} + +//TestClient_getUsers - +func TestClient_getUsers(t *testing.T) { + t.Run("test", func(t *testing.T) { + c, err := getClient() + if err != nil { + t.Errorf("Client.GetUsers() error = %v", err) + return + } + got, err := c.GetUsers(nil) + if err != nil { + t.Errorf("Client.GetUsers() error = %v", err) + return + } + t.Logf("GetUsers %d", len(got)) + }) +} + +// TestClient_addUsers - +func TestClient_addUsers(t *testing.T) { + t.Run("test", func(t *testing.T) { + c, err := getClient() + if err != nil { + t.Errorf("Client.GetUsers() error = %v", err) + return + } + + for i := 1; i <= testItemCount; i++ { + username := "testuser" + fmt.Sprintf("%d", i) + name := "test user name " + fmt.Sprintf("%d", i) + email := username + "@email.com" + userObj := gogitlab.User{Username: username, Email: email, Name: name} + got, statusCode, err := c.AddUser(&userObj, "password") + + if err != nil { + t.Errorf("Client.AddUser() error = %v %d", err, statusCode) + } else { + t.Logf("AddUser %s %d", got.Username, statusCode) + } + } + + }) +} + +func TestClient_getUsersCount(t *testing.T) { + t.Run("test", func(t *testing.T) { + c, err := getClient() + if err != nil { + t.Errorf("Client.getClient() error = %v", err) + return + } + got, err := c.GetUsers(nil) + if err != nil { + t.Errorf("Client.GetUsers() error = %v", err) + return + } + t.Logf("GetUsers - user count: %d", len(got)) + }) +} +func TestClient_deleteUsers(t *testing.T) { + t.Run("test", func(t *testing.T) { + c, err := getClient() + if err != nil { + t.Errorf("Client.GetUsers() error = %v", err) + return + } + + for i := 1; i <= testItemCount; i++ { + username := "testuser" + fmt.Sprintf("%d", i) + statusCode, err := c.DeleteUserByUsername(username, 100, 200) + + if err != nil { + t.Errorf("Client.DeleteUser() error = %v %d", err, statusCode) + } else { + t.Logf("Deleted user %s %d", username, statusCode) + } + } + + }) +} + +func TestClient_getGroupsCount(t *testing.T) { + t.Run("test", func(t *testing.T) { + c, err := getClient() + if err != nil { + t.Errorf("Client.getClient() error = %v", err) + return + } + got, err := c.GetGroups(nil) + if err != nil { + t.Errorf("Client.GetGroups() error = %v", err) + return + } + t.Logf("GetGroups - number of groups %d", len(got)) + }) +} + +func TestClient_addGroups(t *testing.T) { + t.Run("test", func(t *testing.T) { + c, err := getClient() + if err != nil { + t.Errorf("Client.getClient() error = %v", err) + return + } + + for i := 1; i <= testItemCount; i++ { + path := "test-group-" + fmt.Sprintf("%d", i) + name := "test group " + fmt.Sprintf("%d", i) + + groupObj := gogitlab.Group{Name: name, Path: path} + got, statusCode, err := c.AddGroup(&groupObj, nil) + + if err != nil { + t.Errorf("Client.AddGroup() error = %v %d", err, statusCode) + } else { + t.Logf("AddGroup %s %s %s %s %d", got.Name, got.Path, got.FullPath, got.WebURL, statusCode) + } + } + + }) +} + +func TestClient_getGroupsCount2(t *testing.T) { + t.Run("test", func(t *testing.T) { + c, err := getClient() + if err != nil { + t.Errorf("Client.getClient() error = %v", err) + return + } + got, err := c.GetGroups(nil) + if err != nil { + t.Errorf("Client.GetGroups() error = %v", err) + return + } + t.Logf("GetGroups - number of groups %d", len(got)) + }) +} + +func TestClient_deleteGroups(t *testing.T) { + t.Run("test", func(t *testing.T) { + c, err := getClient() + if err != nil { + t.Errorf("Client.getClient() error = %v", err) + return + } + + search := "test-group" + groupList, err := c.GetGroups(&search) + + for x := 0; x < len(groupList); x++ { + group := groupList[x] + statusCode, err := c.DeleteGroup(group.ID, 100, 1200) + + if err != nil { + t.Errorf("Client.DeleteGroup() error = %v %d", err, statusCode) + } else { + t.Logf("DeleteGroup %s %d", group.Name, statusCode) + } + } + + }) +} + +func TestClient_getProjectsCount(t *testing.T) { + t.Run("test", func(t *testing.T) { + c, err := getClient() + if err != nil { + t.Errorf("Client.getClient() error = %v", err) + return + } + got, err := c.GetProjects(nil) + if err != nil { + t.Errorf("Client.GetProjects() error = %v", err) + return + } + t.Logf("Projects - number of projects %d", len(got)) + }) +} + +func TestClient_addProjects1(t *testing.T) { + t.Run("test", func(t *testing.T) { + c, err := getClient() + if err != nil { + t.Errorf("Client.GetUsers() error = %v", err) + return + } + var group *gogitlab.Group + var statusCode int + + // find or create the group to hold projects + groupPath := "test-group-adam1" + groupName := "test group adam1" + groupList, err := c.GetGroups(&groupPath) + if err != nil { + t.Errorf("AddGroup error = %v", err) + return + } + + if len(groupList) == 0 { + groupObj := gogitlab.Group{Name: groupName, Path: groupPath} + group, statusCode, err = c.AddGroup(&groupObj, nil) + if err != nil { + t.Errorf("AddGroup error = %v", err) + return + } + t.Logf("AddGroup: %s %d", group.FullPath, statusCode) + } else { + group = groupList[0] + t.Logf("GetGroups: %s", group.FullPath) + } + + for i := 1; i <= testItemCount; i++ { + projectName := "test project " + fmt.Sprintf("%d", i) + + projectObj := gogitlab.Project{Name: projectName} + got, statusCode, err := c.AddProject(&projectObj, group.ID) + + if err != nil { + t.Errorf("Client.AddProject() error = %v %d", err, statusCode) + } else { + t.Logf("AddProject: [%s, %s, %d]", got.NameWithNamespace, got.HTTPURLToRepo, statusCode) + } + } + + }) +} + +func TestClient_addProjects2(t *testing.T) { + t.Run("test", func(t *testing.T) { + c, err := getClient() + if err != nil { + t.Errorf("Client.GetUsers() error = %v", err) + return + } + + var group *gogitlab.Group + var statusCode int + // find or create the group to hold projects + groupPath := "testX-group-adam2" + groupName := "testX group adam2" + groupList, err := c.GetGroups(&groupPath) + if err != nil { + t.Errorf("AddGroup error = %v", err) + return + } + + if len(groupList) == 0 { + groupObj := gogitlab.Group{Name: groupName, Path: groupPath} + group, statusCode, err = c.AddGroup(&groupObj, nil) + if err != nil { + t.Errorf("AddGroup error = %v", err) + return + } + t.Logf("AddGroup: %s %d", group.FullPath, statusCode) + } else { + group = groupList[0] + t.Logf("GetGroups: %s", group.FullPath) + } + + for i := 1; i <= testItemCount; i++ { + projectName := "test project " + fmt.Sprintf("%d", i) + + projectObj := gogitlab.Project{Name: projectName} + got, statusCode, err := c.AddProject(&projectObj, group.ID) + + if err != nil { + t.Errorf("Client.AddProject() error = %v %d", err, statusCode) + } else { + t.Logf("AddProject: [%s, %s, %d]", got.NameWithNamespace, got.HTTPURLToRepo, statusCode) + } + } + + }) +} + +func TestClient_getProjectsCount2(t *testing.T) { + t.Run("test", func(t *testing.T) { + c, err := getClient() + if err != nil { + t.Errorf("Client.getClient() error = %v", err) + return + } + got, err := c.GetProjects(nil) + if err != nil { + t.Errorf("Client.GetProjects() error = %v", err) + return + } + t.Logf("Projects - number of projects %d", len(got)) + }) +} -- GitLab From f11b28f0aa552f69b465fd8ce88854bcb77d6366 Mon Sep 17 00:00:00 2001 From: "adam.richards" Date: Wed, 23 Jun 2021 15:28:41 -0600 Subject: [PATCH 10/29] standardize all calls. --- clients/gitlab/client.go | 172 ++++++---- clients/gitlab/client_test.go | 23 +- .../gitlab/api/gitlab_api_adhoc_test.go | 6 +- .../gitlab/api/gitlab_api_projects_test.go | 319 ------------------ .../gitlab/api/gitlab_api_test.go | 319 ------------------ 5 files changed, 117 insertions(+), 722 deletions(-) delete mode 100644 integration-tests/gitlab/api/gitlab_api_projects_test.go delete mode 100644 integration-tests/gitlab/api/gitlab_api_test.go diff --git a/clients/gitlab/client.go b/clients/gitlab/client.go index eed1b27..c1294a7 100644 --- a/clients/gitlab/client.go +++ b/clients/gitlab/client.go @@ -36,7 +36,6 @@ package gitlab */ import ( - "fmt" "net/http" "time" @@ -53,6 +52,14 @@ type Client struct { apiURL string } +func processError(logPrefix string, err error) { + rlog.Warnf("%s error: %v", logPrefix, err) +} + +func processComplete(logPrefix string, statusCode int) { + rlog.Debugf("Completed %s. Status Code: %d", logPrefix, statusCode) +} + // NewClient - create new gitlab api client func NewClient(apiURL string, token string, httpClient *http.Client) Client { // apiUrl https://code.il2.dso.mil/api/v4 @@ -83,11 +90,11 @@ func (r Client) GetUser(userID int) (*gogitlab.User, int, error) { } user, res, err := r.client.Users.GetUser(userID, opts) if err != nil { - rlog.Warnf("%s error: %v", logPrefix, err) + processError(logPrefix, err) return nil, 0, err } - rlog.Debugf("Completed %s userID: %d", logPrefix, user.ID) + processComplete(logPrefix, res.StatusCode) return user, res.StatusCode, nil } @@ -119,7 +126,7 @@ func (r Client) GetUsers(search *string) ([]*gogitlab.User, error) { for more { users, res, err := r.client.Users.ListUsers(&opts) if err != nil { - rlog.Warnf("%s error: %v", logPrefix, err) + processError(logPrefix, err) return nil, err } for x := 0; x < len(users); x++ { @@ -133,7 +140,7 @@ func (r Client) GetUsers(search *string) ([]*gogitlab.User, error) { } } - rlog.Debug(fmt.Sprintf("Completed %s. Records returned: %d", logPrefix, len(userList))) + processComplete(logPrefix, http.StatusOK) return userList, nil } @@ -148,7 +155,7 @@ func (r Client) GetUserByUsername(username *string) (*gogitlab.User, int, error) users, _, err := r.client.Users.ListUsers(&opts) if err != nil { - rlog.Warnf("%s error: %v", logPrefix, err) + processError(logPrefix, err) return nil, 0, err } @@ -159,7 +166,7 @@ func (r Client) GetUserByUsername(username *string) (*gogitlab.User, int, error) } statusCode := http.StatusFound - rlog.Debugf("Completed %s. statusCode: %d", logPrefix, statusCode) + processComplete(logPrefix, statusCode) return users[0], statusCode, nil } @@ -174,10 +181,11 @@ func (r Client) AddUser(user *gogitlab.User, password string) (*gogitlab.User, i } newUser, res, err := r.client.Users.CreateUser(&opts) if err != nil { - rlog.Warnf("%s error: %v", logPrefix, err) + processError(logPrefix, err) + return nil, 0, err } - rlog.Debug(fmt.Sprintf("Completed %s. Status Code: %d", logPrefix, res.StatusCode)) + processComplete(logPrefix, res.StatusCode) return newUser, res.StatusCode, nil } @@ -191,10 +199,10 @@ func (r Client) UpdateUser(userID int, modifyUserOptions *gogitlab.ModifyUserOpt } user, res, err := r.client.Users.ModifyUser(userID, &opts) if err != nil { - rlog.Warnf("%s error: %v", logPrefix, err) + processError(logPrefix, err) return nil, 0, err } - rlog.Debug(fmt.Sprintf("Completed %s. Status Code: %d", logPrefix, res.StatusCode)) + processComplete(logPrefix, res.StatusCode) return user, res.StatusCode, nil } @@ -205,7 +213,7 @@ func (r Client) DeleteUser(userID int, waitInterval int, waitCount int) (int, er logPrefix := "DeleteUser" res, err := r.client.Users.DeleteUser(userID) if err != nil { - rlog.Warnf("%s error: %v", logPrefix, err) + processError(logPrefix, err) return 0, err } rlog.Debugf("%s status: %d", logPrefix, res.StatusCode) @@ -221,7 +229,7 @@ func (r Client) DeleteUser(userID int, waitInterval int, waitCount int) (int, er for !done && retryCount > 0 { _, res, err = r.client.Users.GetUser(userID, opts) if err != nil && res == nil { - rlog.Warnf("%s error: %v", logPrefix, err) + processError(logPrefix, err) return 0, err } if res.StatusCode == http.StatusNotFound { @@ -238,7 +246,7 @@ func (r Client) DeleteUser(userID int, waitInterval int, waitCount int) (int, er } } - rlog.Debugf("Completed %s status: %d", logPrefix, res.StatusCode) + processComplete(logPrefix, res.StatusCode) return returnStatusCode, nil } @@ -251,21 +259,24 @@ func (r Client) DeleteUserByUsername(username string, waitInterval int, waitCoun var opts = gogitlab.ListUsersOptions{Username: &username} users, _, err := r.client.Users.ListUsers(&opts) if err != nil { - rlog.Warnf("%s error: %v", logPrefix, err) + processError(logPrefix, err) return 0, err } + if len(users) == 0 { statusCode := http.StatusNotFound - rlog.Debug(fmt.Sprintf("%s User Not Found. Status Code: %d", logPrefix, statusCode)) + processComplete(logPrefix, statusCode) return statusCode, nil } statusCode, err := r.DeleteUser(users[0].ID, waitInterval, waitCount) + if err != nil { - rlog.Warnf("%s error: %v", logPrefix, err) + processError(logPrefix, err) return 0, err } - rlog.Debug(fmt.Sprintf("%s Status Code: %d", logPrefix, statusCode)) + + processComplete(logPrefix, statusCode) return statusCode, nil } @@ -274,11 +285,11 @@ func (r Client) GetGroup(groupID int) (*gogitlab.Group, int, error) { logPrefix := "GetGroup" group, res, err := r.client.Groups.GetGroup(groupID) if err != nil { - rlog.Warnf("%s error: %v", logPrefix, err) + processError(logPrefix, err) return nil, 0, err } - rlog.Debug(fmt.Sprintf("Completed %s. groupID: %d", logPrefix, group.ID)) + processComplete(logPrefix, res.StatusCode) return group, res.StatusCode, nil } @@ -305,7 +316,7 @@ func (r Client) GetGroupByFullPath(fullPath *string) (*gogitlab.Group, int, erro for more && !found { groups, res, err := r.client.Groups.ListGroups(&opts) if err != nil { - rlog.Warnf("%s error: %v", logPrefix, err) + processError(logPrefix, err) return nil, 0, err } for x := 0; x < len(groups) && !found; x++ { @@ -325,10 +336,11 @@ func (r Client) GetGroupByFullPath(fullPath *string) (*gogitlab.Group, int, erro } if !found { - rlog.Debug(fmt.Sprintf("%s NOT FOUND %s", logPrefix, *fullPath)) + processComplete(logPrefix, http.StatusNotFound) return nil, http.StatusNotFound, nil } - rlog.Debug(fmt.Sprintf("%s FOUND %s", logPrefix, group.FullPath)) + + processComplete(logPrefix, http.StatusFound) return group, http.StatusFound, nil } @@ -346,19 +358,22 @@ func (r Client) GetGroups(search *string) ([]*gogitlab.Group, error) { ListOptions: listOptions, OrderBy: &orderBy, } - // if search defined add it to opts + // if search defined add it to opts if search != nil { opts.Search = search } + groupList := []*gogitlab.Group{} var more = true for more { groups, res, err := r.client.Groups.ListGroups(&opts) + if err != nil { - rlog.Warnf("%s error: %v", logPrefix, err) + processError(logPrefix, err) return nil, err } + for x := 0; x < len(groups); x++ { groupList = append(groupList, groups[x]) } @@ -368,9 +383,9 @@ func (r Client) GetGroups(search *string) ([]*gogitlab.Group, error) { } else { more = false } - } - rlog.Debugf("Completed %s. Records returned: %d", logPrefix, len(groupList)) + + processComplete(logPrefix, http.StatusOK) return groupList, nil } @@ -391,10 +406,11 @@ func (r Client) AddGroup(group *gogitlab.Group, parentID *int) (*gogitlab.Group, newGroup, res, err := r.client.Groups.CreateGroup(&opts) if err != nil { - rlog.Warnf("%s error: %v", logPrefix, err) + processError(logPrefix, err) return nil, 0, err } - rlog.Debugf("Completed %s. statusCode: %d", logPrefix, res.StatusCode) + + processComplete(logPrefix, res.StatusCode) return newGroup, res.StatusCode, nil } @@ -409,30 +425,26 @@ func (r Client) AddGroupMember(groupID *int, userID *int, accessLevel gogitlab.A groupMember, res, err := r.client.GroupMembers.AddGroupMember(groupID, &opts) if err != nil { - rlog.Warnf("%s error: %v", logPrefix, err) + processError(logPrefix, err) return nil, 0, err } - rlog.Debugf("Completed %s. statusCode: %d", logPrefix, res.StatusCode) + + processComplete(logPrefix, res.StatusCode) return groupMember, res.StatusCode, nil } // UpdateGroup - -func (r Client) UpdateGroup(group *gogitlab.Group) (*gogitlab.Group, int, error) { - // https://stackoverflow.com/questions/65081356/how-do-you-create-a-project-in-a-specific-group-via-gitlab-api - // force visibility to private - var visibility = gogitlab.PrivateVisibility - var opts = gogitlab.UpdateGroupOptions{ - Name: &group.Name, - Path: &group.Path, - Description: &group.Description, - Visibility: &visibility, - } +func (r Client) UpdateGroup(groupID int, updateGroupOptions *gogitlab.UpdateGroupOptions) (*gogitlab.Group, int, error) { + logPrefix := "UpdateGroup" - group, res, err := r.client.Groups.UpdateGroup(group.ID, &opts) + group, res, err := r.client.Groups.UpdateGroup(groupID, updateGroupOptions) if err != nil { + processError(logPrefix, err) return nil, 0, err } + + processComplete(logPrefix, res.StatusCode) return group, res.StatusCode, nil } @@ -441,12 +453,14 @@ func (r Client) DeleteGroup(groupID int, waitInterval int, waitCount int) (int, // waiting will be skilled if waitCount is 0 // setting wait will use a loop to wait until resource has completed deletion logPrefix := "DeleteGroup" + res, err := r.client.Groups.DeleteGroup(groupID) + if err != nil { - rlog.Warnf("%s error: %v", logPrefix, err) + processError(logPrefix, err) return 0, err } - rlog.Debugf("%s status: %d", logPrefix, res.StatusCode) + returnStatusCode := res.StatusCode // wait for resource to be deleted if waitCount > 0 { @@ -455,7 +469,7 @@ func (r Client) DeleteGroup(groupID int, waitInterval int, waitCount int) (int, for !done && retryCount > 0 { _, res, err = r.client.Groups.GetGroup(groupID) if err != nil && res == nil { - rlog.Warnf("%s error: %v", logPrefix, err) + processError(logPrefix, err) return 0, err } if res.StatusCode == http.StatusNotFound { @@ -471,7 +485,7 @@ func (r Client) DeleteGroup(groupID int, waitInterval int, waitCount int) (int, returnStatusCode = http.StatusRequestTimeout } } - rlog.Debugf("Completed %s statusCode: %d", logPrefix, returnStatusCode) + processComplete(logPrefix, returnStatusCode) return returnStatusCode, nil } @@ -479,13 +493,15 @@ func (r Client) DeleteGroup(groupID int, waitInterval int, waitCount int) (int, func (r Client) GetProject(projectID int) (*gogitlab.Project, error) { logPrefix := "GetProject" opts := gogitlab.GetProjectOptions{} + project, res, err := r.client.Projects.GetProject(projectID, &opts) + if err != nil { - rlog.Warnf("%s hard error: %v", logPrefix, err) + processError(logPrefix, err) return nil, err } - rlog.Debugf("Completed %s. status Code: %d", logPrefix, res.StatusCode) + processComplete(logPrefix, res.StatusCode) return project, nil } @@ -514,9 +530,12 @@ func (r Client) GetProjectByFullPath(fullPath *string) (*gogitlab.Project, int, // while more and not found for more && !found { projects, res, err := r.client.Projects.ListProjects(&opts) + if err != nil { + processError(logPrefix, err) return nil, 0, err } + for x := 0; x < len(projects) && !found; x++ { rlog.Debugf("%s checking %s", logPrefix, projects[x].PathWithNamespace) if projects[x].PathWithNamespace == *fullPath { @@ -531,18 +550,21 @@ func (r Client) GetProjectByFullPath(fullPath *string) (*gogitlab.Project, int, } else { more = false } - } + if !found { - rlog.Debug(fmt.Sprintf("%s NOT FOUND %s", logPrefix, *fullPath)) + processComplete(logPrefix, http.StatusNotFound) return nil, http.StatusNotFound, nil } - rlog.Debug(fmt.Sprintf("%s FOUND %s", logPrefix, project.PathWithNamespace)) + + processComplete(logPrefix, http.StatusFound) return project, http.StatusFound, nil } // GetProjects - func (r Client) GetProjects(search *string) ([]*gogitlab.Project, error) { + logPrefix := "GetProjects" + listOptions := gogitlab.ListOptions{ Page: 1, PerPage: itemsPerPage, @@ -565,6 +587,7 @@ func (r Client) GetProjects(search *string) ([]*gogitlab.Project, error) { for more { projects, res, err := r.client.Projects.ListProjects(&opts) if err != nil { + processError(logPrefix, err) return nil, err } for x := 0; x < len(projects); x++ { @@ -577,19 +600,21 @@ func (r Client) GetProjects(search *string) ([]*gogitlab.Project, error) { more = false } } - rlog.Debugf("Completed call to GetProjects. Records returned: %d", len(projectList)) + + processComplete(logPrefix, http.StatusOK) return projectList, nil } // AddProject - -func (r Client) AddProject(project *gogitlab.Project, groupID int) (*gogitlab.Project, int, error) { +func (r Client) AddProject(createProjectOptions gogitlab.CreateProjectOptions, groupID int) (*gogitlab.Project, int, error) { + logPrefix := "AddProject" // force visibility to private var visibility = gogitlab.PrivateVisibility // copy customizable settings from argument, hard code other options as desired var opts = gogitlab.CreateProjectOptions{ - Name: &project.Name, - Path: &project.Path, - Description: &project.Description, + Name: createProjectOptions.Name, + Path: createProjectOptions.Path, + Description: createProjectOptions.Description, NamespaceID: &groupID, // hard coded values Visibility: &visibility, @@ -597,24 +622,31 @@ func (r Client) AddProject(project *gogitlab.Project, groupID int) (*gogitlab.Pr newProject, res, err := r.client.Projects.CreateProject(&opts) if err != nil { + processError(logPrefix, err) return nil, 0, err } + + processComplete(logPrefix, res.StatusCode) return newProject, res.StatusCode, nil } // UpdateProject - -func (r Client) UpdateProject(project *gogitlab.Project) (*gogitlab.Project, int, error) { +func (r Client) UpdateProject(projectID int, editProjectOptions gogitlab.EditProjectOptions) (*gogitlab.Project, int, error) { + logPrefix := "UpdateProject" // copy customizable settings from argument, hard code other options as desired var opts = gogitlab.EditProjectOptions{ - Name: &project.Name, - Path: &project.Path, - Description: &project.Description, + Name: editProjectOptions.Name, + Path: editProjectOptions.Path, + Description: editProjectOptions.Description, } - project, res, err := r.client.Projects.EditProject(project.ID, &opts) + project, res, err := r.client.Projects.EditProject(projectID, &opts) if err != nil { + processError(logPrefix, err) return nil, 0, err } + + processComplete(logPrefix, res.StatusCode) return project, res.StatusCode, nil } @@ -625,9 +657,11 @@ func (r Client) DeleteProject(projectID int, waitInterval int, waitCount int) (i // setting wait will use a loop to wait until resource has completed deletion res, err := r.client.Projects.DeleteProject(projectID) if err != nil { + processError(logPrefix, err) return 0, err } - rlog.Debugf("% status: %d", logPrefix, res.StatusCode) + + returnStatusCode := res.StatusCode // wait for resource to be deleted withCustomAttributes := false @@ -638,10 +672,11 @@ func (r Client) DeleteProject(projectID int, waitInterval int, waitCount int) (i if waitCount > 0 { done := false retryCount := waitCount + for !done && retryCount > 0 { _, res, err = r.client.Projects.GetProject(projectID, &opts) if err != nil && res == nil { - rlog.Warnf("%s error: %v", logPrefix, err) + processError(logPrefix, err) return 0, err } if res.StatusCode == http.StatusNotFound { @@ -651,7 +686,14 @@ func (r Client) DeleteProject(projectID int, waitInterval int, waitCount int) (i rlog.Debugf("%s wait status: %d", logPrefix, res.StatusCode) time.Sleep((time.Duration(waitInterval) * time.Millisecond)) } + + if done { + returnStatusCode = http.StatusOK + } else { + returnStatusCode = http.StatusRequestTimeout + } } - rlog.Debugf("Completed %s, status: %d", logPrefix, res.StatusCode) - return res.StatusCode, nil + + processComplete(logPrefix, returnStatusCode) + return returnStatusCode, nil } diff --git a/clients/gitlab/client_test.go b/clients/gitlab/client_test.go index d86a2c9..337ced3 100644 --- a/clients/gitlab/client_test.go +++ b/clients/gitlab/client_test.go @@ -349,20 +349,14 @@ func TestClient_DeleteUser(t *testing.T) { testUsername := "testusername" testUser := gogitlab.User{ID: testUserID, Username: testUsername} - // httpmock.RegisterResponder("DELETE", - // `=~^https://test/api/v4/users.*`, - // httpmock.NewJsonResponderOrPanic(202, testUser), - // ) - - // mock to list out the articles counter := 0 - httpmock.RegisterResponder("DELETE", `=~^https://test/api/v4/users.*`, func(req *http.Request) (*http.Response, error) { return httpmock.NewJsonResponse(201, testUser) }, ) + // setup a mock that will change response on every 4th call httpmock.RegisterResponder("GET", `=~^https://test/api/v4/users.*`, func(req *http.Request) (*http.Response, error) { @@ -399,7 +393,7 @@ func TestClient_DeleteUser(t *testing.T) { name: "DeleteUser 1", fields: fields{client: testGitlabClient, token: testToken, apiURL: testAPIUrl}, args: args{userID: testUserID}, - want: 404, + want: 200, wantErr: false, }, } @@ -716,10 +710,6 @@ func TestClient_DeleteGroup(t *testing.T) { httpmock.RegisterResponder("DELETE", `=~^https://test/api/v4/groups.*`, func(req *http.Request) (*http.Response, error) { - // counter = counter + 1 - // if counter%4 == 0 { - // return httpmock.NewJsonResponse(404, testGroup1) - // } return httpmock.NewJsonResponse(201, testGroup1) }, ) @@ -760,7 +750,7 @@ func TestClient_DeleteGroup(t *testing.T) { name: "DeleteGroup 1", fields: fields{client: testGitlabClient, token: testToken, apiURL: testAPIUrl}, args: args{groupID: testGroupID}, - want: 404, + want: 200, wantErr: false, }, } @@ -934,6 +924,7 @@ func TestClient_AddProject(t *testing.T) { testProjectID := 1 testProjectName := "testProjectName" testProject := gogitlab.Project{ID: testProjectID, Name: testProjectName} + createTestProjetOptions := gogitlab.CreateProjectOptions{Name: &testProjectName} httpmock.RegisterResponder("POST", "https://test/api/v4/projects", @@ -952,7 +943,7 @@ func TestClient_AddProject(t *testing.T) { apiURL string } type args struct { - project *gogitlab.Project + createProjectOptions *gogitlab.CreateProjectOptions groupID int } tests := []struct { @@ -966,7 +957,7 @@ func TestClient_AddProject(t *testing.T) { { name: "AddGroup 1", fields: fields{client: testGitlabClient, token: testToken, apiURL: testAPIUrl}, - args: args{project: &testProject}, + args: args{createProjectOptions: &createTestProjetOptions}, want1: 200, want: &testProject, wantErr: false, @@ -979,7 +970,7 @@ func TestClient_AddProject(t *testing.T) { token: tt.fields.token, apiURL: tt.fields.apiURL, } - got, got1, err := r.AddProject(tt.args.project, tt.args.groupID) + got, got1, err := r.AddProject(*tt.args.createProjectOptions, tt.args.groupID) if (err != nil) != tt.wantErr { t.Errorf("Client.AddProject() error = %v, wantErr %v", err, tt.wantErr) return diff --git a/integration-tests/gitlab/api/gitlab_api_adhoc_test.go b/integration-tests/gitlab/api/gitlab_api_adhoc_test.go index 04cff12..c1c340d 100644 --- a/integration-tests/gitlab/api/gitlab_api_adhoc_test.go +++ b/integration-tests/gitlab/api/gitlab_api_adhoc_test.go @@ -195,7 +195,7 @@ func TestClient_AdHoc_addGroups_SubGroup(t *testing.T) { for i := 1; i <= 125; i++ { projectName := "test project " + fmt.Sprintf("%d", i) - projectObj := gogitlab.Project{Name: projectName} + projectObj := gogitlab.CreateProjectOptions{Name: &projectName} got, statusCode, err := c.AddProject(&projectObj, group.ID) if err != nil { @@ -207,8 +207,8 @@ func TestClient_AdHoc_addGroups_SubGroup(t *testing.T) { for i := 1; i <= 125; i++ { projectName := "test project " + fmt.Sprintf("%d", i) - projectObj := gogitlab.Project{Name: projectName} - got, statusCode, err := c.AddProject(&projectObj, subgroup1.ID) + projectObj := gogitlab.CreateProjectOptions{Name: &projectName} + got, statusCode, err := c.AddProject(projectObj, subgroup1.ID) if err != nil { t.Errorf("Client.AddProject() error = %v %d", err, statusCode) diff --git a/integration-tests/gitlab/api/gitlab_api_projects_test.go b/integration-tests/gitlab/api/gitlab_api_projects_test.go deleted file mode 100644 index 9b0c6f1..0000000 --- a/integration-tests/gitlab/api/gitlab_api_projects_test.go +++ /dev/null @@ -1,319 +0,0 @@ -// +build integration - -package integration - -import ( - "errors" - "fmt" - "os" - "testing" - - gogitlab "github.com/xanzy/go-gitlab" - gitlab "valkyrie.dso.mil/valkyrie-api/clients/gitlab" -) - -const testItemCount = 65 - -// getClient - -func getClient() (gitlab.Client, error) { - gitlabAPIURL, ok := os.LookupEnv("GITLAB_API_URL") - if !ok { - return gitlab.Client{}, errors.New("env variable GITLAB_API_URL undefinded") - } - gitlabAPIToken, ok := os.LookupEnv("GITLAB_API_TOKEN") - if !ok { - return gitlab.Client{}, errors.New("env variable GITLAB_API_TOKEN undefinded") - } - var client = gitlab.NewClient(gitlabAPIURL, gitlabAPIToken, nil) - return client, nil -} - -//TestClient_getUsers - -func TestClient_getUsers(t *testing.T) { - t.Run("test", func(t *testing.T) { - c, err := getClient() - if err != nil { - t.Errorf("Client.GetUsers() error = %v", err) - return - } - got, err := c.GetUsers(nil) - if err != nil { - t.Errorf("Client.GetUsers() error = %v", err) - return - } - t.Logf("GetUsers %d", len(got)) - }) -} - -// TestClient_addUsers - -func TestClient_addUsers(t *testing.T) { - t.Run("test", func(t *testing.T) { - c, err := getClient() - if err != nil { - t.Errorf("Client.GetUsers() error = %v", err) - return - } - - for i := 1; i <= testItemCount; i++ { - username := "testuser" + fmt.Sprintf("%d", i) - name := "test user name " + fmt.Sprintf("%d", i) - email := username + "@email.com" - userObj := gogitlab.User{Username: username, Email: email, Name: name} - got, statusCode, err := c.AddUser(&userObj, "password") - - if err != nil { - t.Errorf("Client.AddUser() error = %v %d", err, statusCode) - } else { - t.Logf("AddUser %s %d", got.Username, statusCode) - } - } - - }) -} - -func TestClient_getUsersCount(t *testing.T) { - t.Run("test", func(t *testing.T) { - c, err := getClient() - if err != nil { - t.Errorf("Client.getClient() error = %v", err) - return - } - got, err := c.GetUsers(nil) - if err != nil { - t.Errorf("Client.GetUsers() error = %v", err) - return - } - t.Logf("GetUsers - user count: %d", len(got)) - }) -} -func TestClient_deleteUsers(t *testing.T) { - t.Run("test", func(t *testing.T) { - c, err := getClient() - if err != nil { - t.Errorf("Client.GetUsers() error = %v", err) - return - } - - for i := 1; i <= testItemCount; i++ { - username := "testuser" + fmt.Sprintf("%d", i) - statusCode, err := c.DeleteUserByUsername(username, 100, 200) - - if err != nil { - t.Errorf("Client.DeleteUser() error = %v %d", err, statusCode) - } else { - t.Logf("Deleted user %s %d", username, statusCode) - } - } - - }) -} - -func TestClient_getGroupsCount(t *testing.T) { - t.Run("test", func(t *testing.T) { - c, err := getClient() - if err != nil { - t.Errorf("Client.getClient() error = %v", err) - return - } - got, err := c.GetGroups(nil) - if err != nil { - t.Errorf("Client.GetGroups() error = %v", err) - return - } - t.Logf("GetGroups - number of groups %d", len(got)) - }) -} - -func TestClient_addGroups(t *testing.T) { - t.Run("test", func(t *testing.T) { - c, err := getClient() - if err != nil { - t.Errorf("Client.getClient() error = %v", err) - return - } - - for i := 1; i <= testItemCount; i++ { - path := "test-group-" + fmt.Sprintf("%d", i) - name := "test group " + fmt.Sprintf("%d", i) - - groupObj := gogitlab.Group{Name: name, Path: path} - got, statusCode, err := c.AddGroup(&groupObj, nil) - - if err != nil { - t.Errorf("Client.AddGroup() error = %v %d", err, statusCode) - } else { - t.Logf("AddGroup %s %s %s %s %d", got.Name, got.Path, got.FullPath, got.WebURL, statusCode) - } - } - - }) -} - -func TestClient_getGroupsCount2(t *testing.T) { - t.Run("test", func(t *testing.T) { - c, err := getClient() - if err != nil { - t.Errorf("Client.getClient() error = %v", err) - return - } - got, err := c.GetGroups(nil) - if err != nil { - t.Errorf("Client.GetGroups() error = %v", err) - return - } - t.Logf("GetGroups - number of groups %d", len(got)) - }) -} - -func TestClient_deleteGroups(t *testing.T) { - t.Run("test", func(t *testing.T) { - c, err := getClient() - if err != nil { - t.Errorf("Client.getClient() error = %v", err) - return - } - - search := "test-group" - groupList, err := c.GetGroups(&search) - - for x := 0; x < len(groupList); x++ { - group := groupList[x] - statusCode, err := c.DeleteGroup(group.ID, 100, 1200) - - if err != nil { - t.Errorf("Client.DeleteGroup() error = %v %d", err, statusCode) - } else { - t.Logf("DeleteGroup %s %d", group.Name, statusCode) - } - } - - }) -} - -func TestClient_getProjectsCount(t *testing.T) { - t.Run("test", func(t *testing.T) { - c, err := getClient() - if err != nil { - t.Errorf("Client.getClient() error = %v", err) - return - } - got, err := c.GetProjects(nil) - if err != nil { - t.Errorf("Client.GetProjects() error = %v", err) - return - } - t.Logf("Projects - number of projects %d", len(got)) - }) -} - -func TestClient_addProjects1(t *testing.T) { - t.Run("test", func(t *testing.T) { - c, err := getClient() - if err != nil { - t.Errorf("Client.GetUsers() error = %v", err) - return - } - var group *gogitlab.Group - var statusCode int - - // find or create the group to hold projects - groupPath := "test-group-adam1" - groupName := "test group adam1" - groupList, err := c.GetGroups(&groupPath) - if err != nil { - t.Errorf("AddGroup error = %v", err) - return - } - - if len(groupList) == 0 { - groupObj := gogitlab.Group{Name: groupName, Path: groupPath} - group, statusCode, err = c.AddGroup(&groupObj, nil) - if err != nil { - t.Errorf("AddGroup error = %v", err) - return - } - t.Logf("AddGroup: %s %d", group.FullPath, statusCode) - } else { - group = groupList[0] - t.Logf("GetGroups: %s", group.FullPath) - } - - for i := 1; i <= testItemCount; i++ { - projectName := "test project " + fmt.Sprintf("%d", i) - - projectObj := gogitlab.Project{Name: projectName} - got, statusCode, err := c.AddProject(&projectObj, group.ID) - - if err != nil { - t.Errorf("Client.AddProject() error = %v %d", err, statusCode) - } else { - t.Logf("AddProject: [%s, %s, %d]", got.NameWithNamespace, got.HTTPURLToRepo, statusCode) - } - } - - }) -} - -func TestClient_addProjects2(t *testing.T) { - t.Run("test", func(t *testing.T) { - c, err := getClient() - if err != nil { - t.Errorf("Client.GetUsers() error = %v", err) - return - } - - var group *gogitlab.Group - var statusCode int - // find or create the group to hold projects - groupPath := "testX-group-adam2" - groupName := "testX group adam2" - groupList, err := c.GetGroups(&groupPath) - if err != nil { - t.Errorf("AddGroup error = %v", err) - return - } - - if len(groupList) == 0 { - groupObj := gogitlab.Group{Name: groupName, Path: groupPath} - group, statusCode, err = c.AddGroup(&groupObj, nil) - if err != nil { - t.Errorf("AddGroup error = %v", err) - return - } - t.Logf("AddGroup: %s %d", group.FullPath, statusCode) - } else { - group = groupList[0] - t.Logf("GetGroups: %s", group.FullPath) - } - - for i := 1; i <= testItemCount; i++ { - projectName := "test project " + fmt.Sprintf("%d", i) - - projectObj := gogitlab.Project{Name: projectName} - got, statusCode, err := c.AddProject(&projectObj, group.ID) - - if err != nil { - t.Errorf("Client.AddProject() error = %v %d", err, statusCode) - } else { - t.Logf("AddProject: [%s, %s, %d]", got.NameWithNamespace, got.HTTPURLToRepo, statusCode) - } - } - - }) -} - -func TestClient_getProjectsCount2(t *testing.T) { - t.Run("test", func(t *testing.T) { - c, err := getClient() - if err != nil { - t.Errorf("Client.getClient() error = %v", err) - return - } - got, err := c.GetProjects(nil) - if err != nil { - t.Errorf("Client.GetProjects() error = %v", err) - return - } - t.Logf("Projects - number of projects %d", len(got)) - }) -} diff --git a/integration-tests/gitlab/api/gitlab_api_test.go b/integration-tests/gitlab/api/gitlab_api_test.go deleted file mode 100644 index 9b0c6f1..0000000 --- a/integration-tests/gitlab/api/gitlab_api_test.go +++ /dev/null @@ -1,319 +0,0 @@ -// +build integration - -package integration - -import ( - "errors" - "fmt" - "os" - "testing" - - gogitlab "github.com/xanzy/go-gitlab" - gitlab "valkyrie.dso.mil/valkyrie-api/clients/gitlab" -) - -const testItemCount = 65 - -// getClient - -func getClient() (gitlab.Client, error) { - gitlabAPIURL, ok := os.LookupEnv("GITLAB_API_URL") - if !ok { - return gitlab.Client{}, errors.New("env variable GITLAB_API_URL undefinded") - } - gitlabAPIToken, ok := os.LookupEnv("GITLAB_API_TOKEN") - if !ok { - return gitlab.Client{}, errors.New("env variable GITLAB_API_TOKEN undefinded") - } - var client = gitlab.NewClient(gitlabAPIURL, gitlabAPIToken, nil) - return client, nil -} - -//TestClient_getUsers - -func TestClient_getUsers(t *testing.T) { - t.Run("test", func(t *testing.T) { - c, err := getClient() - if err != nil { - t.Errorf("Client.GetUsers() error = %v", err) - return - } - got, err := c.GetUsers(nil) - if err != nil { - t.Errorf("Client.GetUsers() error = %v", err) - return - } - t.Logf("GetUsers %d", len(got)) - }) -} - -// TestClient_addUsers - -func TestClient_addUsers(t *testing.T) { - t.Run("test", func(t *testing.T) { - c, err := getClient() - if err != nil { - t.Errorf("Client.GetUsers() error = %v", err) - return - } - - for i := 1; i <= testItemCount; i++ { - username := "testuser" + fmt.Sprintf("%d", i) - name := "test user name " + fmt.Sprintf("%d", i) - email := username + "@email.com" - userObj := gogitlab.User{Username: username, Email: email, Name: name} - got, statusCode, err := c.AddUser(&userObj, "password") - - if err != nil { - t.Errorf("Client.AddUser() error = %v %d", err, statusCode) - } else { - t.Logf("AddUser %s %d", got.Username, statusCode) - } - } - - }) -} - -func TestClient_getUsersCount(t *testing.T) { - t.Run("test", func(t *testing.T) { - c, err := getClient() - if err != nil { - t.Errorf("Client.getClient() error = %v", err) - return - } - got, err := c.GetUsers(nil) - if err != nil { - t.Errorf("Client.GetUsers() error = %v", err) - return - } - t.Logf("GetUsers - user count: %d", len(got)) - }) -} -func TestClient_deleteUsers(t *testing.T) { - t.Run("test", func(t *testing.T) { - c, err := getClient() - if err != nil { - t.Errorf("Client.GetUsers() error = %v", err) - return - } - - for i := 1; i <= testItemCount; i++ { - username := "testuser" + fmt.Sprintf("%d", i) - statusCode, err := c.DeleteUserByUsername(username, 100, 200) - - if err != nil { - t.Errorf("Client.DeleteUser() error = %v %d", err, statusCode) - } else { - t.Logf("Deleted user %s %d", username, statusCode) - } - } - - }) -} - -func TestClient_getGroupsCount(t *testing.T) { - t.Run("test", func(t *testing.T) { - c, err := getClient() - if err != nil { - t.Errorf("Client.getClient() error = %v", err) - return - } - got, err := c.GetGroups(nil) - if err != nil { - t.Errorf("Client.GetGroups() error = %v", err) - return - } - t.Logf("GetGroups - number of groups %d", len(got)) - }) -} - -func TestClient_addGroups(t *testing.T) { - t.Run("test", func(t *testing.T) { - c, err := getClient() - if err != nil { - t.Errorf("Client.getClient() error = %v", err) - return - } - - for i := 1; i <= testItemCount; i++ { - path := "test-group-" + fmt.Sprintf("%d", i) - name := "test group " + fmt.Sprintf("%d", i) - - groupObj := gogitlab.Group{Name: name, Path: path} - got, statusCode, err := c.AddGroup(&groupObj, nil) - - if err != nil { - t.Errorf("Client.AddGroup() error = %v %d", err, statusCode) - } else { - t.Logf("AddGroup %s %s %s %s %d", got.Name, got.Path, got.FullPath, got.WebURL, statusCode) - } - } - - }) -} - -func TestClient_getGroupsCount2(t *testing.T) { - t.Run("test", func(t *testing.T) { - c, err := getClient() - if err != nil { - t.Errorf("Client.getClient() error = %v", err) - return - } - got, err := c.GetGroups(nil) - if err != nil { - t.Errorf("Client.GetGroups() error = %v", err) - return - } - t.Logf("GetGroups - number of groups %d", len(got)) - }) -} - -func TestClient_deleteGroups(t *testing.T) { - t.Run("test", func(t *testing.T) { - c, err := getClient() - if err != nil { - t.Errorf("Client.getClient() error = %v", err) - return - } - - search := "test-group" - groupList, err := c.GetGroups(&search) - - for x := 0; x < len(groupList); x++ { - group := groupList[x] - statusCode, err := c.DeleteGroup(group.ID, 100, 1200) - - if err != nil { - t.Errorf("Client.DeleteGroup() error = %v %d", err, statusCode) - } else { - t.Logf("DeleteGroup %s %d", group.Name, statusCode) - } - } - - }) -} - -func TestClient_getProjectsCount(t *testing.T) { - t.Run("test", func(t *testing.T) { - c, err := getClient() - if err != nil { - t.Errorf("Client.getClient() error = %v", err) - return - } - got, err := c.GetProjects(nil) - if err != nil { - t.Errorf("Client.GetProjects() error = %v", err) - return - } - t.Logf("Projects - number of projects %d", len(got)) - }) -} - -func TestClient_addProjects1(t *testing.T) { - t.Run("test", func(t *testing.T) { - c, err := getClient() - if err != nil { - t.Errorf("Client.GetUsers() error = %v", err) - return - } - var group *gogitlab.Group - var statusCode int - - // find or create the group to hold projects - groupPath := "test-group-adam1" - groupName := "test group adam1" - groupList, err := c.GetGroups(&groupPath) - if err != nil { - t.Errorf("AddGroup error = %v", err) - return - } - - if len(groupList) == 0 { - groupObj := gogitlab.Group{Name: groupName, Path: groupPath} - group, statusCode, err = c.AddGroup(&groupObj, nil) - if err != nil { - t.Errorf("AddGroup error = %v", err) - return - } - t.Logf("AddGroup: %s %d", group.FullPath, statusCode) - } else { - group = groupList[0] - t.Logf("GetGroups: %s", group.FullPath) - } - - for i := 1; i <= testItemCount; i++ { - projectName := "test project " + fmt.Sprintf("%d", i) - - projectObj := gogitlab.Project{Name: projectName} - got, statusCode, err := c.AddProject(&projectObj, group.ID) - - if err != nil { - t.Errorf("Client.AddProject() error = %v %d", err, statusCode) - } else { - t.Logf("AddProject: [%s, %s, %d]", got.NameWithNamespace, got.HTTPURLToRepo, statusCode) - } - } - - }) -} - -func TestClient_addProjects2(t *testing.T) { - t.Run("test", func(t *testing.T) { - c, err := getClient() - if err != nil { - t.Errorf("Client.GetUsers() error = %v", err) - return - } - - var group *gogitlab.Group - var statusCode int - // find or create the group to hold projects - groupPath := "testX-group-adam2" - groupName := "testX group adam2" - groupList, err := c.GetGroups(&groupPath) - if err != nil { - t.Errorf("AddGroup error = %v", err) - return - } - - if len(groupList) == 0 { - groupObj := gogitlab.Group{Name: groupName, Path: groupPath} - group, statusCode, err = c.AddGroup(&groupObj, nil) - if err != nil { - t.Errorf("AddGroup error = %v", err) - return - } - t.Logf("AddGroup: %s %d", group.FullPath, statusCode) - } else { - group = groupList[0] - t.Logf("GetGroups: %s", group.FullPath) - } - - for i := 1; i <= testItemCount; i++ { - projectName := "test project " + fmt.Sprintf("%d", i) - - projectObj := gogitlab.Project{Name: projectName} - got, statusCode, err := c.AddProject(&projectObj, group.ID) - - if err != nil { - t.Errorf("Client.AddProject() error = %v %d", err, statusCode) - } else { - t.Logf("AddProject: [%s, %s, %d]", got.NameWithNamespace, got.HTTPURLToRepo, statusCode) - } - } - - }) -} - -func TestClient_getProjectsCount2(t *testing.T) { - t.Run("test", func(t *testing.T) { - c, err := getClient() - if err != nil { - t.Errorf("Client.getClient() error = %v", err) - return - } - got, err := c.GetProjects(nil) - if err != nil { - t.Errorf("Client.GetProjects() error = %v", err) - return - } - t.Logf("Projects - number of projects %d", len(got)) - }) -} -- GitLab From a793814976118d1cbc82df168434389da3ada45f Mon Sep 17 00:00:00 2001 From: "adam.richards" Date: Wed, 23 Jun 2021 15:59:20 -0600 Subject: [PATCH 11/29] fix integration tests --- integration-tests/gitlab/api/gitlab_api_adhoc_test.go | 2 +- integration-tests/gitlab/api/gitlab_api_users_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/integration-tests/gitlab/api/gitlab_api_adhoc_test.go b/integration-tests/gitlab/api/gitlab_api_adhoc_test.go index c1c340d..b07cf68 100644 --- a/integration-tests/gitlab/api/gitlab_api_adhoc_test.go +++ b/integration-tests/gitlab/api/gitlab_api_adhoc_test.go @@ -196,7 +196,7 @@ func TestClient_AdHoc_addGroups_SubGroup(t *testing.T) { projectName := "test project " + fmt.Sprintf("%d", i) projectObj := gogitlab.CreateProjectOptions{Name: &projectName} - got, statusCode, err := c.AddProject(&projectObj, group.ID) + got, statusCode, err := c.AddProject(projectObj, group.ID) if err != nil { t.Errorf("Client.AddProject() error = %v %d", err, statusCode) diff --git a/integration-tests/gitlab/api/gitlab_api_users_test.go b/integration-tests/gitlab/api/gitlab_api_users_test.go index 3cf5c2c..28f3f74 100644 --- a/integration-tests/gitlab/api/gitlab_api_users_test.go +++ b/integration-tests/gitlab/api/gitlab_api_users_test.go @@ -96,8 +96,8 @@ func TestClient_UpdateUser(t *testing.T) { // update user testName := "Updated Username 01" - user.Name=testName - _, statusCode, err = c.UpdateUser(user) + modifyUserOptions := gogitlab.ModifyUserOptions{Name: &testName} + _, statusCode, err = c.UpdateUser(user.ID, &modifyUserOptions) // get user user, statusCode, err = c.GetUserByUsername(&username) -- GitLab From 9ff0eed5672328301fe8a6d31dad43f7e831d6b4 Mon Sep 17 00:00:00 2001 From: "adam.richards" Date: Wed, 23 Jun 2021 16:33:56 -0600 Subject: [PATCH 12/29] use gogitlab objects for create and update calls --- clients/gitlab/client.go | 55 +++++++++++++++++++++++++---------- clients/gitlab/client_test.go | 23 ++++++++------- 2 files changed, 53 insertions(+), 25 deletions(-) diff --git a/clients/gitlab/client.go b/clients/gitlab/client.go index c1294a7..42fb342 100644 --- a/clients/gitlab/client.go +++ b/clients/gitlab/client.go @@ -52,10 +52,12 @@ type Client struct { apiURL string } +// processError - helper func processError(logPrefix string, err error) { rlog.Warnf("%s error: %v", logPrefix, err) } +// processComplete - helper func processComplete(logPrefix string, statusCode int) { rlog.Debugf("Completed %s. Status Code: %d", logPrefix, statusCode) } @@ -84,11 +86,14 @@ func NewClient(apiURL string, token string, httpClient *http.Client) Client { // GetUser - func (r Client) GetUser(userID int) (*gogitlab.User, int, error) { logPrefix := "GetUser" + getCustomeAttributes := true opts := gogitlab.GetUsersOptions{ WithCustomAttributes: &getCustomeAttributes, } + user, res, err := r.client.Users.GetUser(userID, opts) + if err != nil { processError(logPrefix, err) return nil, 0, err @@ -171,20 +176,22 @@ func (r Client) GetUserByUsername(username *string) (*gogitlab.User, int, error) } // AddUser - -func (r Client) AddUser(user *gogitlab.User, password string) (*gogitlab.User, int, error) { +func (r Client) AddUser(createUserOptions *gogitlab.CreateUserOptions) (*gogitlab.User, int, error) { logPrefix := "AddUser" var opts = gogitlab.CreateUserOptions{ - Username: &user.Username, - Email: &user.Email, - Name: &user.Name, - Password: &password, + Username: createUserOptions.Username, + Email: createUserOptions.Email, + Name: createUserOptions.Name, + Password: createUserOptions.Password, } + newUser, res, err := r.client.Users.CreateUser(&opts) + if err != nil { processError(logPrefix, err) - return nil, 0, err } + processComplete(logPrefix, res.StatusCode) return newUser, res.StatusCode, nil } @@ -197,11 +204,14 @@ func (r Client) UpdateUser(userID int, modifyUserOptions *gogitlab.ModifyUserOpt Email: modifyUserOptions.Email, Name: modifyUserOptions.Name, } + user, res, err := r.client.Users.ModifyUser(userID, &opts) + if err != nil { processError(logPrefix, err) return nil, 0, err } + processComplete(logPrefix, res.StatusCode) return user, res.StatusCode, nil } @@ -211,13 +221,16 @@ func (r Client) DeleteUser(userID int, waitInterval int, waitCount int) (int, er // waiting will be skilled if waitCount is 0 // setting wait will use a loop to wait until resource has completed deletion logPrefix := "DeleteUser" + res, err := r.client.Users.DeleteUser(userID) + if err != nil { processError(logPrefix, err) return 0, err } - rlog.Debugf("%s status: %d", logPrefix, res.StatusCode) + returnStatusCode := res.StatusCode + // wait for resource to be deleted if waitCount > 0 { done := false @@ -256,8 +269,11 @@ func (r Client) DeleteUserByUsername(username string, waitInterval int, waitCoun // setting wait will use a loop to wait until resource has completed deletion // expect return status code of http.StatusNoContent 204 or http.StatusNotFound 404 logPrefix := "Completed call to DeleteUserByUsername." + var opts = gogitlab.ListUsersOptions{Username: &username} + users, _, err := r.client.Users.ListUsers(&opts) + if err != nil { processError(logPrefix, err) return 0, err @@ -283,7 +299,9 @@ func (r Client) DeleteUserByUsername(username string, waitInterval int, waitCoun // GetGroup - func (r Client) GetGroup(groupID int) (*gogitlab.Group, int, error) { logPrefix := "GetGroup" + group, res, err := r.client.Groups.GetGroup(groupID) + if err != nil { processError(logPrefix, err) return nil, 0, err @@ -297,6 +315,7 @@ func (r Client) GetGroup(groupID int) (*gogitlab.Group, int, error) { func (r Client) GetGroupByFullPath(fullPath *string) (*gogitlab.Group, int, error) { // slow. get all groups and check for exact match to fullPath logPrefix := "GetGroupByFullPath" + listOptions := gogitlab.ListOptions{ Page: 1, PerPage: itemsPerPage, @@ -335,6 +354,7 @@ func (r Client) GetGroupByFullPath(fullPath *string) (*gogitlab.Group, int, erro } } + if !found { processComplete(logPrefix, http.StatusNotFound) return nil, http.StatusNotFound, nil @@ -347,6 +367,7 @@ func (r Client) GetGroupByFullPath(fullPath *string) (*gogitlab.Group, int, erro // GetGroups - func (r Client) GetGroups(search *string) ([]*gogitlab.Group, error) { logPrefix := "GetGroups" + listOptions := gogitlab.ListOptions{ Page: 1, PerPage: itemsPerPage, @@ -365,6 +386,7 @@ func (r Client) GetGroups(search *string) ([]*gogitlab.Group, error) { } groupList := []*gogitlab.Group{} + var more = true for more { groups, res, err := r.client.Groups.ListGroups(&opts) @@ -390,21 +412,23 @@ func (r Client) GetGroups(search *string) ([]*gogitlab.Group, error) { } // AddGroup - -func (r Client) AddGroup(group *gogitlab.Group, parentID *int) (*gogitlab.Group, int, error) { +func (r Client) AddGroup(createGroupOptions gogitlab.CreateGroupOptions) (*gogitlab.Group, int, error) { // https://stackoverflow.com/questions/65081356/how-do-you-create-a-project-in-a-specific-group-via-gitlab-api // SubGroup will be created if a parentID is supplied logPrefix := "AddGroup" + // force visibility to private var visibility = gogitlab.PrivateVisibility var opts = gogitlab.CreateGroupOptions{ - Name: &group.Name, - Path: &group.Path, - Description: &group.Description, - ParentID: parentID, + Name: createGroupOptions.Name, + Path: createGroupOptions.Path, + Description: createGroupOptions.Description, + ParentID: createGroupOptions.ParentID, Visibility: &visibility, } newGroup, res, err := r.client.Groups.CreateGroup(&opts) + if err != nil { processError(logPrefix, err) return nil, 0, err @@ -490,7 +514,7 @@ func (r Client) DeleteGroup(groupID int, waitInterval int, waitCount int) (int, } // GetProject - -func (r Client) GetProject(projectID int) (*gogitlab.Project, error) { +func (r Client) GetProject(projectID int) (*gogitlab.Project, int, error) { logPrefix := "GetProject" opts := gogitlab.GetProjectOptions{} @@ -498,11 +522,11 @@ func (r Client) GetProject(projectID int) (*gogitlab.Project, error) { if err != nil { processError(logPrefix, err) - return nil, err + return nil, 0, err } processComplete(logPrefix, res.StatusCode) - return project, nil + return project, res.StatusCode, nil } // GetProjectByFullPath - @@ -621,6 +645,7 @@ func (r Client) AddProject(createProjectOptions gogitlab.CreateProjectOptions, g } newProject, res, err := r.client.Projects.CreateProject(&opts) + if err != nil { processError(logPrefix, err) return nil, 0, err diff --git a/clients/gitlab/client_test.go b/clients/gitlab/client_test.go index 337ced3..5c9e86d 100644 --- a/clients/gitlab/client_test.go +++ b/clients/gitlab/client_test.go @@ -214,7 +214,10 @@ func TestClient_AddUser(t *testing.T) { httpmock.ActivateNonDefault(testHTTPClient) defer httpmock.DeactivateAndReset() - testUser1 := gogitlab.User{ID: 1, Username: "joedirt"} + testName := "test Joe Dirt" + testUsername := "testjoedirt" + testPassword := "password" + testUser1 := gogitlab.User{ID: 1, Username: testUsername, Name: testName} httpmock.RegisterResponder("POST", "https://test/api/v4/users", @@ -233,8 +236,7 @@ func TestClient_AddUser(t *testing.T) { apiURL string } type args struct { - user *gogitlab.User - password string + createUserOptions *gogitlab.CreateUserOptions } tests := []struct { name string @@ -247,7 +249,7 @@ func TestClient_AddUser(t *testing.T) { { name: "AddUser 1", fields: fields{client: testGitlabClient, token: testToken, apiURL: testAPIUrl}, - args: args{user: &testUser1, password: "test"}, + args: args{createUserOptions: &gogitlab.CreateUserOptions{Name: &testName, Username: &testUsername, Password: &testPassword}}, want: &testUser1, wantErr: false, }, @@ -259,7 +261,7 @@ func TestClient_AddUser(t *testing.T) { token: tt.fields.token, apiURL: tt.fields.apiURL, } - got, _, err := r.AddUser(tt.args.user, tt.args.password) + got, _, err := r.AddUser(tt.args.createUserOptions) if (err != nil) != tt.wantErr { t.Errorf("Client.AddUser() error = %v, wantErr %v", err, tt.wantErr) return @@ -640,7 +642,8 @@ func TestClient_AddGroup(t *testing.T) { defer httpmock.DeactivateAndReset() // empty gitlab User array - testGroup := gogitlab.Group{ID: 1, Name: "joedirt"} + testGroupName := "test group name" + testGroup := gogitlab.Group{ID: 1, Name: testGroupName} httpmock.RegisterResponder("POST", "https://test/api/v4/groups", @@ -659,7 +662,7 @@ func TestClient_AddGroup(t *testing.T) { apiURL string } type args struct { - group *gogitlab.Group + createGroupOptions gogitlab.CreateGroupOptions } tests := []struct { name string @@ -672,7 +675,7 @@ func TestClient_AddGroup(t *testing.T) { { name: "AddGroup 1", fields: fields{client: testGitlabClient, token: testToken, apiURL: testAPIUrl}, - args: args{group: &testGroup}, + args: args{createGroupOptions: gogitlab.CreateGroupOptions{Name: &testGroupName}}, want: &testGroup, wantErr: false, }, @@ -684,7 +687,7 @@ func TestClient_AddGroup(t *testing.T) { token: tt.fields.token, apiURL: tt.fields.apiURL, } - got, _, err := r.AddGroup(tt.args.group, nil) + got, _, err := r.AddGroup(tt.args.createGroupOptions) if (err != nil) != tt.wantErr { t.Errorf("Client.AddGroup() error = %v, wantErr %v", err, tt.wantErr) return @@ -828,7 +831,7 @@ func TestClient_GetProject(t *testing.T) { token: tt.fields.token, apiURL: tt.fields.apiURL, } - got, err := r.GetProject(tt.args.projectID) + got, _, err := r.GetProject(tt.args.projectID) if (err != nil) != tt.wantErr { t.Errorf("Client.GetProject() error = %v, wantErr %v", err, tt.wantErr) return -- GitLab From 24b17d844fd2a35710acc809421e6c1afedabee8 Mon Sep 17 00:00:00 2001 From: "adam.richards" Date: Thu, 24 Jun 2021 09:31:30 -0600 Subject: [PATCH 13/29] fix Merge from master issues --- Dockerfile | 2 +- clients/gitlab/client.go | 9 +++---- clients/gitlab/client_test.go | 44 ++++++++++++++++++++++++----------- rlog.conf | 3 ++- 4 files changed, 39 insertions(+), 19 deletions(-) diff --git a/Dockerfile b/Dockerfile index b3cd263..bedb426 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # NOTE: USING THIS IMAGE UNTIL registry1 credentials are added to IL2 runners # Build the manager binary -FROM registry.il2.dso.mil/platform-one/devops/pipeline-templates/pipeline-job/golang-builder-1.6:1.0 as builder +FROM registry.il2.dso.mil/platform-one/devops/pipeline-templates/valkyrie/golang-builder-1.6:1.0 as builder WORKDIR /workspace # Copy the Go Modules manifests diff --git a/clients/gitlab/client.go b/clients/gitlab/client.go index 26fd352..e849787 100644 --- a/clients/gitlab/client.go +++ b/clients/gitlab/client.go @@ -264,9 +264,10 @@ func (r Client) DeleteUser(userID int, waitInterval int, waitCount int) (int, er } // DeleteUserByUsername - convenience method -func (r Client) DeleteUserByUsername(username string) (int, error) { - // expect return status code of http.StatusNoContent 204 or http.StatusNotFound 404 - logPrefix := "Completed call to DeleteUserByUsername." +func (r Client) DeleteUserByUsername(username string, waitInterval int, waitCount int) (int, error) { + // waiting will be skilled if waitCount is 0 + // setting wait will use a loop to wait until resource has completed deletion + logPrefix := "DeleteUserByUsername" var opts = gogitlab.ListUsersOptions{Username: &username} @@ -283,7 +284,7 @@ func (r Client) DeleteUserByUsername(username string) (int, error) { return statusCode, nil } - statusCode, err := r.DeleteUser(users[0].ID, 0, 0) + statusCode, err := r.DeleteUser(users[0].ID, waitInterval, waitCount) if err != nil { processError(logPrefix, err) diff --git a/clients/gitlab/client_test.go b/clients/gitlab/client_test.go index bdb96d7..c37374f 100644 --- a/clients/gitlab/client_test.go +++ b/clients/gitlab/client_test.go @@ -236,7 +236,7 @@ func TestClient_AddUser(t *testing.T) { apiURL string } type args struct { - createUserOptions *gogitlab.CreateUserOptions + createUserOptions *gogitlab.CreateUserOptions } tests := []struct { name string @@ -350,6 +350,8 @@ func TestClient_DeleteUser(t *testing.T) { testUserID := 1 testUsername := "testusername" testUser := gogitlab.User{ID: testUserID, Username: testUsername} + testUserArray := []*gogitlab.User{} + testUserArray = append(testUserArray, &testUser) counter := 0 httpmock.RegisterResponder("DELETE", @@ -364,9 +366,9 @@ func TestClient_DeleteUser(t *testing.T) { func(req *http.Request) (*http.Response, error) { counter = counter + 1 if counter%4 == 0 { - return httpmock.NewJsonResponse(404, testUser) + return httpmock.NewJsonResponse(404, testUserArray) } - return httpmock.NewJsonResponse(202, testUser) + return httpmock.NewJsonResponse(202, testUserArray) }, ) @@ -406,7 +408,7 @@ func TestClient_DeleteUser(t *testing.T) { token: tt.fields.token, apiURL: tt.fields.apiURL, } - got, err := r.DeleteUser(tt.args.userID) + got, err := r.DeleteUser(tt.args.userID, 1000, 10) if (err != nil) != tt.wantErr { t.Errorf("Client.DeleteUser() error = %v, wantErr %v", err, tt.wantErr) return @@ -430,13 +432,24 @@ func TestClient_DeleteUserByUsername(t *testing.T) { testUserArray := []*gogitlab.User{} testUserArray = append(testUserArray, &testUser) + counter := 0 httpmock.RegisterResponder("DELETE", `=~^https://test/api/v4/users.*`, - httpmock.NewJsonResponderOrPanic(202, testUser), + func(req *http.Request) (*http.Response, error) { + return httpmock.NewJsonResponse(201, testUser) + }, ) + + // setup a mock that will change response on every 4th call httpmock.RegisterResponder("GET", `=~^https://test/api/v4/users.*`, - httpmock.NewJsonResponderOrPanic(200, testUserArray), + func(req *http.Request) (*http.Response, error) { + counter = counter + 1 + if counter%4 == 0 { + return httpmock.NewJsonResponse(404, testUserArray) + } + return httpmock.NewJsonResponse(202, testUserArray) + }, ) // test objects @@ -462,10 +475,10 @@ func TestClient_DeleteUserByUsername(t *testing.T) { wantErr bool }{ { - name: "DeleteUser 1", + name: "DeleteUserByUsername 1", fields: fields{client: testGitlabClient, token: testToken, apiURL: testAPIUrl}, args: args{username: testUsername}, - want: 202, + want: 200, wantErr: false, }, } @@ -476,7 +489,7 @@ func TestClient_DeleteUserByUsername(t *testing.T) { token: tt.fields.token, apiURL: tt.fields.apiURL, } - got, err := r.DeleteUserByUsername(tt.args.username) + got, err := r.DeleteUserByUsername(tt.args.username, 1000, 10) if (err != nil) != tt.wantErr { t.Errorf("Client.DeleteUserByUsername() error = %v, wantErr %v", err, tt.wantErr) return @@ -718,8 +731,13 @@ func TestClient_DeleteGroup(t *testing.T) { ) httpmock.RegisterResponder("GET", `=~^https://test/api/v4/groups.*`, - httpmock.NewJsonResponderOrPanic(200, testGroupArray), - ) + func(req *http.Request) (*http.Response, error) { + counter = counter + 1 + if counter%4 == 0 { + return httpmock.NewJsonResponse(404, testGroup1) + } + return httpmock.NewJsonResponse(202, testGroup1) + }) // test objects testGroupID := 1 @@ -758,7 +776,7 @@ func TestClient_DeleteGroup(t *testing.T) { token: tt.fields.token, apiURL: tt.fields.apiURL, } - got, err := r.DeleteGroup(tt.args.groupID) + got, err := r.DeleteGroup(tt.args.groupID, 1000, 10) if (err != nil) != tt.wantErr { t.Errorf("Client.DeleteGroup() error = %v, wantErr %v", err, tt.wantErr) return @@ -941,7 +959,7 @@ func TestClient_AddProject(t *testing.T) { } type args struct { createProjectOptions *gogitlab.CreateProjectOptions - groupID int + groupID int } tests := []struct { name string diff --git a/rlog.conf b/rlog.conf index 725cd1f..3af4f5c 100644 --- a/rlog.conf +++ b/rlog.conf @@ -1,8 +1,9 @@ # Comment lines start with a '#' +# any variable can be prefixed with ! to prevent override by ENV variables # export RLOG_CONF_FILE= RLOG_LOG_LEVEL = DEBUG RLOG_LOG_STREAM = stdout -# time format - use ! to prevent override by environment variable +# time format - RLOG_TIME_FORMAT="2006/01/06 15:04:05.00" # RLOG_TIME_FORMAT= UnixDate # optional list caller information -- GitLab From 6872bfec6d04bdbae68558f99844158bcb8b5c77 Mon Sep 17 00:00:00 2001 From: "adam.richards" Date: Thu, 24 Jun 2021 09:32:34 -0600 Subject: [PATCH 14/29] make fmt changes --- clients/gitlab/client.go | 4 ++-- controllers/gitlab/mocks_test.go | 8 ++++---- controllers/gitlab/project_controller.go | 6 +++--- controllers/gitlab/project_controller_test.go | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/clients/gitlab/client.go b/clients/gitlab/client.go index e849787..45e7cd9 100644 --- a/clients/gitlab/client.go +++ b/clients/gitlab/client.go @@ -644,7 +644,7 @@ func (r Client) AddProject(createProjectOptions gogitlab.CreateProjectOptions, g } newProject, res, err := r.client.Projects.CreateProject(&opts) - + if err != nil { processError(logPrefix, err) return nil, 0, err @@ -696,7 +696,7 @@ func (r Client) DeleteProject(projectID int, waitInterval int, waitCount int) (i if waitCount > 0 { done := false retryCount := waitCount - + for !done && retryCount > 0 { _, res, err = r.client.Projects.GetProject(projectID, &opts) if err != nil && res == nil { diff --git a/controllers/gitlab/mocks_test.go b/controllers/gitlab/mocks_test.go index bde2409..358bf60 100755 --- a/controllers/gitlab/mocks_test.go +++ b/controllers/gitlab/mocks_test.go @@ -98,11 +98,11 @@ func (m *MockClient) DeleteAllOf(ctx context.Context, obj client.Object, opts .. } type MockStatusWriter struct { - updateFunction func(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error + updateFunction func(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error updateFunctionCalled bool - patchFunction func(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error - updatedObject client.Object - updatedOptions []client.UpdateOption + patchFunction func(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error + updatedObject client.Object + updatedOptions []client.UpdateOption } func (m *MockStatusWriter) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { diff --git a/controllers/gitlab/project_controller.go b/controllers/gitlab/project_controller.go index aa0974e..d67ea4c 100644 --- a/controllers/gitlab/project_controller.go +++ b/controllers/gitlab/project_controller.go @@ -40,9 +40,9 @@ type ProjectReconciler struct { // errors const ( - errorWhileLookingUpProject string = "error while looking up project" - errorGettingGroupFromGitlab string = "Error while getting Group from Gitlab." - errorUpdatingStatusOfProject = "Error updating status of project" + errorWhileLookingUpProject string = "error while looking up project" + errorGettingGroupFromGitlab string = "Error while getting Group from Gitlab." + errorUpdatingStatusOfProject = "Error updating status of project" ) // statuses diff --git a/controllers/gitlab/project_controller_test.go b/controllers/gitlab/project_controller_test.go index 0668288..9751329 100644 --- a/controllers/gitlab/project_controller_test.go +++ b/controllers/gitlab/project_controller_test.go @@ -53,7 +53,7 @@ func getControllerWithMocksInGreenTestState() (ProjectReconciler, *MockManager, Language: "golang", }, Status: gitlabv1alpha1.ProjectStatus{ - URL: "", + URL: "", State: "", }, } -- GitLab From 71969fd054bcba7980e40a9dc0bca1a614337ec7 Mon Sep 17 00:00:00 2001 From: "adam.richards" Date: Thu, 24 Jun 2021 15:42:05 -0600 Subject: [PATCH 15/29] fmt and lint --- clients/gitlab/client.go | 2 +- .../gitlab/api/gitlab_api_adhoc_test.go | 100 ++---------- .../gitlab/api/gitlab_api_groups_test.go | 146 ++++++++++++++++-- .../gitlab/api/gitlab_api_users_test.go | 74 ++++++--- 4 files changed, 199 insertions(+), 123 deletions(-) diff --git a/clients/gitlab/client.go b/clients/gitlab/client.go index 45e7cd9..a79dcd5 100644 --- a/clients/gitlab/client.go +++ b/clients/gitlab/client.go @@ -599,8 +599,8 @@ func (r Client) GetProjects(search *string) ([]*gogitlab.Project, error) { ListOptions: listOptions, OrderBy: &orderBy, } - // if search defined add it to opts + // if search defined add it to opts if search != nil { opts.Search = search } diff --git a/integration-tests/gitlab/api/gitlab_api_adhoc_test.go b/integration-tests/gitlab/api/gitlab_api_adhoc_test.go index b21e399..4e50afd 100644 --- a/integration-tests/gitlab/api/gitlab_api_adhoc_test.go +++ b/integration-tests/gitlab/api/gitlab_api_adhoc_test.go @@ -3,15 +3,13 @@ package integration import ( - "testing" "errors" "os" + "testing" gitlab "valkyrie.dso.mil/valkyrie-api/clients/gitlab" - - gogitlab "github.com/xanzy/go-gitlab" ) -// getClient_AdHoc - +// getClient_AdHoc - func getClient_AdHoc() (gitlab.Client, error) { gitlabAPIURL, ok := os.LookupEnv("GITLAB_API_URL") if !ok { @@ -21,7 +19,7 @@ func getClient_AdHoc() (gitlab.Client, error) { if !ok { return gitlab.Client{}, errors.New("env variable GITLAB_API_TOKEN undefinded") } - var client = gitlab.NewClient(gitlabAPIURL, gitlabAPIToken,nil) + var client = gitlab.NewClient(gitlabAPIURL, gitlabAPIToken, nil) return client, nil } func TestClient_AdHoc_getUsers_All(t *testing.T) { @@ -37,7 +35,7 @@ func TestClient_AdHoc_getUsers_All(t *testing.T) { return } t.Logf("GetUsers %d", len(got)) - for x:=0;x Date: Thu, 24 Jun 2021 16:18:38 -0600 Subject: [PATCH 16/29] project integration tests --- clients/gitlab/client.go | 4 +- clients/gitlab/client_test.go | 6 +- .../gitlab/api/gitlab_api_projects_test.go | 211 ++++++++++++++++++ .../gitlab/api/gitlab_api_users_test.go | 2 +- 4 files changed, 217 insertions(+), 6 deletions(-) create mode 100644 integration-tests/gitlab/api/gitlab_api_projects_test.go diff --git a/clients/gitlab/client.go b/clients/gitlab/client.go index a79dcd5..7b39504 100644 --- a/clients/gitlab/client.go +++ b/clients/gitlab/client.go @@ -629,7 +629,7 @@ func (r Client) GetProjects(search *string) ([]*gogitlab.Project, error) { } // AddProject - -func (r Client) AddProject(createProjectOptions gogitlab.CreateProjectOptions, groupID int) (*gogitlab.Project, int, error) { +func (r Client) AddProject(createProjectOptions gogitlab.CreateProjectOptions) (*gogitlab.Project, int, error) { logPrefix := "AddProject" // force visibility to private var visibility = gogitlab.PrivateVisibility @@ -638,7 +638,7 @@ func (r Client) AddProject(createProjectOptions gogitlab.CreateProjectOptions, g Name: createProjectOptions.Name, Path: createProjectOptions.Path, Description: createProjectOptions.Description, - NamespaceID: &groupID, + NamespaceID: createProjectOptions.NamespaceID, // hard coded values Visibility: &visibility, } diff --git a/clients/gitlab/client_test.go b/clients/gitlab/client_test.go index c37374f..9073ddc 100644 --- a/clients/gitlab/client_test.go +++ b/clients/gitlab/client_test.go @@ -937,9 +937,10 @@ func TestClient_AddProject(t *testing.T) { defer httpmock.DeactivateAndReset() testProjectID := 1 + testGroupID := 12 testProjectName := "testProjectName" testProject := gogitlab.Project{ID: testProjectID, Name: testProjectName} - createTestProjetOptions := gogitlab.CreateProjectOptions{Name: &testProjectName} + createTestProjetOptions := gogitlab.CreateProjectOptions{Name: &testProjectName, NamespaceID: &testGroupID} httpmock.RegisterResponder("POST", "https://test/api/v4/projects", @@ -959,7 +960,6 @@ func TestClient_AddProject(t *testing.T) { } type args struct { createProjectOptions *gogitlab.CreateProjectOptions - groupID int } tests := []struct { name string @@ -985,7 +985,7 @@ func TestClient_AddProject(t *testing.T) { token: tt.fields.token, apiURL: tt.fields.apiURL, } - got, got1, err := r.AddProject(*tt.args.createProjectOptions, tt.args.groupID) + got, got1, err := r.AddProject(*tt.args.createProjectOptions) if (err != nil) != tt.wantErr { t.Errorf("Client.AddProject() error = %v, wantErr %v", err, tt.wantErr) return diff --git a/integration-tests/gitlab/api/gitlab_api_projects_test.go b/integration-tests/gitlab/api/gitlab_api_projects_test.go new file mode 100644 index 0000000..b4c4d3a --- /dev/null +++ b/integration-tests/gitlab/api/gitlab_api_projects_test.go @@ -0,0 +1,211 @@ +// +build integration + +package integration + +import ( + "errors" + "fmt" + "net/http" + "os" + "testing" + + gogitlab "github.com/xanzy/go-gitlab" + gitlab "valkyrie.dso.mil/valkyrie-api/clients/gitlab" +) + +const projectsTestItemCount = 65 +const testProjectsPrefix = "IntTestProject" + +var integrationRootGroupID = 0 + +// getClientProjects - +func getClientProjects() (gitlab.Client, error) { + gitlabAPIURL, ok := os.LookupEnv("GITLAB_API_URL") + if !ok { + return gitlab.Client{}, errors.New("env variable GITLAB_API_URL undefinded") + } + gitlabAPIToken, ok := os.LookupEnv("GITLAB_API_TOKEN") + if !ok { + return gitlab.Client{}, errors.New("env variable GITLAB_API_TOKEN undefinded") + } + var client = gitlab.NewClient(gitlabAPIURL, gitlabAPIToken, nil) + return client, nil +} + +func TestClient_GetProjects(t *testing.T) { + t.Run("test", func(t *testing.T) { + c, err := getClientProjects() + if err != nil { + t.Errorf("Client.getClientProjects() error = %v", err) + return + } + got, err := c.GetProjects(nil) + if err != nil { + t.Errorf("Client.GetProjects() error = %v", err) + return + } + t.Logf("GetProjects - number of projects %d", len(got)) + }) +} + +func TestClient_GetProjects_Search(t *testing.T) { + t.Run("test", func(t *testing.T) { + c, err := getClientProjects() + if err != nil { + t.Errorf("Client.getClientProjects() error = %v", err) + return + } + + searchString := "monitor" + + got, err := c.GetProjects(&searchString) + if err != nil { + t.Errorf("Client.GetProjects() error = %v", err) + return + } + t.Logf("GetProjects - number of projects %d", len(got)) + }) +} + +// Insure the integration test root level group exists +func TestClient_AddRootGroup(t *testing.T) { + t.Run("test", func(t *testing.T) { + c, err := getClientProjects() + if err != nil { + t.Errorf("Client.getClientProjects() error = %v", err) + return + } + + path := "integration-root-group" + name := "integration root group" + description := "Integration Test Root Level Group" + + group, statusCode, err := c.GetGroupByFullPath(&path) + if err != nil { + // dont fail if group already exists + t.Errorf("Client.GetGroupByFullPath() error = %v %d", err, statusCode) + return + } + + if statusCode == http.StatusFound { + integrationRootGroupID = group.ID + t.Logf("AddGroup %s %s %s %s %d", group.Name, group.Path, group.FullPath, group.WebURL, statusCode) + } else { + groupOpts := gogitlab.CreateGroupOptions{Name: &name, Path: &path, Description: &description} + got, statusCode, err := c.AddGroup(groupOpts) + + if err != nil { + // dont fail if group already exists + t.Logf("Client.AddGroup() error = %v %d", err, statusCode) + } else { + integrationRootGroupID = got.ID + t.Logf("AddGroup %s %s %s %s %d", got.Name, got.Path, got.FullPath, got.WebURL, statusCode) + } + } + }) +} + +// add groups under the integration test root level group +func TestClient_AddProjects(t *testing.T) { + t.Run("test", func(t *testing.T) { + c, err := getClientProjects() + if err != nil { + t.Errorf("Client.getClientProjects() error = %v", err) + return + } + + for i := 1; i <= projectsTestItemCount; i++ { + path := testProjectsPrefix + "-project-" + fmt.Sprintf("%d", i) + name := testProjectsPrefix + " project " + fmt.Sprintf("%d", i) + description := "My fancy test project " + fmt.Sprintf("%d", i) + + projectOpts := gogitlab.CreateProjectOptions{ + Name: &name, + Path: &path, + Description: &description, + NamespaceID: &integrationRootGroupID, + } + got, statusCode, err := c.AddProject(projectOpts) + + if err != nil { + t.Errorf("Client.AddProject() error = %v %d", err, statusCode) + } else { + t.Logf("AddProject %s %s %s %s %d", got.Name, got.Path, got.PathWithNamespace, got.WebURL, statusCode) + } + } + + }) +} + +func TestClient_UpdateProject(t *testing.T) { + t.Run("test", func(t *testing.T) { + c, err := getClientProjects() + if err != nil { + t.Errorf("Client.getClientProjects() error = %v", err) + return + } + + // create a group to contain test objects + path := testProjectsPrefix + "-project-update" + name := testProjectsPrefix + " project update" + description := "My fancy test project " + + projectOpts := gogitlab.CreateProjectOptions{ + Name: &name, + Path: &path, + Description: &description, + NamespaceID: &integrationRootGroupID, + } + newProject, statusCode, err := c.AddProject(projectOpts) + if err != nil { + t.Errorf("Client.AddProject() error = %v", err) + return + } + t.Logf(" created project %s path %s status code %d", newProject.Name, newProject.PathWithNamespace, statusCode) + + // update user + testDescription := "Updated Description" + updateProjectOptions := gogitlab.EditProjectOptions{Description: &testDescription} + updatedProject, statusCode, err := c.UpdateProject(newProject.ID, updateProjectOptions) + if err != nil { + t.Errorf("Client.UpdateProject() error = %v", err) + return + } + t.Logf("UpdateProject: %s %d", updatedProject.Description, statusCode) + + // cleanup + statusCode, _ = c.DeleteProject(newProject.ID, 0, 0) + t.Logf("DeleteProject no wait statusCode: %d", statusCode) + + }) +} + + +func TestClient_deleteProjects(t *testing.T) { + t.Run("test", func(t *testing.T) { + c, err := getClientProjects() + if err != nil { + t.Errorf("Client.getClient() error = %v", err) + return + } + + search := testProjectsPrefix + projectList, err := c.GetProjects(&search) + if err != nil { + t.Errorf("Client.GetProjects() error = %v", err) + return + } + + for x := 0; x < len(projectList); x++ { + project := projectList[x] + statusCode, err := c.DeleteProject(project.ID, 100, 1200) + + if err != nil { + t.Errorf("Client.DeleteProject() error = %v %d", err, statusCode) + } else { + t.Logf("DeleteProject %s %d", project.Name, statusCode) + } + } + + }) +} diff --git a/integration-tests/gitlab/api/gitlab_api_users_test.go b/integration-tests/gitlab/api/gitlab_api_users_test.go index bb77152..b9a416d 100644 --- a/integration-tests/gitlab/api/gitlab_api_users_test.go +++ b/integration-tests/gitlab/api/gitlab_api_users_test.go @@ -13,7 +13,7 @@ import ( ) const testItemCount = 65 -const testUserPrefix = "IntTesttestuser" +const testUserPrefix = "IntTestUser" // getClient - func getClientUsers() (gitlab.Client, error) { -- GitLab From 894c430cc6cfd2e8a4543743ae0c285c345e2994 Mon Sep 17 00:00:00 2001 From: "adam.richards" Date: Thu, 24 Jun 2021 19:39:26 -0600 Subject: [PATCH 17/29] add custom code specifc for p1 --- custom/p1/project.go | 11 ++++++ custom/p1/types.go | 39 +++++++++++++++++++ scripts/template.sh | 92 -------------------------------------------- 3 files changed, 50 insertions(+), 92 deletions(-) create mode 100644 custom/p1/project.go create mode 100644 custom/p1/types.go delete mode 100755 scripts/template.sh diff --git a/custom/p1/project.go b/custom/p1/project.go new file mode 100644 index 0000000..1ea6943 --- /dev/null +++ b/custom/p1/project.go @@ -0,0 +1,11 @@ +package p1 + +import "github.com/romana/rlog" + +// CreateProject - +func CreateProject(projectConfig ProjectConfig) { + rlog.Debugf("CreateProject config %s %s %s", + projectConfig.projectName, + projectConfig.parentGroupFullPath, + projectConfig.stackType.pipelineTemplatePath) +} diff --git a/custom/p1/types.go b/custom/p1/types.go new file mode 100644 index 0000000..98f7d4c --- /dev/null +++ b/custom/p1/types.go @@ -0,0 +1,39 @@ +package p1 + +// ProjectConfig - +type ProjectConfig struct { + projectName string + parentGroupFullPath string + stackType StackType +} + +// enumeration for tech stack types +const ( + StackTypeNodejsNpm = "StackTypeNodejsNpm" + StackTypeNodejsYarn = "StackTypeNodejsYarn" + StackTypeJavaGradle = "StackTypeJavaGradle" + StackTypeJavaMaven = "StackTypeJavaMaven" + StackTypePythonPip = "StackTypePythonPip" + StackTypePythonPipEnv = "StackTypePythonPipEnv" + StackTypeReact = "StackTypeReact" + StackTypeAngular = "StackTypeAngular" + StackTypeGoLang = "StackTypeGoLang" + StackTypeCPP = "StackTypeCPP" +) + +// StackType - +type StackType struct { + stackType string + pipelineTemplatePath string +} + +// StackTypes - map of available stack types in P1 and corresponding pipeline template path +var StackTypes map[string]StackType + +// initialize stackTypes +func init() { + StackTypes[StackTypeNodejsNpm] = StackType{stackType: StackTypeNodejsNpm, pipelineTemplatePath: "templates/cpp.yml"} + StackTypes[StackTypeNodejsYarn] = StackType{stackType: StackTypeNodejsYarn, pipelineTemplatePath: "templates/yarn.yml"} + StackTypes[StackTypeJavaGradle] = StackType{stackType: StackTypeJavaGradle, pipelineTemplatePath: "templates/gradle.yml"} + StackTypes[StackTypeJavaMaven] = StackType{stackType: StackTypeJavaMaven, pipelineTemplatePath: "templates/maven.yml"} +} diff --git a/scripts/template.sh b/scripts/template.sh deleted file mode 100755 index f36ceea..0000000 --- a/scripts/template.sh +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env bash -########################################## -# Boilerplate -# Author: Adam Richards -########################################## -SCRIPT=$(basename "${0}") -SCRIPTPATH=$(cd "$(dirname "${0}")" && pwd -P) -FULLSCRIPT="${SCRIPTPATH}/${SCRIPT}" -NO_ROOT=1 -########################################## -echo_stderr() { - # echo to stderr - echo "$@" 1>&2 -} -########################################## -if ((NO_ROOT == 1)); then - id=$(id -u) - if [[ "$id" -eq "0" ]]; then - echo_stderr "Running as root not allowed" - exit 2 - fi -fi -########################################## -function interrupt_handler() { - echo_stderr "INTERRUPT Received!" - # cleanup - exit 3 -} -# bind function to interrupts SIGINT and SIGTERM: -trap interrupt_handler SIGINT -trap interrupt_handler SIGTERM -########################################## -function syntax() { - echo_stderr "Syntax $SCRIPT args args args" - echo_stderr "ex: $SCRIPT args args args" - echo_stderr "SCRIPT: $SCRIPT" - echo_stderr "SCRIPTPATH: $SCRIPTPATH" - echo_stderr "FULLSCRIPT: $FULLSCRIPT" - exit 1 -} -########################################## -# using getopts rather than getopt as getopt may not be available on system -while getopts ":hn:t:" options; do # Loop: Get the next option; - # use silent error checking- leading ':'; - # options that take args are followed by ':' - case "${options}" in - h) # If the option is n, - syntax - ;; - n) - NAME=${OPTARG} - ;; - t) - TIMES=${OPTARG} # Set $TIMES to specified value. - re_isanum='^[0-9]+$' # Regex: match whole numbers only - if ! [[ $TIMES =~ $re_isanum ]]; then # if $TIMES not a whole number: - echo "Error: TIMES must be a positive, whole number." - syntax - exit 1 - elif [ "$TIMES" -eq "0" ]; then # If it's zero: - echo "Error: TIMES must be greater than zero." - syntax # Exit abnormally. - fi - ;; - :) # If expected argument omitted: - echo_stderr "Error: option -${OPTARG} requires an argument." - syntax - ;; - *) # If unknown (any other) option: - echo_stderr "Error: unknown option -${OPTARG}." - syntax - ;; - esac -done -########################################## -echo "$NAME" -echo "$TIMES" -# Do The Work -M=3 -while ( $TRUE ) && (($M > 0)); do - echo "loop $M" - M=$((M - 1)) - sleep 2 -done -########################################## -## loops that will properly handle files with spaces and backslash -find . -name "*.*" -print0 | while read -rd $'\0' file; do - echo "FILE: $file" -done -while read -rd $'\0' file; do - echo "FILE: $file" -done < <(find . -type f -name '*.*' -print0) -- GitLab From a088ef1159b8bf09ab096052c7376230a818babf Mon Sep 17 00:00:00 2001 From: "adam.richards" Date: Fri, 25 Jun 2021 10:09:40 -0600 Subject: [PATCH 18/29] add simple p1 gitlab methods --- custom/p1/project.go | 28 ++++++++++++++++++++++++---- custom/p1/types.go | 2 +- custom/p1/utils.go | 25 +++++++++++++++++++++++++ custom/p1/utils_test.go | 36 ++++++++++++++++++++++++++++++++++++ 4 files changed, 86 insertions(+), 5 deletions(-) create mode 100644 custom/p1/utils.go create mode 100644 custom/p1/utils_test.go diff --git a/custom/p1/project.go b/custom/p1/project.go index 1ea6943..3daf1b7 100644 --- a/custom/p1/project.go +++ b/custom/p1/project.go @@ -1,11 +1,31 @@ package p1 -import "github.com/romana/rlog" +import ( + "github.com/romana/rlog" +) -// CreateProject - -func CreateProject(projectConfig ProjectConfig) { - rlog.Debugf("CreateProject config %s %s %s", +func logStart(logPrefix string, projectConfig ProjectConfig) { + rlog.Debugf("%s start. projectConfig %s %s %s", + logPrefix, projectConfig.projectName, projectConfig.parentGroupFullPath, projectConfig.stackType.pipelineTemplatePath) } + +// CreateProject - +func CreateProject(projectConfig ProjectConfig) { + logPrefix := "CreateProject" + logStart(logPrefix, projectConfig) +} + +// DeleteProject - +func DeleteProject(projectConfig ProjectConfig) { + logPrefix := "CreateProject" + logStart(logPrefix, projectConfig) +} + +// UpdateProject - +func UpdateProject(projectConfig ProjectConfig) { + logPrefix := "UpdateProject" + logStart(logPrefix, projectConfig) +} \ No newline at end of file diff --git a/custom/p1/types.go b/custom/p1/types.go index 98f7d4c..e1e154c 100644 --- a/custom/p1/types.go +++ b/custom/p1/types.go @@ -28,7 +28,7 @@ type StackType struct { } // StackTypes - map of available stack types in P1 and corresponding pipeline template path -var StackTypes map[string]StackType +var StackTypes map[string]StackType = map[string]StackType{} // initialize stackTypes func init() { diff --git a/custom/p1/utils.go b/custom/p1/utils.go new file mode 100644 index 0000000..4f3400b --- /dev/null +++ b/custom/p1/utils.go @@ -0,0 +1,25 @@ +package p1 + +import ( + "regexp" +) + +// GenerateGitlabPath - use gitlab name to generate a path +func GenerateGitlabPath(name string) (string, error) { + var re *regexp.Regexp + var path = name + + // remove any leading dash or whitespace + re, _ = regexp.Compile(`^[\s-]+`) + path = re.ReplaceAllString(path, "") + + // remove any trailing dash or whitespace + re, _ = regexp.Compile(`[\s-]+$`) + path = re.ReplaceAllString(path, "") + + // replace any whitespace with dash + re, _ = regexp.Compile(`\s+`) + path = re.ReplaceAllString(path, "-") + + return path, nil +} diff --git a/custom/p1/utils_test.go b/custom/p1/utils_test.go new file mode 100644 index 0000000..ee31ddd --- /dev/null +++ b/custom/p1/utils_test.go @@ -0,0 +1,36 @@ +package p1 + +import "testing" + +func TestGenerateGitlabPath(t *testing.T) { + type args struct { + name string + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + {name: "re test 1", args: args {name: "a b c"}, want: "a-b-c", wantErr: false}, + {name: "re test 2", args: args {name: "- a b c - "}, want: "a-b-c", wantErr: false}, + {name: "re test 2", args: args {name: "-a b c- "}, want: "a-b-c", wantErr: false}, + {name: "re test 2", args: args {name: " - - a b c- - "}, want: "a-b-c", wantErr: false}, + {name: "re test 3", args: args {name: "a b c "}, want: "a-b-c", wantErr: false}, + {name: "re test 4", args: args {name: " a b c"}, want: "a-b-c", wantErr: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := GenerateGitlabPath(tt.args.name) + if (err != nil) != tt.wantErr { + t.Errorf("GenerateGitlabPath() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("GenerateGitlabPath() FAIL '%v', want '%v'", got, tt.want) + } else { + t.Logf("GenerateGitlabPath() PASS input '%s' '%s', want '%s'", tt.args.name, got, tt.want) + } + }) + } +} -- GitLab From bcd764e3cee76137ef8ebb0580b5579ea4ee0461 Mon Sep 17 00:00:00 2001 From: "adam.richards" Date: Fri, 25 Jun 2021 10:32:03 -0600 Subject: [PATCH 19/29] fix package level conflicts --- integration-tests/gitlab/api/gitlab_api_projects_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration-tests/gitlab/api/gitlab_api_projects_test.go b/integration-tests/gitlab/api/gitlab_api_projects_test.go index b4c4d3a..48d5792 100644 --- a/integration-tests/gitlab/api/gitlab_api_projects_test.go +++ b/integration-tests/gitlab/api/gitlab_api_projects_test.go @@ -16,7 +16,7 @@ import ( const projectsTestItemCount = 65 const testProjectsPrefix = "IntTestProject" -var integrationRootGroupID = 0 +var integrationRootGroupIDProjects = 0 // getClientProjects - func getClientProjects() (gitlab.Client, error) { @@ -68,7 +68,7 @@ func TestClient_GetProjects_Search(t *testing.T) { } // Insure the integration test root level group exists -func TestClient_AddRootGroup(t *testing.T) { +func TestClient_AddRootGroup_Projects(t *testing.T) { t.Run("test", func(t *testing.T) { c, err := getClientProjects() if err != nil { -- GitLab From 4dce1c216d1c1737f5b6ab00e41ebdd60ddfeb9e Mon Sep 17 00:00:00 2001 From: "adam.richards" Date: Fri, 25 Jun 2021 10:33:32 -0600 Subject: [PATCH 20/29] fix package scope collisions --- integration-tests/gitlab/api/gitlab_api_projects_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/integration-tests/gitlab/api/gitlab_api_projects_test.go b/integration-tests/gitlab/api/gitlab_api_projects_test.go index 48d5792..320e257 100644 --- a/integration-tests/gitlab/api/gitlab_api_projects_test.go +++ b/integration-tests/gitlab/api/gitlab_api_projects_test.go @@ -88,7 +88,7 @@ func TestClient_AddRootGroup_Projects(t *testing.T) { } if statusCode == http.StatusFound { - integrationRootGroupID = group.ID + integrationRootGroupIDProjects = group.ID t.Logf("AddGroup %s %s %s %s %d", group.Name, group.Path, group.FullPath, group.WebURL, statusCode) } else { groupOpts := gogitlab.CreateGroupOptions{Name: &name, Path: &path, Description: &description} @@ -98,7 +98,7 @@ func TestClient_AddRootGroup_Projects(t *testing.T) { // dont fail if group already exists t.Logf("Client.AddGroup() error = %v %d", err, statusCode) } else { - integrationRootGroupID = got.ID + integrationRootGroupIDProjects = got.ID t.Logf("AddGroup %s %s %s %s %d", got.Name, got.Path, got.FullPath, got.WebURL, statusCode) } } @@ -123,7 +123,7 @@ func TestClient_AddProjects(t *testing.T) { Name: &name, Path: &path, Description: &description, - NamespaceID: &integrationRootGroupID, + NamespaceID: &integrationRootGroupIDProjects, } got, statusCode, err := c.AddProject(projectOpts) @@ -154,7 +154,7 @@ func TestClient_UpdateProject(t *testing.T) { Name: &name, Path: &path, Description: &description, - NamespaceID: &integrationRootGroupID, + NamespaceID: &integrationRootGroupIDProjects, } newProject, statusCode, err := c.AddProject(projectOpts) if err != nil { -- GitLab From 145e64187a24feae8ea33ae2c4d48610e50c899d Mon Sep 17 00:00:00 2001 From: "adam.richards" Date: Fri, 25 Jun 2021 13:19:21 -0600 Subject: [PATCH 21/29] update unit tests for project type --- custom/p1/project.go | 12 ++++----- custom/p1/types.go | 38 +++++++++++++++++++++++++--- custom/p1/types_test.go | 56 +++++++++++++++++++++++++++++++++++++++++ custom/p1/utils_test.go | 12 ++++----- 4 files changed, 103 insertions(+), 15 deletions(-) create mode 100644 custom/p1/types_test.go diff --git a/custom/p1/project.go b/custom/p1/project.go index 3daf1b7..34a50bf 100644 --- a/custom/p1/project.go +++ b/custom/p1/project.go @@ -6,10 +6,10 @@ import ( func logStart(logPrefix string, projectConfig ProjectConfig) { rlog.Debugf("%s start. projectConfig %s %s %s", - logPrefix, - projectConfig.projectName, - projectConfig.parentGroupFullPath, - projectConfig.stackType.pipelineTemplatePath) + logPrefix, + projectConfig.projectName, + projectConfig.parentGroupFullPath, + projectConfig.stackType.pipelineTemplatePath) } // CreateProject - @@ -20,7 +20,7 @@ func CreateProject(projectConfig ProjectConfig) { // DeleteProject - func DeleteProject(projectConfig ProjectConfig) { - logPrefix := "CreateProject" + logPrefix := "DeleteProject" logStart(logPrefix, projectConfig) } @@ -28,4 +28,4 @@ func DeleteProject(projectConfig ProjectConfig) { func UpdateProject(projectConfig ProjectConfig) { logPrefix := "UpdateProject" logStart(logPrefix, projectConfig) -} \ No newline at end of file +} diff --git a/custom/p1/types.go b/custom/p1/types.go index e1e154c..16dfdf0 100644 --- a/custom/p1/types.go +++ b/custom/p1/types.go @@ -1,5 +1,9 @@ package p1 +import ( + "fmt" +) + // ProjectConfig - type ProjectConfig struct { projectName string @@ -18,12 +22,13 @@ const ( StackTypeReact = "StackTypeReact" StackTypeAngular = "StackTypeAngular" StackTypeGoLang = "StackTypeGoLang" - StackTypeCPP = "StackTypeCPP" + StackTypeCpp = "StackTypeCpp" + StackTypeUnknown = "StackTypeUnknown" ) // StackType - type StackType struct { - stackType string + stackType string pipelineTemplatePath string } @@ -32,8 +37,35 @@ var StackTypes map[string]StackType = map[string]StackType{} // initialize stackTypes func init() { - StackTypes[StackTypeNodejsNpm] = StackType{stackType: StackTypeNodejsNpm, pipelineTemplatePath: "templates/cpp.yml"} + StackTypes[StackTypeNodejsNpm] = StackType{stackType: StackTypeNodejsNpm, pipelineTemplatePath: "templates/npm.yml"} StackTypes[StackTypeNodejsYarn] = StackType{stackType: StackTypeNodejsYarn, pipelineTemplatePath: "templates/yarn.yml"} StackTypes[StackTypeJavaGradle] = StackType{stackType: StackTypeJavaGradle, pipelineTemplatePath: "templates/gradle.yml"} StackTypes[StackTypeJavaMaven] = StackType{stackType: StackTypeJavaMaven, pipelineTemplatePath: "templates/maven.yml"} + StackTypes[StackTypePythonPip] = StackType{stackType: StackTypePythonPip, pipelineTemplatePath: "templates/maven.yml"} + StackTypes[StackTypePythonPipEnv] = StackType{stackType: StackTypePythonPipEnv, pipelineTemplatePath: "templates/maven.yml"} + StackTypes[StackTypeReact] = StackType{stackType: StackTypeReact, pipelineTemplatePath: "templates/npm.yml"} + StackTypes[StackTypeAngular] = StackType{stackType: StackTypeAngular, pipelineTemplatePath: "templates/npm.yml"} + StackTypes[StackTypeGoLang] = StackType{stackType: StackTypeGoLang, pipelineTemplatePath: "templates/go.yml"} + StackTypes[StackTypeCpp] = StackType{stackType: StackTypeCpp, pipelineTemplatePath: "templates/cpp.yml"} + StackTypes[StackTypeUnknown] = StackType{stackType: StackTypeUnknown, pipelineTemplatePath: ""} +} + +// CreateProjectConfig - +func CreateProjectConfig(name string, path string, stackTypeName string) (ProjectConfig, error) { + stackType, ok := StackTypes[stackTypeName] + if !ok { + keys := make([]string, 0, len(StackTypes)) + for k := range StackTypes { + keys = append(keys, k) + } + keysStr := fmt.Sprintf("valid keys %v", keys) + return ProjectConfig{}, error(fmt.Errorf(keysStr)) + } + + projectConfig := ProjectConfig{ + projectName: name, + parentGroupFullPath: path, + stackType: stackType, + } + return projectConfig, nil } diff --git a/custom/p1/types_test.go b/custom/p1/types_test.go new file mode 100644 index 0000000..33cbc86 --- /dev/null +++ b/custom/p1/types_test.go @@ -0,0 +1,56 @@ +package p1 + +import ( + "reflect" + "testing" +) + +func TestCreateProjectConfig(t *testing.T) { + type args struct { + name string + path string + stackTypeName string + } + testName := "test project name" + testGroupPath := "root/group1/group2" + testStackType := "StackTypeNodejsNpm" + testStackTypeNotOK := "badindex" + + testProjectConfigOK := ProjectConfig{ + projectName: testName, + parentGroupFullPath: testGroupPath, + stackType: StackTypes[testStackType], + } + testProjectConfigNotOK := ProjectConfig{ + projectName: testName, + parentGroupFullPath: testGroupPath, + stackType: StackTypes[testStackTypeNotOK], + } + + tests := []struct { + name string + args args + want ProjectConfig + wantErr bool + }{ + {name: "config 1", args: args{name: testName, path: testGroupPath, stackTypeName: testStackType}, want: testProjectConfigOK, wantErr: false}, + {name: "config 2", args: args{name: testName, path: testGroupPath, stackTypeName: testStackTypeNotOK}, want: testProjectConfigNotOK, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := CreateProjectConfig(tt.args.name, tt.args.path, tt.args.stackTypeName) + if (err != nil) { + if (!tt.wantErr) { + t.Errorf("CreateProjectConfig() error = %v, wantErr %v", err, tt.wantErr) + return + } else { + t.Logf("CreateProjectConfig() received EXPECTED error = %v, wantErr %v", err, tt.wantErr) + return + } + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("CreateProjectConfig() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/custom/p1/utils_test.go b/custom/p1/utils_test.go index ee31ddd..88722d6 100644 --- a/custom/p1/utils_test.go +++ b/custom/p1/utils_test.go @@ -12,12 +12,12 @@ func TestGenerateGitlabPath(t *testing.T) { want string wantErr bool }{ - {name: "re test 1", args: args {name: "a b c"}, want: "a-b-c", wantErr: false}, - {name: "re test 2", args: args {name: "- a b c - "}, want: "a-b-c", wantErr: false}, - {name: "re test 2", args: args {name: "-a b c- "}, want: "a-b-c", wantErr: false}, - {name: "re test 2", args: args {name: " - - a b c- - "}, want: "a-b-c", wantErr: false}, - {name: "re test 3", args: args {name: "a b c "}, want: "a-b-c", wantErr: false}, - {name: "re test 4", args: args {name: " a b c"}, want: "a-b-c", wantErr: false}, + {name: "re test 1", args: args{name: "a b c"}, want: "a-b-c", wantErr: false}, + {name: "re test 2", args: args{name: "- a b c - "}, want: "a-b-c", wantErr: false}, + {name: "re test 2", args: args{name: "-a b c- "}, want: "a-b-c", wantErr: false}, + {name: "re test 2", args: args{name: " - - a b c- - "}, want: "a-b-c", wantErr: false}, + {name: "re test 3", args: args{name: "a b c "}, want: "a-b-c", wantErr: false}, + {name: "re test 4", args: args{name: " a b c"}, want: "a-b-c", wantErr: false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { -- GitLab From 333a01c6bdcd814def67ef2a5996cc9a5652e956 Mon Sep 17 00:00:00 2001 From: "adam.richards" Date: Fri, 25 Jun 2021 14:10:28 -0600 Subject: [PATCH 22/29] add project go as interface class for controllers --- clients/gitlab/client.go | 16 ++++---- custom/p1/project.go | 40 +++++++++++++++++-- custom/p1/types.go | 4 ++ custom/p1/types_test.go | 12 +++--- .../gitlab/api/gitlab_api_projects_test.go | 2 + 5 files changed, 58 insertions(+), 16 deletions(-) diff --git a/clients/gitlab/client.go b/clients/gitlab/client.go index 7b39504..711663d 100644 --- a/clients/gitlab/client.go +++ b/clients/gitlab/client.go @@ -635,10 +635,11 @@ func (r Client) AddProject(createProjectOptions gogitlab.CreateProjectOptions) ( var visibility = gogitlab.PrivateVisibility // copy customizable settings from argument, hard code other options as desired var opts = gogitlab.CreateProjectOptions{ - Name: createProjectOptions.Name, - Path: createProjectOptions.Path, - Description: createProjectOptions.Description, - NamespaceID: createProjectOptions.NamespaceID, + Name: createProjectOptions.Name, + Path: createProjectOptions.Path, + Description: createProjectOptions.Description, + CIConfigPath: createProjectOptions.CIConfigPath, + NamespaceID: createProjectOptions.NamespaceID, // hard coded values Visibility: &visibility, } @@ -659,9 +660,10 @@ func (r Client) UpdateProject(projectID int, editProjectOptions gogitlab.EditPro logPrefix := "UpdateProject" // copy customizable settings from argument, hard code other options as desired var opts = gogitlab.EditProjectOptions{ - Name: editProjectOptions.Name, - Path: editProjectOptions.Path, - Description: editProjectOptions.Description, + Name: editProjectOptions.Name, + Path: editProjectOptions.Path, + Description: editProjectOptions.Description, + CIConfigPath: editProjectOptions.CIConfigPath, } project, res, err := r.client.Projects.EditProject(projectID, &opts) diff --git a/custom/p1/project.go b/custom/p1/project.go index 34a50bf..3d50451 100644 --- a/custom/p1/project.go +++ b/custom/p1/project.go @@ -1,7 +1,12 @@ package p1 import ( + "fmt" + "net/http" + "github.com/romana/rlog" + gogitlab "github.com/xanzy/go-gitlab" + gitlab "valkyrie.dso.mil/valkyrie-api/clients/gitlab" ) func logStart(logPrefix string, projectConfig ProjectConfig) { @@ -13,19 +18,48 @@ func logStart(logPrefix string, projectConfig ProjectConfig) { } // CreateProject - -func CreateProject(projectConfig ProjectConfig) { +func CreateProject(projetCredentials ProjectCredentials, projectConfig ProjectConfig) error { logPrefix := "CreateProject" logStart(logPrefix, projectConfig) + ciConfigPath := generateCIConfigPath(projectConfig.projectName, projectConfig.parentGroupFullPath) + + client := gitlab.NewClient(projetCredentials.serverURL, projetCredentials.serverToken, nil) + + parentGroup, statusCode, err := client.GetGroupByFullPath(&projectConfig.parentGroupFullPath) + if err != nil { + return err + } + + if (statusCode == http.StatusNotFound ) { + return fmt.Errorf(fmt.Sprintf("Faild to find group with path %s",projectConfig.parentGroupFullPath)) + } + + projectOptions := gogitlab.CreateProjectOptions{ + Name: &projectConfig.projectName, + NamespaceID: &parentGroup.ID, + CIConfigPath: &ciConfigPath, + } + + client.AddProject(projectOptions) + return nil } // DeleteProject - -func DeleteProject(projectConfig ProjectConfig) { +func DeleteProject(projectConfig ProjectConfig) error { logPrefix := "DeleteProject" logStart(logPrefix, projectConfig) + return nil } // UpdateProject - -func UpdateProject(projectConfig ProjectConfig) { +func UpdateProject(projectConfig ProjectConfig) error { logPrefix := "UpdateProject" logStart(logPrefix, projectConfig) + return nil } + +func generateCIConfigPath(projectName string, parentGroupFullPath string) (string) { + projectPathString, _ := GenerateGitlabPath(projectName) + ciConfigPath := fmt.Sprintf("%s/%s-ci.yml@platform-one/devops/pipeline-products",parentGroupFullPath,projectPathString) + return ciConfigPath + } \ No newline at end of file diff --git a/custom/p1/types.go b/custom/p1/types.go index 16dfdf0..f3fde0c 100644 --- a/custom/p1/types.go +++ b/custom/p1/types.go @@ -10,6 +10,10 @@ type ProjectConfig struct { parentGroupFullPath string stackType StackType } +type ProjectCredentials struct { + serverURL string + serverToken string +} // enumeration for tech stack types const ( diff --git a/custom/p1/types_test.go b/custom/p1/types_test.go index 33cbc86..645bf30 100644 --- a/custom/p1/types_test.go +++ b/custom/p1/types_test.go @@ -39,13 +39,13 @@ func TestCreateProjectConfig(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, err := CreateProjectConfig(tt.args.name, tt.args.path, tt.args.stackTypeName) - if (err != nil) { - if (!tt.wantErr) { - t.Errorf("CreateProjectConfig() error = %v, wantErr %v", err, tt.wantErr) - return + if err != nil { + if !tt.wantErr { + t.Errorf("CreateProjectConfig() error = %v, wantErr %v", err, tt.wantErr) + return } else { - t.Logf("CreateProjectConfig() received EXPECTED error = %v, wantErr %v", err, tt.wantErr) - return + t.Logf("CreateProjectConfig() received EXPECTED error = %v, wantErr %v", err, tt.wantErr) + return } } if !reflect.DeepEqual(got, tt.want) { diff --git a/integration-tests/gitlab/api/gitlab_api_projects_test.go b/integration-tests/gitlab/api/gitlab_api_projects_test.go index 320e257..a14fd85 100644 --- a/integration-tests/gitlab/api/gitlab_api_projects_test.go +++ b/integration-tests/gitlab/api/gitlab_api_projects_test.go @@ -118,12 +118,14 @@ func TestClient_AddProjects(t *testing.T) { path := testProjectsPrefix + "-project-" + fmt.Sprintf("%d", i) name := testProjectsPrefix + " project " + fmt.Sprintf("%d", i) description := "My fancy test project " + fmt.Sprintf("%d", i) + configPath := "a/b/config.yml@group1/group2/project" projectOpts := gogitlab.CreateProjectOptions{ Name: &name, Path: &path, Description: &description, NamespaceID: &integrationRootGroupIDProjects, + CIConfigPath: &configPath, } got, statusCode, err := c.AddProject(projectOpts) -- GitLab From 79005d350bb9238f02dce3b65d076f46165faa6d Mon Sep 17 00:00:00 2001 From: "adam.richards" Date: Fri, 25 Jun 2021 14:36:00 -0600 Subject: [PATCH 23/29] correct pipeline findings --- custom/p1/project.go | 14 +++++++------- custom/p1/types.go | 40 +++++++++++++++++++++++++++------------- custom/p1/types_test.go | 5 ++--- 3 files changed, 36 insertions(+), 23 deletions(-) diff --git a/custom/p1/project.go b/custom/p1/project.go index 3d50451..a808397 100644 --- a/custom/p1/project.go +++ b/custom/p1/project.go @@ -30,13 +30,13 @@ func CreateProject(projetCredentials ProjectCredentials, projectConfig ProjectCo return err } - if (statusCode == http.StatusNotFound ) { - return fmt.Errorf(fmt.Sprintf("Faild to find group with path %s",projectConfig.parentGroupFullPath)) + if statusCode == http.StatusNotFound { + return fmt.Errorf(fmt.Sprintf("Faild to find group with path %s", projectConfig.parentGroupFullPath)) } projectOptions := gogitlab.CreateProjectOptions{ - Name: &projectConfig.projectName, - NamespaceID: &parentGroup.ID, + Name: &projectConfig.projectName, + NamespaceID: &parentGroup.ID, CIConfigPath: &ciConfigPath, } @@ -58,8 +58,8 @@ func UpdateProject(projectConfig ProjectConfig) error { return nil } -func generateCIConfigPath(projectName string, parentGroupFullPath string) (string) { +func generateCIConfigPath(projectName string, parentGroupFullPath string) string { projectPathString, _ := GenerateGitlabPath(projectName) - ciConfigPath := fmt.Sprintf("%s/%s-ci.yml@platform-one/devops/pipeline-products",parentGroupFullPath,projectPathString) + ciConfigPath := fmt.Sprintf("%s/%s-ci.yml@platform-one/devops/pipeline-products", parentGroupFullPath, projectPathString) return ciConfigPath - } \ No newline at end of file +} diff --git a/custom/p1/types.go b/custom/p1/types.go index f3fde0c..ab4b2d2 100644 --- a/custom/p1/types.go +++ b/custom/p1/types.go @@ -10,11 +10,25 @@ type ProjectConfig struct { parentGroupFullPath string stackType StackType } + +// ProjectCredentials - type ProjectCredentials struct { - serverURL string - serverToken string + serverURL string + serverToken string } +// enumeration for pipeline yaml template paths +const ( + templatePathNpm = "templates/npm.yml" + templatePathYarn = "templates/yarn.yml" + templatePathGradle = "templates/gradle.yml" + templatePathMaven = "templates/maven.yml" + templatePathPip = "templates/pip.yml" + templatePathGoLang = "templates/go.yml" + templatePathCpp = "templates/cpp.yml" + templatePathUnknown = "" +) + // enumeration for tech stack types const ( StackTypeNodejsNpm = "StackTypeNodejsNpm" @@ -41,17 +55,17 @@ var StackTypes map[string]StackType = map[string]StackType{} // initialize stackTypes func init() { - StackTypes[StackTypeNodejsNpm] = StackType{stackType: StackTypeNodejsNpm, pipelineTemplatePath: "templates/npm.yml"} - StackTypes[StackTypeNodejsYarn] = StackType{stackType: StackTypeNodejsYarn, pipelineTemplatePath: "templates/yarn.yml"} - StackTypes[StackTypeJavaGradle] = StackType{stackType: StackTypeJavaGradle, pipelineTemplatePath: "templates/gradle.yml"} - StackTypes[StackTypeJavaMaven] = StackType{stackType: StackTypeJavaMaven, pipelineTemplatePath: "templates/maven.yml"} - StackTypes[StackTypePythonPip] = StackType{stackType: StackTypePythonPip, pipelineTemplatePath: "templates/maven.yml"} - StackTypes[StackTypePythonPipEnv] = StackType{stackType: StackTypePythonPipEnv, pipelineTemplatePath: "templates/maven.yml"} - StackTypes[StackTypeReact] = StackType{stackType: StackTypeReact, pipelineTemplatePath: "templates/npm.yml"} - StackTypes[StackTypeAngular] = StackType{stackType: StackTypeAngular, pipelineTemplatePath: "templates/npm.yml"} - StackTypes[StackTypeGoLang] = StackType{stackType: StackTypeGoLang, pipelineTemplatePath: "templates/go.yml"} - StackTypes[StackTypeCpp] = StackType{stackType: StackTypeCpp, pipelineTemplatePath: "templates/cpp.yml"} - StackTypes[StackTypeUnknown] = StackType{stackType: StackTypeUnknown, pipelineTemplatePath: ""} + StackTypes[StackTypeNodejsNpm] = StackType{stackType: StackTypeNodejsNpm, pipelineTemplatePath: templatePathNpm} + StackTypes[StackTypeNodejsYarn] = StackType{stackType: StackTypeNodejsYarn, pipelineTemplatePath: templatePathYarn} + StackTypes[StackTypeJavaGradle] = StackType{stackType: StackTypeJavaGradle, pipelineTemplatePath: templatePathGradle} + StackTypes[StackTypeJavaMaven] = StackType{stackType: StackTypeJavaMaven, pipelineTemplatePath: templatePathMaven} + StackTypes[StackTypePythonPip] = StackType{stackType: StackTypePythonPip, pipelineTemplatePath: templatePathPip} + StackTypes[StackTypePythonPipEnv] = StackType{stackType: StackTypePythonPipEnv, pipelineTemplatePath: templatePathPip} + StackTypes[StackTypeReact] = StackType{stackType: StackTypeReact, pipelineTemplatePath: templatePathNpm} + StackTypes[StackTypeAngular] = StackType{stackType: StackTypeAngular, pipelineTemplatePath: templatePathNpm} + StackTypes[StackTypeGoLang] = StackType{stackType: StackTypeGoLang, pipelineTemplatePath: templatePathGoLang} + StackTypes[StackTypeCpp] = StackType{stackType: StackTypeCpp, pipelineTemplatePath: templatePathCpp} + StackTypes[StackTypeUnknown] = StackType{stackType: StackTypeUnknown, pipelineTemplatePath: templatePathUnknown} } // CreateProjectConfig - diff --git a/custom/p1/types_test.go b/custom/p1/types_test.go index 645bf30..db0283a 100644 --- a/custom/p1/types_test.go +++ b/custom/p1/types_test.go @@ -43,10 +43,9 @@ func TestCreateProjectConfig(t *testing.T) { if !tt.wantErr { t.Errorf("CreateProjectConfig() error = %v, wantErr %v", err, tt.wantErr) return - } else { - t.Logf("CreateProjectConfig() received EXPECTED error = %v, wantErr %v", err, tt.wantErr) - return } + t.Logf("CreateProjectConfig() received EXPECTED error = %v, wantErr %v", err, tt.wantErr) + return } if !reflect.DeepEqual(got, tt.want) { t.Errorf("CreateProjectConfig() = %v, want %v", got, tt.want) -- GitLab From a9d9342f310142a5fa106a63bf75859fd49895ae Mon Sep 17 00:00:00 2001 From: "adam.richards" Date: Fri, 25 Jun 2021 15:45:25 -0600 Subject: [PATCH 24/29] add coverage --- clients/gitlab/client_test.go | 203 +++++++++++++++++++++++++++++++--- 1 file changed, 190 insertions(+), 13 deletions(-) diff --git a/clients/gitlab/client_test.go b/clients/gitlab/client_test.go index 9073ddc..98d6aca 100644 --- a/clients/gitlab/client_test.go +++ b/clients/gitlab/client_test.go @@ -116,6 +116,94 @@ func TestClient_GetUser(t *testing.T) { } } +func TestClient_GetUserByUsername(t *testing.T) { + // setup a http client for use in mocking + testHTTPClient := &http.Client{} + httpmock.ActivateNonDefault(testHTTPClient) + defer httpmock.DeactivateAndReset() + + testUserID := 1 + testUsername := "testusername" + testUser := gogitlab.User{ID: testUserID, Username: testUsername} + testUserArray := []*gogitlab.User{} + testUserArray = append(testUserArray, &testUser) + testUserArrayEmpty := []*gogitlab.User{} + + // test objects + testAPIUrl := "https://test/api/v4/" + testToken := "token" + // create a gitlab Client object, inject http client to allow for mocking using httpmock + testGitlabClient, _ := gogitlab.NewClient(testToken, gogitlab.WithBaseURL(testAPIUrl), gogitlab.WithHTTPClient(testHTTPClient)) + + type fields struct { + client *gogitlab.Client + token string + apiURL string + } + type args struct { + username *string + } + tests := []struct { + name string + fields fields + args args + want *gogitlab.User + want1 int + wantErr bool + }{ + { + name: "GetUserBuUsername 1", + fields: fields{client: testGitlabClient, token: testToken, apiURL: testAPIUrl}, + args: args{username: &testUsername}, + want: &testUser, + want1: http.StatusFound, + wantErr: false, + }, + { + name: "GetUserBuUsername 2", + fields: fields{client: testGitlabClient, token: testToken, apiURL: testAPIUrl}, + args: args{username: &testUsername}, + want: nil, + want1: http.StatusNotFound, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // default mock responder + httpmock.RegisterResponder("GET", + `=~^https://test/api/v4/users.*`, + httpmock.NewJsonResponderOrPanic(http.StatusOK, testUserArray), + ) + + // if we want to test no results found use this responder + if tt.want1 == http.StatusNotFound { + httpmock.RegisterResponder("GET", + `=~^https://test/api/v4/users.*`, + httpmock.NewJsonResponderOrPanic(http.StatusOK, testUserArrayEmpty), + ) + } + + r := Client{ + client: tt.fields.client, + token: tt.fields.token, + apiURL: tt.fields.apiURL, + } + got, got1, err := r.GetUserByUsername(tt.args.username) + if (err != nil) != tt.wantErr { + t.Errorf("Client.GetUserByUsername() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Client.GetUserByUsername() got = %v, want %v", got, tt.want) + } + if got1 != tt.want1 { + t.Errorf("Client.GetUserByUsername() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} + func TestClient_GetUsers(t *testing.T) { // setup a http client for use in mocking testHTTPClient := &http.Client{} @@ -431,6 +519,7 @@ func TestClient_DeleteUserByUsername(t *testing.T) { testUser := gogitlab.User{ID: 1, Username: "joedirt"} testUserArray := []*gogitlab.User{} testUserArray = append(testUserArray, &testUser) + testUserArrayEmpty := []*gogitlab.User{} counter := 0 httpmock.RegisterResponder("DELETE", @@ -440,18 +529,6 @@ func TestClient_DeleteUserByUsername(t *testing.T) { }, ) - // setup a mock that will change response on every 4th call - httpmock.RegisterResponder("GET", - `=~^https://test/api/v4/users.*`, - func(req *http.Request) (*http.Response, error) { - counter = counter + 1 - if counter%4 == 0 { - return httpmock.NewJsonResponse(404, testUserArray) - } - return httpmock.NewJsonResponse(202, testUserArray) - }, - ) - // test objects testUsername := "testusername" testAPIUrl := "https://test/api/v4/" @@ -478,11 +555,36 @@ func TestClient_DeleteUserByUsername(t *testing.T) { name: "DeleteUserByUsername 1", fields: fields{client: testGitlabClient, token: testToken, apiURL: testAPIUrl}, args: args{username: testUsername}, - want: 200, + want: http.StatusOK, + wantErr: false, + }, + { + name: "DeleteUserByUsername 2", + fields: fields{client: testGitlabClient, token: testToken, apiURL: testAPIUrl}, + args: args{username: testUsername}, + want: http.StatusNotFound, wantErr: false, }, } for _, tt := range tests { + // default responder + // setup a mock that will change response on every 4th call + httpmock.RegisterResponder("GET", + `=~^https://test/api/v4/users.*`, + func(req *http.Request) (*http.Response, error) { + counter = counter + 1 + if counter%4 == 0 { + return httpmock.NewJsonResponse(404, testUserArray) + } + return httpmock.NewJsonResponse(202, testUserArray) + }, + ) + if tt.want == http.StatusNotFound { + httpmock.RegisterResponder("GET", + `=~^https://test/api/v4/users.*`, + httpmock.NewJsonResponderOrPanic(http.StatusOK, testUserArrayEmpty), + ) + } t.Run(tt.name, func(t *testing.T) { r := Client{ client: tt.fields.client, @@ -570,6 +672,81 @@ func TestClient_GetGroup(t *testing.T) { }) } } + +func TestClient_GetGroupByFullPath(t *testing.T) { + // setup a http client for use in mocking + testHTTPClient := &http.Client{} + httpmock.ActivateNonDefault(testHTTPClient) + defer httpmock.DeactivateAndReset() + + // empty gitlab User array + testFullPath1 := "group1/group2" + testFullPath2 := "group1/group3" + testGroupArray := []*gogitlab.Group{} + testGroup1 := gogitlab.Group{ID: 1, Name: "joedirt", FullPath: testFullPath1} + testGroup2 := gogitlab.Group{ID: 2, Name: "spongebob", FullName: testFullPath2} + testGroupArray = append(testGroupArray, &testGroup1, &testGroup2) + + // use regex url matcher + httpmock.RegisterResponder("GET", + `=~^https://test/api/v4/groups.*`, + httpmock.NewJsonResponderOrPanic(http.StatusOK, testGroupArray), + ) + + // test objects + + testAPIUrl := "https://test/api/v4/" + testToken := "token" + // create a gitlab Client object, inject http client to allow for mocking using httpmock + testGitlabClient, _ := gogitlab.NewClient(testToken, gogitlab.WithBaseURL(testAPIUrl), gogitlab.WithHTTPClient(testHTTPClient)) + + type fields struct { + client *gogitlab.Client + token string + apiURL string + } + type args struct { + fullPath *string + } + tests := []struct { + name string + fields fields + args args + want *gogitlab.Group + want1 int + wantErr bool + }{ + { + name: "GetGroups 1", + fields: fields{client: testGitlabClient, token: testToken, apiURL: testAPIUrl}, + args: args{fullPath: &testFullPath1}, + want: &testGroup1, + want1: http.StatusFound, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := Client{ + client: tt.fields.client, + token: tt.fields.token, + apiURL: tt.fields.apiURL, + } + got, got1, err := r.GetGroupByFullPath(tt.args.fullPath) + if (err != nil) != tt.wantErr { + t.Errorf("Client.GetGroupByFullPath() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Client.GetGroupByFullPath() got = %v, want %v", got, tt.want) + } + if got1 != tt.want1 { + t.Errorf("Client.GetGroupByFullPath() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} + func TestClient_GetGroups(t *testing.T) { // setup a http client for use in mocking testHTTPClient := &http.Client{} -- GitLab From b164b8ef64c36c0b5f9114ac9e30b43a91f9db28 Mon Sep 17 00:00:00 2001 From: "adam.richards" Date: Sat, 26 Jun 2021 01:46:13 -0600 Subject: [PATCH 25/29] test coverage --- README.md | 2 + clients/gitlab/client.go | 2 +- clients/gitlab/client_test.go | 76 +++++++ .../custom/p1/p1_projects_test.go | 213 ++++++++++++++++++ 4 files changed, 292 insertions(+), 1 deletion(-) create mode 100644 integration-tests/custom/p1/p1_projects_test.go diff --git a/README.md b/README.md index f15ad9d..7d96782 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,8 @@ package integration ``` - run integration tests as follows - `go test -v -tags "integration" ./integration-tests/...` +- clean test cache + - `go clean -testcache` ### Twistlock Controller Setup 1) Create a secret that holds the credential for Twistlock API diff --git a/clients/gitlab/client.go b/clients/gitlab/client.go index 711663d..21374d6 100644 --- a/clients/gitlab/client.go +++ b/clients/gitlab/client.go @@ -446,7 +446,7 @@ func (r Client) AddGroupMember(groupID *int, userID *int, accessLevel gogitlab.A AccessLevel: &accessLevel, } - groupMember, res, err := r.client.GroupMembers.AddGroupMember(groupID, &opts) + groupMember, res, err := r.client.GroupMembers.AddGroupMember(*groupID, &opts) if err != nil { processError(logPrefix, err) return nil, 0, err diff --git a/clients/gitlab/client_test.go b/clients/gitlab/client_test.go index 98d6aca..3b0377b 100644 --- a/clients/gitlab/client_test.go +++ b/clients/gitlab/client_test.go @@ -891,6 +891,81 @@ func TestClient_AddGroup(t *testing.T) { } } +func TestClient_AddGroupMember(t *testing.T) { + // setup a http client for use in mocking + testHTTPClient := &http.Client{} + httpmock.ActivateNonDefault(testHTTPClient) + defer httpmock.DeactivateAndReset() + + // empty gitlab User array + testGroupID := 1 + testUserID := 1 + // testGroupName := "test group name" + // testGroup := gogitlab.Group{ID: testGroupID, Name: testGroupName} + testAccessLevel := gogitlab.OwnerPermissions + testGroupMember := gogitlab.GroupMember{ID: testUserID} + + httpmock.RegisterResponder("POST", + `=~^https://test/api/v4/groups.*`, + httpmock.NewJsonResponderOrPanic(200, testGroupMember), + ) + + // test objects + testAPIUrl := "https://test/api/v4/" + testToken := "token" + // create a gitlab Client object, inject http client to allow for mocking using httpmock + testGitlabClient, _ := gogitlab.NewClient(testToken, gogitlab.WithBaseURL(testAPIUrl), gogitlab.WithHTTPClient(testHTTPClient)) + + type fields struct { + client *gogitlab.Client + token string + apiURL string + } + type args struct { + groupID *int + userID *int + accessLevel gogitlab.AccessLevelValue + } + tests := []struct { + name string + fields fields + args args + want *gogitlab.GroupMember + want1 int + wantErr bool + }{ + { + name: "AddGroupMember 1", + fields: fields{client: testGitlabClient, token: testToken, apiURL: testAPIUrl}, + args: args{groupID: &testGroupID, userID: &testUserID, accessLevel:testAccessLevel }, + want: &testGroupMember, + want1: http.StatusOK, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := Client{ + client: tt.fields.client, + token: tt.fields.token, + apiURL: tt.fields.apiURL, + } + got, got1, err := r.AddGroupMember(tt.args.groupID, tt.args.userID, tt.args.accessLevel) + if (err != nil) != tt.wantErr { + t.Errorf("Client.AddGroupMember() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Client.AddGroupMember() got = %v, want %v", got, tt.want) + } + if got1 != tt.want1 { + t.Errorf("Client.AddGroupMember() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} + + func TestClient_DeleteGroup(t *testing.T) { // setup a http client for use in mocking @@ -1176,3 +1251,4 @@ func TestClient_AddProject(t *testing.T) { }) } } + diff --git a/integration-tests/custom/p1/p1_projects_test.go b/integration-tests/custom/p1/p1_projects_test.go new file mode 100644 index 0000000..a14fd85 --- /dev/null +++ b/integration-tests/custom/p1/p1_projects_test.go @@ -0,0 +1,213 @@ +// +build integration + +package integration + +import ( + "errors" + "fmt" + "net/http" + "os" + "testing" + + gogitlab "github.com/xanzy/go-gitlab" + gitlab "valkyrie.dso.mil/valkyrie-api/clients/gitlab" +) + +const projectsTestItemCount = 65 +const testProjectsPrefix = "IntTestProject" + +var integrationRootGroupIDProjects = 0 + +// getClientProjects - +func getClientProjects() (gitlab.Client, error) { + gitlabAPIURL, ok := os.LookupEnv("GITLAB_API_URL") + if !ok { + return gitlab.Client{}, errors.New("env variable GITLAB_API_URL undefinded") + } + gitlabAPIToken, ok := os.LookupEnv("GITLAB_API_TOKEN") + if !ok { + return gitlab.Client{}, errors.New("env variable GITLAB_API_TOKEN undefinded") + } + var client = gitlab.NewClient(gitlabAPIURL, gitlabAPIToken, nil) + return client, nil +} + +func TestClient_GetProjects(t *testing.T) { + t.Run("test", func(t *testing.T) { + c, err := getClientProjects() + if err != nil { + t.Errorf("Client.getClientProjects() error = %v", err) + return + } + got, err := c.GetProjects(nil) + if err != nil { + t.Errorf("Client.GetProjects() error = %v", err) + return + } + t.Logf("GetProjects - number of projects %d", len(got)) + }) +} + +func TestClient_GetProjects_Search(t *testing.T) { + t.Run("test", func(t *testing.T) { + c, err := getClientProjects() + if err != nil { + t.Errorf("Client.getClientProjects() error = %v", err) + return + } + + searchString := "monitor" + + got, err := c.GetProjects(&searchString) + if err != nil { + t.Errorf("Client.GetProjects() error = %v", err) + return + } + t.Logf("GetProjects - number of projects %d", len(got)) + }) +} + +// Insure the integration test root level group exists +func TestClient_AddRootGroup_Projects(t *testing.T) { + t.Run("test", func(t *testing.T) { + c, err := getClientProjects() + if err != nil { + t.Errorf("Client.getClientProjects() error = %v", err) + return + } + + path := "integration-root-group" + name := "integration root group" + description := "Integration Test Root Level Group" + + group, statusCode, err := c.GetGroupByFullPath(&path) + if err != nil { + // dont fail if group already exists + t.Errorf("Client.GetGroupByFullPath() error = %v %d", err, statusCode) + return + } + + if statusCode == http.StatusFound { + integrationRootGroupIDProjects = group.ID + t.Logf("AddGroup %s %s %s %s %d", group.Name, group.Path, group.FullPath, group.WebURL, statusCode) + } else { + groupOpts := gogitlab.CreateGroupOptions{Name: &name, Path: &path, Description: &description} + got, statusCode, err := c.AddGroup(groupOpts) + + if err != nil { + // dont fail if group already exists + t.Logf("Client.AddGroup() error = %v %d", err, statusCode) + } else { + integrationRootGroupIDProjects = got.ID + t.Logf("AddGroup %s %s %s %s %d", got.Name, got.Path, got.FullPath, got.WebURL, statusCode) + } + } + }) +} + +// add groups under the integration test root level group +func TestClient_AddProjects(t *testing.T) { + t.Run("test", func(t *testing.T) { + c, err := getClientProjects() + if err != nil { + t.Errorf("Client.getClientProjects() error = %v", err) + return + } + + for i := 1; i <= projectsTestItemCount; i++ { + path := testProjectsPrefix + "-project-" + fmt.Sprintf("%d", i) + name := testProjectsPrefix + " project " + fmt.Sprintf("%d", i) + description := "My fancy test project " + fmt.Sprintf("%d", i) + configPath := "a/b/config.yml@group1/group2/project" + + projectOpts := gogitlab.CreateProjectOptions{ + Name: &name, + Path: &path, + Description: &description, + NamespaceID: &integrationRootGroupIDProjects, + CIConfigPath: &configPath, + } + got, statusCode, err := c.AddProject(projectOpts) + + if err != nil { + t.Errorf("Client.AddProject() error = %v %d", err, statusCode) + } else { + t.Logf("AddProject %s %s %s %s %d", got.Name, got.Path, got.PathWithNamespace, got.WebURL, statusCode) + } + } + + }) +} + +func TestClient_UpdateProject(t *testing.T) { + t.Run("test", func(t *testing.T) { + c, err := getClientProjects() + if err != nil { + t.Errorf("Client.getClientProjects() error = %v", err) + return + } + + // create a group to contain test objects + path := testProjectsPrefix + "-project-update" + name := testProjectsPrefix + " project update" + description := "My fancy test project " + + projectOpts := gogitlab.CreateProjectOptions{ + Name: &name, + Path: &path, + Description: &description, + NamespaceID: &integrationRootGroupIDProjects, + } + newProject, statusCode, err := c.AddProject(projectOpts) + if err != nil { + t.Errorf("Client.AddProject() error = %v", err) + return + } + t.Logf(" created project %s path %s status code %d", newProject.Name, newProject.PathWithNamespace, statusCode) + + // update user + testDescription := "Updated Description" + updateProjectOptions := gogitlab.EditProjectOptions{Description: &testDescription} + updatedProject, statusCode, err := c.UpdateProject(newProject.ID, updateProjectOptions) + if err != nil { + t.Errorf("Client.UpdateProject() error = %v", err) + return + } + t.Logf("UpdateProject: %s %d", updatedProject.Description, statusCode) + + // cleanup + statusCode, _ = c.DeleteProject(newProject.ID, 0, 0) + t.Logf("DeleteProject no wait statusCode: %d", statusCode) + + }) +} + + +func TestClient_deleteProjects(t *testing.T) { + t.Run("test", func(t *testing.T) { + c, err := getClientProjects() + if err != nil { + t.Errorf("Client.getClient() error = %v", err) + return + } + + search := testProjectsPrefix + projectList, err := c.GetProjects(&search) + if err != nil { + t.Errorf("Client.GetProjects() error = %v", err) + return + } + + for x := 0; x < len(projectList); x++ { + project := projectList[x] + statusCode, err := c.DeleteProject(project.ID, 100, 1200) + + if err != nil { + t.Errorf("Client.DeleteProject() error = %v %d", err, statusCode) + } else { + t.Logf("DeleteProject %s %d", project.Name, statusCode) + } + } + + }) +} -- GitLab From 9f4e12858031e4c8ccce3a0401d4a9ce0b2a3469 Mon Sep 17 00:00:00 2001 From: "adam.richards" Date: Mon, 28 Jun 2021 11:00:05 -0600 Subject: [PATCH 26/29] addproject and deleteproject for P1 --- clients/gitlab/client_test.go | 150 ++++++++++++++++++++++++++++++++-- custom/p1/project.go | 74 +++++++++++++---- custom/p1/types.go | 83 ++++++++----------- custom/p1/types_test.go | 44 +++------- 4 files changed, 251 insertions(+), 100 deletions(-) diff --git a/clients/gitlab/client_test.go b/clients/gitlab/client_test.go index 3b0377b..f991e15 100644 --- a/clients/gitlab/client_test.go +++ b/clients/gitlab/client_test.go @@ -906,7 +906,7 @@ func TestClient_AddGroupMember(t *testing.T) { testGroupMember := gogitlab.GroupMember{ID: testUserID} httpmock.RegisterResponder("POST", - `=~^https://test/api/v4/groups.*`, + `=~^https://test/api/v4/groups.*`, httpmock.NewJsonResponderOrPanic(200, testGroupMember), ) @@ -937,9 +937,9 @@ func TestClient_AddGroupMember(t *testing.T) { { name: "AddGroupMember 1", fields: fields{client: testGitlabClient, token: testToken, apiURL: testAPIUrl}, - args: args{groupID: &testGroupID, userID: &testUserID, accessLevel:testAccessLevel }, + args: args{groupID: &testGroupID, userID: &testUserID, accessLevel: testAccessLevel}, want: &testGroupMember, - want1: http.StatusOK, + want1: http.StatusOK, wantErr: false, }, } @@ -965,6 +965,75 @@ func TestClient_AddGroupMember(t *testing.T) { } } +func TestClient_UpdateGroup(t *testing.T) { + // setup a http client for use in mocking + testHTTPClient := &http.Client{} + httpmock.ActivateNonDefault(testHTTPClient) + defer httpmock.DeactivateAndReset() + + testGroupID := 1 + testUpdatedName := "test group UPDATED name" + testUpdatedGroup := gogitlab.Group{ID: testGroupID, Name: testUpdatedName} + + httpmock.RegisterResponder("PUT", + `=~^https://test/api/v4/groups.*`, + httpmock.NewJsonResponderOrPanic(http.StatusOK, testUpdatedGroup), + ) + + // test objects + testAPIUrl := "https://test/api/v4/" + testToken := "token" + // create a gitlab Client object, inject http client to allow for mocking using httpmock + testGitlabClient, _ := gogitlab.NewClient(testToken, gogitlab.WithBaseURL(testAPIUrl), gogitlab.WithHTTPClient(testHTTPClient)) + + type fields struct { + client *gogitlab.Client + token string + apiURL string + } + type args struct { + groupID int + updateGroupOptions *gogitlab.UpdateGroupOptions + } + tests := []struct { + name string + fields fields + args args + want *gogitlab.Group + want1 int + wantErr bool + }{ + { + name: "UpdateGroup 1", + fields: fields{client: testGitlabClient, token: testToken, apiURL: testAPIUrl}, + args: args{groupID: testGroupID, updateGroupOptions: &gogitlab.UpdateGroupOptions{Name: &testUpdatedName}}, + want: &testUpdatedGroup, + want1: http.StatusOK, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := Client{ + client: tt.fields.client, + token: tt.fields.token, + apiURL: tt.fields.apiURL, + } + got, got1, err := r.UpdateGroup(tt.args.groupID, tt.args.updateGroupOptions) + if (err != nil) != tt.wantErr { + t.Errorf("Client.UpdateGroup() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Client.UpdateGroup() got = %v, want %v", got, tt.want) + } + if got1 != tt.want1 { + t.Errorf("Client.UpdateGroup() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} func TestClient_DeleteGroup(t *testing.T) { @@ -1017,7 +1086,7 @@ func TestClient_DeleteGroup(t *testing.T) { name: "DeleteGroup 1", fields: fields{client: testGitlabClient, token: testToken, apiURL: testAPIUrl}, args: args{groupID: testGroupID}, - want: 200, + want: http.StatusOK, wantErr: false, }, } @@ -1181,6 +1250,78 @@ func TestClient_GetProjects(t *testing.T) { } } +func TestClient_GetProjectByFullPath(t *testing.T) { + // setup a http client for use in mocking + testHTTPClient := &http.Client{} + httpmock.ActivateNonDefault(testHTTPClient) + defer httpmock.DeactivateAndReset() + + // empty gitlab User array + testProjectArray := []*gogitlab.Project{} + testProjectFullPath1 := "root/group1/group2/project1" + testProjectFullPath2 := "root/group1/project2" + testProject1 := gogitlab.Project{ID: 1, Name: "joedirt", PathWithNamespace: testProjectFullPath1} + testProject2 := gogitlab.Project{ID: 2, Name: "spongebob", PathWithNamespace: testProjectFullPath2} + testProjectArray = append(testProjectArray, &testProject1, &testProject2) + + // use regex url matcher + httpmock.RegisterResponder("GET", + `=~^https://test/api/v4/projects.*`, + httpmock.NewJsonResponderOrPanic(http.StatusOK, testProjectArray), + ) + // test objects + testAPIUrl := "https://test/api/v4/" + testToken := "token" + // create a gitlab Client object, inject http client to allow for mocking using httpmock + testGitlabClient, _ := gogitlab.NewClient(testToken, gogitlab.WithBaseURL(testAPIUrl), gogitlab.WithHTTPClient(testHTTPClient)) + + type fields struct { + client *gogitlab.Client + token string + apiURL string + } + type args struct { + fullPath *string + } + tests := []struct { + name string + fields fields + args args + want *gogitlab.Project + want1 int + wantErr bool + }{ + { + name: "GetProjects 1", + fields: fields{client: testGitlabClient, token: testToken, apiURL: testAPIUrl}, + args: args{fullPath: &testProjectFullPath1}, + want: &testProject1, + want1: http.StatusFound, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := Client{ + client: tt.fields.client, + token: tt.fields.token, + apiURL: tt.fields.apiURL, + } + got, got1, err := r.GetProjectByFullPath(tt.args.fullPath) + if (err != nil) != tt.wantErr { + t.Errorf("Client.GetProjectByFullPath() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Client.GetProjectByFullPath() got = %v, want %v", got, tt.want) + } + if got1 != tt.want1 { + t.Errorf("Client.GetProjectByFullPath() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} + func TestClient_AddProject(t *testing.T) { // setup a http client for use in mocking @@ -1251,4 +1392,3 @@ func TestClient_AddProject(t *testing.T) { }) } } - diff --git a/custom/p1/project.go b/custom/p1/project.go index a808397..911d37b 100644 --- a/custom/p1/project.go +++ b/custom/p1/project.go @@ -6,55 +6,97 @@ import ( "github.com/romana/rlog" gogitlab "github.com/xanzy/go-gitlab" + apigitlab "valkyrie.dso.mil/valkyrie-api/apis/gitlab/v1alpha1" gitlab "valkyrie.dso.mil/valkyrie-api/clients/gitlab" ) -func logStart(logPrefix string, projectConfig ProjectConfig) { +func logStart(logPrefix string, projectSpec apigitlab.ProjectSpec) { rlog.Debugf("%s start. projectConfig %s %s %s", logPrefix, - projectConfig.projectName, - projectConfig.parentGroupFullPath, - projectConfig.stackType.pipelineTemplatePath) + projectSpec.Name, + projectSpec.GroupID, + projectSpec.Language) } // CreateProject - -func CreateProject(projetCredentials ProjectCredentials, projectConfig ProjectConfig) error { +func CreateProject(projetCredentials ProjectCredentials, projectSpec apigitlab.ProjectSpec, httpClient *http.Client) error { logPrefix := "CreateProject" - logStart(logPrefix, projectConfig) - ciConfigPath := generateCIConfigPath(projectConfig.projectName, projectConfig.parentGroupFullPath) + logStart(logPrefix, projectSpec) - client := gitlab.NewClient(projetCredentials.serverURL, projetCredentials.serverToken, nil) + client := gitlab.NewClient(projetCredentials.serverURL, projetCredentials.serverToken, httpClient) - parentGroup, statusCode, err := client.GetGroupByFullPath(&projectConfig.parentGroupFullPath) + group, _, err := client.GetGroup(int(projectSpec.GroupID)) + if err != nil { + return fmt.Errorf(fmt.Sprintf("Faild to find group with ID %d", int(projectSpec.GroupID))) + } + parentGroupFullPath := group.FullPath + + ciConfigPath := generateCIConfigPath(projectSpec.Name, parentGroupFullPath) + + parentGroup, statusCode, err := client.GetGroupByFullPath(&parentGroupFullPath) if err != nil { return err } if statusCode == http.StatusNotFound { - return fmt.Errorf(fmt.Sprintf("Faild to find group with path %s", projectConfig.parentGroupFullPath)) + return fmt.Errorf(fmt.Sprintf("Faild to find group with path %s", parentGroupFullPath)) } projectOptions := gogitlab.CreateProjectOptions{ - Name: &projectConfig.projectName, + Name: &projectSpec.Name, NamespaceID: &parentGroup.ID, CIConfigPath: &ciConfigPath, } - client.AddProject(projectOptions) + // TODO: add or check branch with group name to pipeline-products + // TODO: add initial product yaml file on that branch based on language type + // TODO: add MR to pipeline product to create project-ci.yml file that POINTS to the branch + + project, statusCode, err := client.AddProject(projectOptions) + if err != nil { + return err + } + rlog.Debugf("Added project %s to path %s with status code %d", project.Name, project.NameWithNamespace, statusCode) + return nil } // DeleteProject - -func DeleteProject(projectConfig ProjectConfig) error { +func DeleteProject(projetCredentials ProjectCredentials, projectSpec apigitlab.ProjectSpec, httpClient *http.Client) error { logPrefix := "DeleteProject" - logStart(logPrefix, projectConfig) + logStart(logPrefix, projectSpec) + + client := gitlab.NewClient(projetCredentials.serverURL, projetCredentials.serverToken, httpClient) + group, _, err := client.GetGroup(int(projectSpec.GroupID)) + if err != nil { + return fmt.Errorf(fmt.Sprintf("Faild to find group with ID %d", int(projectSpec.GroupID))) + } + parentGroupFullPath := group.FullPath + // get scrubbed pathname based on actual name + projectPathString, _ := GenerateGitlabPath(projectSpec.Name) + projectFullPath := parentGroupFullPath + "/" + projectPathString + project, statusCode, err := client.GetProjectByFullPath(&projectFullPath) + if err != nil { + rlog.Errorf("Error in %s %v", logPrefix, err) + return err + } + if statusCode != http.StatusFound { + return fmt.Errorf(fmt.Sprintf("Failed to find project with full path %s", projectFullPath)) + } + + _, err = client.DeleteProject(project.ID, 0, 0) + + if err != nil { + rlog.Errorf("Error in %s %v", logPrefix, err) + return fmt.Errorf(fmt.Sprintf("Faild to delete project with ID %d", int(project.ID))) + } return nil } // UpdateProject - -func UpdateProject(projectConfig ProjectConfig) error { +func UpdateProject(projectSpec apigitlab.ProjectSpec) error { logPrefix := "UpdateProject" - logStart(logPrefix, projectConfig) + logStart(logPrefix, projectSpec) return nil } diff --git a/custom/p1/types.go b/custom/p1/types.go index ab4b2d2..ec10834 100644 --- a/custom/p1/types.go +++ b/custom/p1/types.go @@ -4,13 +4,6 @@ import ( "fmt" ) -// ProjectConfig - -type ProjectConfig struct { - projectName string - parentGroupFullPath string - stackType StackType -} - // ProjectCredentials - type ProjectCredentials struct { serverURL string @@ -31,59 +24,53 @@ const ( // enumeration for tech stack types const ( - StackTypeNodejsNpm = "StackTypeNodejsNpm" - StackTypeNodejsYarn = "StackTypeNodejsYarn" - StackTypeJavaGradle = "StackTypeJavaGradle" - StackTypeJavaMaven = "StackTypeJavaMaven" - StackTypePythonPip = "StackTypePythonPip" - StackTypePythonPipEnv = "StackTypePythonPipEnv" - StackTypeReact = "StackTypeReact" - StackTypeAngular = "StackTypeAngular" - StackTypeGoLang = "StackTypeGoLang" - StackTypeCpp = "StackTypeCpp" - StackTypeUnknown = "StackTypeUnknown" + LangTypeNodejsNpm = "LangTypeNodejsNpm" + LangTypeNodejsYarn = "LangTypeNodejsYarn" + LangTypeJavaGradle = "LangTypeJavaGradle" + LangTypeJavaMaven = "LangTypeJavaMaven" + LangTypePythonPip = "LangTypePythonPip" + LangTypePythonPipEnv = "LangTypePythonPipEnv" + LangTypeReact = "LangTypeReact" + LangTypeAngular = "LangTypeAngular" + LangTypeGoLang = "LangTypeGoLang" + LangTypeCpp = "LangTypeCpp" + LangTypeUnknown = "LangTypeUnknown" ) -// StackType - -type StackType struct { - stackType string +// LangType - +type LangType struct { + langType string pipelineTemplatePath string } -// StackTypes - map of available stack types in P1 and corresponding pipeline template path -var StackTypes map[string]StackType = map[string]StackType{} +// LangTypes - map of available stack types in P1 and corresponding pipeline template path +var LangTypes map[string]LangType = map[string]LangType{} -// initialize stackTypes +// initialize langTypes func init() { - StackTypes[StackTypeNodejsNpm] = StackType{stackType: StackTypeNodejsNpm, pipelineTemplatePath: templatePathNpm} - StackTypes[StackTypeNodejsYarn] = StackType{stackType: StackTypeNodejsYarn, pipelineTemplatePath: templatePathYarn} - StackTypes[StackTypeJavaGradle] = StackType{stackType: StackTypeJavaGradle, pipelineTemplatePath: templatePathGradle} - StackTypes[StackTypeJavaMaven] = StackType{stackType: StackTypeJavaMaven, pipelineTemplatePath: templatePathMaven} - StackTypes[StackTypePythonPip] = StackType{stackType: StackTypePythonPip, pipelineTemplatePath: templatePathPip} - StackTypes[StackTypePythonPipEnv] = StackType{stackType: StackTypePythonPipEnv, pipelineTemplatePath: templatePathPip} - StackTypes[StackTypeReact] = StackType{stackType: StackTypeReact, pipelineTemplatePath: templatePathNpm} - StackTypes[StackTypeAngular] = StackType{stackType: StackTypeAngular, pipelineTemplatePath: templatePathNpm} - StackTypes[StackTypeGoLang] = StackType{stackType: StackTypeGoLang, pipelineTemplatePath: templatePathGoLang} - StackTypes[StackTypeCpp] = StackType{stackType: StackTypeCpp, pipelineTemplatePath: templatePathCpp} - StackTypes[StackTypeUnknown] = StackType{stackType: StackTypeUnknown, pipelineTemplatePath: templatePathUnknown} + LangTypes[LangTypeNodejsNpm] = LangType{langType: LangTypeNodejsNpm, pipelineTemplatePath: templatePathNpm} + LangTypes[LangTypeNodejsYarn] = LangType{langType: LangTypeNodejsYarn, pipelineTemplatePath: templatePathYarn} + LangTypes[LangTypeJavaGradle] = LangType{langType: LangTypeJavaGradle, pipelineTemplatePath: templatePathGradle} + LangTypes[LangTypeJavaMaven] = LangType{langType: LangTypeJavaMaven, pipelineTemplatePath: templatePathMaven} + LangTypes[LangTypePythonPip] = LangType{langType: LangTypePythonPip, pipelineTemplatePath: templatePathPip} + LangTypes[LangTypePythonPipEnv] = LangType{langType: LangTypePythonPipEnv, pipelineTemplatePath: templatePathPip} + LangTypes[LangTypeReact] = LangType{langType: LangTypeReact, pipelineTemplatePath: templatePathNpm} + LangTypes[LangTypeAngular] = LangType{langType: LangTypeAngular, pipelineTemplatePath: templatePathNpm} + LangTypes[LangTypeGoLang] = LangType{langType: LangTypeGoLang, pipelineTemplatePath: templatePathGoLang} + LangTypes[LangTypeCpp] = LangType{langType: LangTypeCpp, pipelineTemplatePath: templatePathCpp} + LangTypes[LangTypeUnknown] = LangType{langType: LangTypeUnknown, pipelineTemplatePath: templatePathUnknown} } -// CreateProjectConfig - -func CreateProjectConfig(name string, path string, stackTypeName string) (ProjectConfig, error) { - stackType, ok := StackTypes[stackTypeName] +// GetProjectLanguage - +func GetProjectLanguage(langTypeName string) (LangType, error) { + langType, ok := LangTypes[langTypeName] if !ok { - keys := make([]string, 0, len(StackTypes)) - for k := range StackTypes { + keys := make([]string, 0, len(LangTypes)) + for k := range LangTypes { keys = append(keys, k) } keysStr := fmt.Sprintf("valid keys %v", keys) - return ProjectConfig{}, error(fmt.Errorf(keysStr)) - } - - projectConfig := ProjectConfig{ - projectName: name, - parentGroupFullPath: path, - stackType: stackType, + return LangType{}, error(fmt.Errorf(keysStr)) } - return projectConfig, nil + return langType, nil } diff --git a/custom/p1/types_test.go b/custom/p1/types_test.go index db0283a..677c001 100644 --- a/custom/p1/types_test.go +++ b/custom/p1/types_test.go @@ -5,50 +5,32 @@ import ( "testing" ) -func TestCreateProjectConfig(t *testing.T) { +func TestGetProjectLanguage(t *testing.T) { type args struct { - name string - path string - stackTypeName string + langTypeName string } - testName := "test project name" - testGroupPath := "root/group1/group2" - testStackType := "StackTypeNodejsNpm" - testStackTypeNotOK := "badindex" - - testProjectConfigOK := ProjectConfig{ - projectName: testName, - parentGroupFullPath: testGroupPath, - stackType: StackTypes[testStackType], - } - testProjectConfigNotOK := ProjectConfig{ - projectName: testName, - parentGroupFullPath: testGroupPath, - stackType: StackTypes[testStackTypeNotOK], - } - tests := []struct { name string args args - want ProjectConfig + want LangType wantErr bool }{ - {name: "config 1", args: args{name: testName, path: testGroupPath, stackTypeName: testStackType}, want: testProjectConfigOK, wantErr: false}, - {name: "config 2", args: args{name: testName, path: testGroupPath, stackTypeName: testStackTypeNotOK}, want: testProjectConfigNotOK, wantErr: true}, + { + name: "project language test", + args: args{langTypeName: LangTypeCpp}, + want: LangTypes[LangTypeCpp], + wantErr: false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := CreateProjectConfig(tt.args.name, tt.args.path, tt.args.stackTypeName) - if err != nil { - if !tt.wantErr { - t.Errorf("CreateProjectConfig() error = %v, wantErr %v", err, tt.wantErr) - return - } - t.Logf("CreateProjectConfig() received EXPECTED error = %v, wantErr %v", err, tt.wantErr) + got, err := GetProjectLanguage(tt.args.langTypeName) + if (err != nil) != tt.wantErr { + t.Errorf("GetProjectLanguage() error = %v, wantErr %v", err, tt.wantErr) return } if !reflect.DeepEqual(got, tt.want) { - t.Errorf("CreateProjectConfig() = %v, want %v", got, tt.want) + t.Errorf("GetProjectLanguage() = %v, want %v", got, tt.want) } }) } -- GitLab From 666a45ace089c86bc7cda52a354493ccc3cbcac6 Mon Sep 17 00:00:00 2001 From: "adam.richards" Date: Mon, 28 Jun 2021 11:55:31 -0600 Subject: [PATCH 27/29] additional coverage --- clients/gitlab/client_test.go | 191 ++++++++++++++++++++++++++++++++-- 1 file changed, 182 insertions(+), 9 deletions(-) diff --git a/clients/gitlab/client_test.go b/clients/gitlab/client_test.go index f991e15..4e19255 100644 --- a/clients/gitlab/client_test.go +++ b/clients/gitlab/client_test.go @@ -473,6 +473,8 @@ func TestClient_DeleteUser(t *testing.T) { } type args struct { userID int + waitInterval int + waitCount int } tests := []struct { name string @@ -482,10 +484,17 @@ func TestClient_DeleteUser(t *testing.T) { wantErr bool }{ { - name: "DeleteUser 1", + name: "DeleteUser Success", fields: fields{client: testGitlabClient, token: testToken, apiURL: testAPIUrl}, - args: args{userID: testUserID}, - want: 200, + args: args{userID: testUserID, waitInterval: 1000, waitCount: 10}, + want: http.StatusOK, + wantErr: false, + }, + { + name: "DeleteUser Timeout", + fields: fields{client: testGitlabClient, token: testToken, apiURL: testAPIUrl}, + args: args{userID: testUserID, waitInterval: 1000, waitCount: 1}, + want: http.StatusRequestTimeout, wantErr: false, }, } @@ -496,7 +505,7 @@ func TestClient_DeleteUser(t *testing.T) { token: tt.fields.token, apiURL: tt.fields.apiURL, } - got, err := r.DeleteUser(tt.args.userID, 1000, 10) + got, err := r.DeleteUser(tt.args.userID, tt.args.waitInterval, tt.args.waitCount) if (err != nil) != tt.wantErr { t.Errorf("Client.DeleteUser() error = %v, wantErr %v", err, tt.wantErr) return @@ -552,14 +561,14 @@ func TestClient_DeleteUserByUsername(t *testing.T) { wantErr bool }{ { - name: "DeleteUserByUsername 1", + name: "DeleteUserByUsername Success", fields: fields{client: testGitlabClient, token: testToken, apiURL: testAPIUrl}, args: args{username: testUsername}, want: http.StatusOK, wantErr: false, }, { - name: "DeleteUserByUsername 2", + name: "DeleteUserByUsername Not Found", fields: fields{client: testGitlabClient, token: testToken, apiURL: testAPIUrl}, args: args{username: testUsername}, want: http.StatusNotFound, @@ -1074,6 +1083,8 @@ func TestClient_DeleteGroup(t *testing.T) { } type args struct { groupID int + waitInterval int + waitCount int } tests := []struct { name string @@ -1083,12 +1094,19 @@ func TestClient_DeleteGroup(t *testing.T) { wantErr bool }{ { - name: "DeleteGroup 1", + name: "DeleteGroup Success", fields: fields{client: testGitlabClient, token: testToken, apiURL: testAPIUrl}, - args: args{groupID: testGroupID}, + args: args{groupID: testGroupID, waitInterval: 1000, waitCount: 10}, want: http.StatusOK, wantErr: false, }, + { + name: "DeleteGroup Timeout", + fields: fields{client: testGitlabClient, token: testToken, apiURL: testAPIUrl}, + args: args{groupID: testGroupID, waitInterval: 1000, waitCount: 1}, + want: http.StatusRequestTimeout, + wantErr: false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -1097,7 +1115,7 @@ func TestClient_DeleteGroup(t *testing.T) { token: tt.fields.token, apiURL: tt.fields.apiURL, } - got, err := r.DeleteGroup(tt.args.groupID, 1000, 10) + got, err := r.DeleteGroup(tt.args.groupID, tt.args.waitInterval, tt.args.waitCount) if (err != nil) != tt.wantErr { t.Errorf("Client.DeleteGroup() error = %v, wantErr %v", err, tt.wantErr) return @@ -1392,3 +1410,158 @@ func TestClient_AddProject(t *testing.T) { }) } } + +func TestClient_UpdateProject(t *testing.T) { + + // setup a http client for use in mocking + testHTTPClient := &http.Client{} + httpmock.ActivateNonDefault(testHTTPClient) + defer httpmock.DeactivateAndReset() + + testProjectID := 1 + testUpdatedProjectName := "testProjectUPDATEDName" + testUpdatedProject := gogitlab.Project{ID: testProjectID, Name: testUpdatedProjectName} + + httpmock.RegisterResponder("PUT", + `=~^https://test/api/v4/project.*`, + func(req *http.Request) (*http.Response, error) { + return httpmock.NewJsonResponse(http.StatusOK, testUpdatedProject) + }, + ) + + // test objects + testAPIUrl := "https://test/api/v4/" + testToken := "token" + // create a gitlab Client object, inject http client to allow for mocking using httpmock + testGitlabClient, _ := gogitlab.NewClient(testToken, gogitlab.WithBaseURL(testAPIUrl), gogitlab.WithHTTPClient(testHTTPClient)) + + type fields struct { + client *gogitlab.Client + token string + apiURL string + } + type args struct { + projectID int + editProjectOptions gogitlab.EditProjectOptions + } + tests := []struct { + name string + fields fields + args args + want *gogitlab.Project + want1 int + wantErr bool + }{ + { + name: "UpdateProject Success", + fields: fields{client: testGitlabClient, token: testToken, apiURL: testAPIUrl}, + args: args{projectID: testProjectID}, + want: &testUpdatedProject, + want1: http.StatusOK, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := Client{ + client: tt.fields.client, + token: tt.fields.token, + apiURL: tt.fields.apiURL, + } + got, got1, err := r.UpdateProject(tt.args.projectID, tt.args.editProjectOptions) + if (err != nil) != tt.wantErr { + t.Errorf("Client.UpdateProject() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Client.UpdateProject() got = %v, want %v", got, tt.want) + } + if got1 != tt.want1 { + t.Errorf("Client.UpdateProject() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} + +func TestClient_DeleteProject(t *testing.T) { + // setup a http client for use in mocking + testHTTPClient := &http.Client{} + httpmock.ActivateNonDefault(testHTTPClient) + defer httpmock.DeactivateAndReset() + + testProjectID := 1 + testProjectName := "testProjectName" + testProject1 := gogitlab.Project{ID: testProjectID, Name: testProjectName} + + var counter = 0 + httpmock.RegisterResponder("DELETE", + `=~^https://test/api/v4/project.*`, + func(req *http.Request) (*http.Response, error) { + return httpmock.NewJsonResponse(201, testProject1) + }, + ) + httpmock.RegisterResponder("GET", + `=~^https://test/api/v4/projects.*`, + func(req *http.Request) (*http.Response, error) { + counter = counter + 1 + if counter%4 == 0 { + return httpmock.NewJsonResponse(404, testProject1) + } + return httpmock.NewJsonResponse(202, testProject1) + }) + // test objects + testAPIUrl := "https://test/api/v4/" + testToken := "token" + // create a gitlab Client object, inject http client to allow for mocking using httpmock + testGitlabClient, _ := gogitlab.NewClient(testToken, gogitlab.WithBaseURL(testAPIUrl), gogitlab.WithHTTPClient(testHTTPClient)) + + type fields struct { + client *gogitlab.Client + token string + apiURL string + } + type args struct { + projectID int + waitInterval int + waitCount int + } + tests := []struct { + name string + fields fields + args args + want int + wantErr bool + }{ + { + name: "DeleteProject Success", + fields: fields{client: testGitlabClient, token: testToken, apiURL: testAPIUrl}, + args: args{projectID: testProjectID, waitInterval: 1000, waitCount: 10}, + want: http.StatusOK, + wantErr: false, + }, + { + name: "DeleteProject Timeout", + fields: fields{client: testGitlabClient, token: testToken, apiURL: testAPIUrl}, + args: args{projectID: testProjectID, waitInterval: 1000, waitCount: 1}, + want: http.StatusRequestTimeout, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := Client{ + client: tt.fields.client, + token: tt.fields.token, + apiURL: tt.fields.apiURL, + } + got, err := r.DeleteProject(tt.args.projectID, tt.args.waitInterval, tt.args.waitCount) + if (err != nil) != tt.wantErr { + t.Errorf("Client.DeleteProject() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("Client.DeleteProject() = %v, want %v", got, tt.want) + } + }) + } +} -- GitLab From 586aa1659b22650ee1dfb21f6c645571298d7388 Mon Sep 17 00:00:00 2001 From: "adam.richards" Date: Mon, 28 Jun 2021 16:41:43 -0600 Subject: [PATCH 28/29] p1 project code to interface with controller --- clients/gitlab/client_test.go | 8 +- custom/p1/project.go | 71 ++++-- custom/p1/types.go | 4 +- custom/p1/utils.go | 5 +- integration-tests/custom/p1/p1_main_test.go | 122 ++++++++++ .../custom/p1/p1_projects_test.go | 212 ++++-------------- 6 files changed, 230 insertions(+), 192 deletions(-) create mode 100644 integration-tests/custom/p1/p1_main_test.go diff --git a/clients/gitlab/client_test.go b/clients/gitlab/client_test.go index 4e19255..ecde48d 100644 --- a/clients/gitlab/client_test.go +++ b/clients/gitlab/client_test.go @@ -472,9 +472,9 @@ func TestClient_DeleteUser(t *testing.T) { apiURL string } type args struct { - userID int + userID int waitInterval int - waitCount int + waitCount int } tests := []struct { name string @@ -1082,9 +1082,9 @@ func TestClient_DeleteGroup(t *testing.T) { apiURL string } type args struct { - groupID int + groupID int waitInterval int - waitCount int + waitCount int } tests := []struct { name string diff --git a/custom/p1/project.go b/custom/p1/project.go index 911d37b..3c5c22a 100644 --- a/custom/p1/project.go +++ b/custom/p1/project.go @@ -19,29 +19,32 @@ func logStart(logPrefix string, projectSpec apigitlab.ProjectSpec) { } // CreateProject - -func CreateProject(projetCredentials ProjectCredentials, projectSpec apigitlab.ProjectSpec, httpClient *http.Client) error { +func CreateProject(projectCredentials ProjectCredentials, projectSpec apigitlab.ProjectSpec, httpClient *http.Client) error { logPrefix := "CreateProject" logStart(logPrefix, projectSpec) - client := gitlab.NewClient(projetCredentials.serverURL, projetCredentials.serverToken, httpClient) + client := gitlab.NewClient(projectCredentials.ServerURL, projectCredentials.ServerToken, httpClient) - group, _, err := client.GetGroup(int(projectSpec.GroupID)) + // confirm parent group by ID + group, statusCode, err := client.GetGroup(int(projectSpec.GroupID)) if err != nil { - return fmt.Errorf(fmt.Sprintf("Faild to find group with ID %d", int(projectSpec.GroupID))) + return fmt.Errorf(fmt.Sprintf("Failed to find group with ID %d. error: %v", int(projectSpec.GroupID), err)) + } + if statusCode != http.StatusOK { + return fmt.Errorf(fmt.Sprintf("Failed to find group with ID %d. status code: %d", int(projectSpec.GroupID), statusCode)) } - parentGroupFullPath := group.FullPath - - ciConfigPath := generateCIConfigPath(projectSpec.Name, parentGroupFullPath) + // confirm parent group full path + parentGroupFullPath := group.FullPath parentGroup, statusCode, err := client.GetGroupByFullPath(&parentGroupFullPath) if err != nil { - return err + return fmt.Errorf(fmt.Sprintf("Failed to find group with ID %d. error: %v", int(projectSpec.GroupID), err)) } - - if statusCode == http.StatusNotFound { - return fmt.Errorf(fmt.Sprintf("Faild to find group with path %s", parentGroupFullPath)) + if statusCode != http.StatusFound { + return fmt.Errorf(fmt.Sprintf("Failed to find group with ID %d. status code: %d", int(projectSpec.GroupID), statusCode)) } + ciConfigPath := generateCIConfigPath(projectSpec.Name, parentGroupFullPath) projectOptions := gogitlab.CreateProjectOptions{ Name: &projectSpec.Name, NamespaceID: &parentGroup.ID, @@ -54,7 +57,11 @@ func CreateProject(projetCredentials ProjectCredentials, projectSpec apigitlab.P project, statusCode, err := client.AddProject(projectOptions) if err != nil { - return err + return fmt.Errorf(fmt.Sprintf("Failed to add project with name %s to group path %s. error: %v", *projectOptions.Name, parentGroupFullPath, err)) + } + if statusCode != http.StatusCreated { + return fmt.Errorf(fmt.Sprintf("Failed to add project with name %s to group path %s. statusCode: %d", *projectOptions.Name, parentGroupFullPath, statusCode)) + } rlog.Debugf("Added project %s to path %s with status code %d", project.Name, project.NameWithNamespace, statusCode) @@ -62,11 +69,11 @@ func CreateProject(projetCredentials ProjectCredentials, projectSpec apigitlab.P } // DeleteProject - -func DeleteProject(projetCredentials ProjectCredentials, projectSpec apigitlab.ProjectSpec, httpClient *http.Client) error { +func DeleteProject(projectCredentials ProjectCredentials, projectSpec apigitlab.ProjectSpec, httpClient *http.Client) error { logPrefix := "DeleteProject" logStart(logPrefix, projectSpec) - client := gitlab.NewClient(projetCredentials.serverURL, projetCredentials.serverToken, httpClient) + client := gitlab.NewClient(projectCredentials.ServerURL, projectCredentials.ServerToken, httpClient) group, _, err := client.GetGroup(int(projectSpec.GroupID)) if err != nil { return fmt.Errorf(fmt.Sprintf("Faild to find group with ID %d", int(projectSpec.GroupID))) @@ -94,9 +101,43 @@ func DeleteProject(projetCredentials ProjectCredentials, projectSpec apigitlab.P } // UpdateProject - -func UpdateProject(projectSpec apigitlab.ProjectSpec) error { +func UpdateProject(projectCredentials ProjectCredentials, projectSpec apigitlab.ProjectSpec, httpClient *http.Client) error { logPrefix := "UpdateProject" logStart(logPrefix, projectSpec) + + client := gitlab.NewClient(projectCredentials.ServerURL, projectCredentials.ServerToken, httpClient) + + // Identify parent Group for project + group, _, err := client.GetGroup(int(projectSpec.GroupID)) + if err != nil { + return fmt.Errorf(fmt.Sprintf("Faild to find group with ID %d", int(projectSpec.GroupID))) + } + parentGroupFullPath := group.FullPath + // get scrubbed pathname based on actual name + projectPathString, _ := GenerateGitlabPath(projectSpec.Name) + projectFullPath := parentGroupFullPath + "/" + projectPathString + project, statusCode, err := client.GetProjectByFullPath(&projectFullPath) + if err != nil { + rlog.Errorf("Error in %s %v", logPrefix, err) + return err + } + if statusCode != http.StatusFound { + return fmt.Errorf(fmt.Sprintf("Failed to find project with full path %s", projectFullPath)) + } + + ciConfigPath := generateCIConfigPath(projectSpec.Name, parentGroupFullPath) + projectOptions := gogitlab.EditProjectOptions{ + Name: &projectSpec.Name, + CIConfigPath: &ciConfigPath, + } + + project, statusCode, err = client.UpdateProject(project.ID, projectOptions) + + if err != nil { + rlog.Errorf("Error in %s %v", logPrefix, err) + return fmt.Errorf(fmt.Sprintf("Faild to update project with ID %d error: %v", int(project.ID), err)) + } + rlog.Debugf("Updated project %s with status code %d", project.Name, statusCode) return nil } diff --git a/custom/p1/types.go b/custom/p1/types.go index ec10834..c73ab78 100644 --- a/custom/p1/types.go +++ b/custom/p1/types.go @@ -6,8 +6,8 @@ import ( // ProjectCredentials - type ProjectCredentials struct { - serverURL string - serverToken string + ServerURL string + ServerToken string } // enumeration for pipeline yaml template paths diff --git a/custom/p1/utils.go b/custom/p1/utils.go index 4f3400b..4cc2117 100644 --- a/custom/p1/utils.go +++ b/custom/p1/utils.go @@ -2,6 +2,7 @@ package p1 import ( "regexp" + "strings" ) // GenerateGitlabPath - use gitlab name to generate a path @@ -21,5 +22,7 @@ func GenerateGitlabPath(name string) (string, error) { re, _ = regexp.Compile(`\s+`) path = re.ReplaceAllString(path, "-") - return path, nil + pathLower := strings.ToLower(path) + + return pathLower, nil } diff --git a/integration-tests/custom/p1/p1_main_test.go b/integration-tests/custom/p1/p1_main_test.go new file mode 100644 index 0000000..de480b8 --- /dev/null +++ b/integration-tests/custom/p1/p1_main_test.go @@ -0,0 +1,122 @@ + +// +build integration + +package integration + +import ( + "errors" + "fmt" + "net/http" + "os" + "testing" + + "github.com/romana/rlog" + gogitlab "github.com/xanzy/go-gitlab" + gitlab "valkyrie.dso.mil/valkyrie-api/clients/gitlab" +) + +const p1IntegrationRootGroupPath = "integration-root-group" + +var P1Config struct { + gitlabAPIURL string + gitlabAPIToken string + testProjectGroupPath string + integrationRootGroupID int +} + +// TestMain - setup and teardown reusable code. wraps around other tests +func TestMain(m *testing.M) { + err := packageSetup() + if err != nil { + fmt.Printf("packageSetup error: %v", err) + os.Exit(1) + } + resultCode := m.Run() + packageTeardown() + os.Exit(resultCode) +} + +func packageSetup() error { + var err error + // initialize + P1Config.testProjectGroupPath = p1IntegrationRootGroupPath + err = getEnvVariables() + if err != nil { + return err + } + err = addRootGroup() + if err != nil { + return err + } + return nil +} + +func packageTeardown() { + +} + +// getEnvVariables - read environment variables +func getEnvVariables() error { + gitlabAPIURL, ok := os.LookupEnv("GITLAB_API_URL") + if !ok { + return errors.New("env variable GITLAB_API_URL undefinded") + } + + gitlabAPIToken, ok := os.LookupEnv("GITLAB_API_TOKEN") + if !ok { + return errors.New("env variable GITLAB_API_TOKEN undefinded") + } + P1Config.gitlabAPIToken = gitlabAPIToken + P1Config.gitlabAPIURL = gitlabAPIURL + return nil +} + +// addRootGroup - add a integration test root group if it does not exist +func addRootGroup() error { + + c, err := getClient() + if err != nil { + return err + } + + // create root integration test group if does not yet exists + path := P1Config.testProjectGroupPath + name := "integration root group" + description := "Integration Test Root Level Group" + + group, statusCode, err := c.GetGroupByFullPath(&path) + if err != nil { + return err + } + + if statusCode == http.StatusFound { + P1Config.integrationRootGroupID = group.ID + rlog.Debugf("AddGroup %s %s %s %s %d", group.Name, group.Path, group.FullPath, group.WebURL, statusCode) + } else { + groupOpts := gogitlab.CreateGroupOptions{Name: &name, Path: &path, Description: &description} + got, statusCode, err := c.AddGroup(groupOpts) + + if err != nil { + // dont fail if group already exists + rlog.Warnf("AddGroup() error = %v %d", err, statusCode) + } else { + P1Config.integrationRootGroupID = got.ID + rlog.Debugf("AddGroup %s %s %s %s %d", got.Name, got.Path, got.FullPath, got.WebURL, statusCode) + } + } + return nil +} + +// getClientGroups - +func getClient() (gitlab.Client, error) { + gitlabAPIURL, ok := os.LookupEnv("GITLAB_API_URL") + if !ok { + return gitlab.Client{}, errors.New("env variable GITLAB_API_URL undefinded") + } + gitlabAPIToken, ok := os.LookupEnv("GITLAB_API_TOKEN") + if !ok { + return gitlab.Client{}, errors.New("env variable GITLAB_API_TOKEN undefinded") + } + var client = gitlab.NewClient(gitlabAPIURL, gitlabAPIToken, nil) + return client, nil +} diff --git a/integration-tests/custom/p1/p1_projects_test.go b/integration-tests/custom/p1/p1_projects_test.go index a14fd85..ded9acd 100644 --- a/integration-tests/custom/p1/p1_projects_test.go +++ b/integration-tests/custom/p1/p1_projects_test.go @@ -3,211 +3,83 @@ package integration import ( - "errors" - "fmt" - "net/http" - "os" "testing" - gogitlab "github.com/xanzy/go-gitlab" - gitlab "valkyrie.dso.mil/valkyrie-api/clients/gitlab" + "valkyrie.dso.mil/valkyrie-api/apis/gitlab/v1alpha1" + custom_p1 "valkyrie.dso.mil/valkyrie-api/custom/p1" ) -const projectsTestItemCount = 65 -const testProjectsPrefix = "IntTestProject" - -var integrationRootGroupIDProjects = 0 - -// getClientProjects - -func getClientProjects() (gitlab.Client, error) { - gitlabAPIURL, ok := os.LookupEnv("GITLAB_API_URL") - if !ok { - return gitlab.Client{}, errors.New("env variable GITLAB_API_URL undefinded") - } - gitlabAPIToken, ok := os.LookupEnv("GITLAB_API_TOKEN") - if !ok { - return gitlab.Client{}, errors.New("env variable GITLAB_API_TOKEN undefinded") - } - var client = gitlab.NewClient(gitlabAPIURL, gitlabAPIToken, nil) - return client, nil -} +/* -func TestClient_GetProjects(t *testing.T) { - t.Run("test", func(t *testing.T) { - c, err := getClientProjects() - if err != nil { - t.Errorf("Client.getClientProjects() error = %v", err) - return - } - got, err := c.GetProjects(nil) - if err != nil { - t.Errorf("Client.GetProjects() error = %v", err) - return - } - t.Logf("GetProjects - number of projects %d", len(got)) - }) -} +P1Config is initialized in the TestMain wrapper method -func TestClient_GetProjects_Search(t *testing.T) { - t.Run("test", func(t *testing.T) { - c, err := getClientProjects() - if err != nil { - t.Errorf("Client.getClientProjects() error = %v", err) - return - } - - searchString := "monitor" - - got, err := c.GetProjects(&searchString) - if err != nil { - t.Errorf("Client.GetProjects() error = %v", err) - return - } - t.Logf("GetProjects - number of projects %d", len(got)) - }) -} +*/ -// Insure the integration test root level group exists -func TestClient_AddRootGroup_Projects(t *testing.T) { +func Test_P1_AddProject(t *testing.T) { + logPrefix := "Test_P1_AddProject" t.Run("test", func(t *testing.T) { - c, err := getClientProjects() - if err != nil { - t.Errorf("Client.getClientProjects() error = %v", err) - return - } - - path := "integration-root-group" - name := "integration root group" - description := "Integration Test Root Level Group" - - group, statusCode, err := c.GetGroupByFullPath(&path) - if err != nil { - // dont fail if group already exists - t.Errorf("Client.GetGroupByFullPath() error = %v %d", err, statusCode) - return + projectSpec := v1alpha1.ProjectSpec{ + Name: "My Test Project", + GroupID: int64(P1Config.integrationRootGroupID), + Language: custom_p1.LangTypeAngular, } - if statusCode == http.StatusFound { - integrationRootGroupIDProjects = group.ID - t.Logf("AddGroup %s %s %s %s %d", group.Name, group.Path, group.FullPath, group.WebURL, statusCode) - } else { - groupOpts := gogitlab.CreateGroupOptions{Name: &name, Path: &path, Description: &description} - got, statusCode, err := c.AddGroup(groupOpts) - - if err != nil { - // dont fail if group already exists - t.Logf("Client.AddGroup() error = %v %d", err, statusCode) - } else { - integrationRootGroupIDProjects = got.ID - t.Logf("AddGroup %s %s %s %s %d", got.Name, got.Path, got.FullPath, got.WebURL, statusCode) - } + creds := custom_p1.ProjectCredentials{ + ServerURL: P1Config.gitlabAPIURL, + ServerToken: P1Config.gitlabAPIToken, } - }) -} -// add groups under the integration test root level group -func TestClient_AddProjects(t *testing.T) { - t.Run("test", func(t *testing.T) { - c, err := getClientProjects() + err := custom_p1.CreateProject(creds, projectSpec, nil) if err != nil { - t.Errorf("Client.getClientProjects() error = %v", err) + t.Errorf("CreateProject() error = %v", err) return } - for i := 1; i <= projectsTestItemCount; i++ { - path := testProjectsPrefix + "-project-" + fmt.Sprintf("%d", i) - name := testProjectsPrefix + " project " + fmt.Sprintf("%d", i) - description := "My fancy test project " + fmt.Sprintf("%d", i) - configPath := "a/b/config.yml@group1/group2/project" - - projectOpts := gogitlab.CreateProjectOptions{ - Name: &name, - Path: &path, - Description: &description, - NamespaceID: &integrationRootGroupIDProjects, - CIConfigPath: &configPath, - } - got, statusCode, err := c.AddProject(projectOpts) - - if err != nil { - t.Errorf("Client.AddProject() error = %v %d", err, statusCode) - } else { - t.Logf("AddProject %s %s %s %s %d", got.Name, got.Path, got.PathWithNamespace, got.WebURL, statusCode) - } - } - + t.Logf("%s created project %v", logPrefix, projectSpec) }) } -func TestClient_UpdateProject(t *testing.T) { +func Test_P1_UpdateProject(t *testing.T) { + logPrefix := "Test_P1_UpdateProject" t.Run("test", func(t *testing.T) { - c, err := getClientProjects() - if err != nil { - t.Errorf("Client.getClientProjects() error = %v", err) - return + projectSpec := v1alpha1.ProjectSpec{ + Name: "My Test Project", + GroupID: int64(P1Config.integrationRootGroupID), + Language: custom_p1.LangTypeJavaMaven, } - // create a group to contain test objects - path := testProjectsPrefix + "-project-update" - name := testProjectsPrefix + " project update" - description := "My fancy test project " - - projectOpts := gogitlab.CreateProjectOptions{ - Name: &name, - Path: &path, - Description: &description, - NamespaceID: &integrationRootGroupIDProjects, + creds := custom_p1.ProjectCredentials{ + ServerURL: P1Config.gitlabAPIURL, + ServerToken: P1Config.gitlabAPIToken, } - newProject, statusCode, err := c.AddProject(projectOpts) - if err != nil { - t.Errorf("Client.AddProject() error = %v", err) - return - } - t.Logf(" created project %s path %s status code %d", newProject.Name, newProject.PathWithNamespace, statusCode) - // update user - testDescription := "Updated Description" - updateProjectOptions := gogitlab.EditProjectOptions{Description: &testDescription} - updatedProject, statusCode, err := c.UpdateProject(newProject.ID, updateProjectOptions) + err := custom_p1.UpdateProject(creds, projectSpec, nil) if err != nil { - t.Errorf("Client.UpdateProject() error = %v", err) + t.Errorf("UpdateProject() error = %v", err) return } - t.Logf("UpdateProject: %s %d", updatedProject.Description, statusCode) - - // cleanup - statusCode, _ = c.DeleteProject(newProject.ID, 0, 0) - t.Logf("DeleteProject no wait statusCode: %d", statusCode) - + t.Logf("%s update project %v", logPrefix, projectSpec) }) } - -func TestClient_deleteProjects(t *testing.T) { +func Test_P1_DeleteProject(t *testing.T) { + logPrefix := "Test_P1_DeleteProject" t.Run("test", func(t *testing.T) { - c, err := getClientProjects() - if err != nil { - t.Errorf("Client.getClient() error = %v", err) - return + projectSpec := v1alpha1.ProjectSpec{ + Name: "My Test Project", + GroupID: int64(P1Config.integrationRootGroupID), } - search := testProjectsPrefix - projectList, err := c.GetProjects(&search) - if err != nil { - t.Errorf("Client.GetProjects() error = %v", err) - return + creds := custom_p1.ProjectCredentials{ + ServerURL: P1Config.gitlabAPIURL, + ServerToken: P1Config.gitlabAPIToken, } - for x := 0; x < len(projectList); x++ { - project := projectList[x] - statusCode, err := c.DeleteProject(project.ID, 100, 1200) - - if err != nil { - t.Errorf("Client.DeleteProject() error = %v %d", err, statusCode) - } else { - t.Logf("DeleteProject %s %d", project.Name, statusCode) - } + err := custom_p1.DeleteProject(creds, projectSpec, nil) + if err != nil { + t.Errorf("DeleteProject() error = %v", err) + return } - + t.Logf("%s deleted project %v", logPrefix, projectSpec) }) } -- GitLab From 985910382511296e2d709fde268ed6d05203adbe Mon Sep 17 00:00:00 2001 From: "adam.richards" Date: Mon, 28 Jun 2021 16:52:22 -0600 Subject: [PATCH 29/29] sonarqube findings --- custom/p1/project.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/custom/p1/project.go b/custom/p1/project.go index 3c5c22a..c4c6064 100644 --- a/custom/p1/project.go +++ b/custom/p1/project.go @@ -10,6 +10,11 @@ import ( gitlab "valkyrie.dso.mil/valkyrie-api/clients/gitlab" ) +// processError - helper +func processError(logPrefix string, err error) { + rlog.Warnf("%s error: %v", logPrefix, err) +} + func logStart(logPrefix string, projectSpec apigitlab.ProjectSpec) { rlog.Debugf("%s start. projectConfig %s %s %s", logPrefix, @@ -84,7 +89,7 @@ func DeleteProject(projectCredentials ProjectCredentials, projectSpec apigitlab. projectFullPath := parentGroupFullPath + "/" + projectPathString project, statusCode, err := client.GetProjectByFullPath(&projectFullPath) if err != nil { - rlog.Errorf("Error in %s %v", logPrefix, err) + processError(logPrefix, err) return err } if statusCode != http.StatusFound { @@ -94,7 +99,6 @@ func DeleteProject(projectCredentials ProjectCredentials, projectSpec apigitlab. _, err = client.DeleteProject(project.ID, 0, 0) if err != nil { - rlog.Errorf("Error in %s %v", logPrefix, err) return fmt.Errorf(fmt.Sprintf("Faild to delete project with ID %d", int(project.ID))) } return nil @@ -110,6 +114,7 @@ func UpdateProject(projectCredentials ProjectCredentials, projectSpec apigitlab. // Identify parent Group for project group, _, err := client.GetGroup(int(projectSpec.GroupID)) if err != nil { + processError(logPrefix, err) return fmt.Errorf(fmt.Sprintf("Faild to find group with ID %d", int(projectSpec.GroupID))) } parentGroupFullPath := group.FullPath @@ -118,7 +123,7 @@ func UpdateProject(projectCredentials ProjectCredentials, projectSpec apigitlab. projectFullPath := parentGroupFullPath + "/" + projectPathString project, statusCode, err := client.GetProjectByFullPath(&projectFullPath) if err != nil { - rlog.Errorf("Error in %s %v", logPrefix, err) + processError(logPrefix, err) return err } if statusCode != http.StatusFound { @@ -134,7 +139,7 @@ func UpdateProject(projectCredentials ProjectCredentials, projectSpec apigitlab. project, statusCode, err = client.UpdateProject(project.ID, projectOptions) if err != nil { - rlog.Errorf("Error in %s %v", logPrefix, err) + processError(logPrefix, err) return fmt.Errorf(fmt.Sprintf("Faild to update project with ID %d error: %v", int(project.ID), err)) } rlog.Debugf("Updated project %s with status code %d", project.Name, statusCode) -- GitLab