diff --git a/apis/gitlab/clientConfiguration.go b/apis/gitlab/clientConfiguration.go index ccd22b858f17d28b9a2d4dfb36f2871584b3304b..39d6fb01d9823fc516810824b56df53f8e91d60b 100755 --- a/apis/gitlab/clientConfiguration.go +++ b/apis/gitlab/clientConfiguration.go @@ -7,7 +7,7 @@ import ( "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - gitlabv1alpha1 "valkyrie.dso.mil/valkyrie-api/apis/gitlab/v1alpha1" + gitlabAPIVer "valkyrie.dso.mil/valkyrie-api/apis/gitlab/v1alpha1" gitlabClient "valkyrie.dso.mil/valkyrie-api/clients/gitlab" ) @@ -20,7 +20,7 @@ const ( // ClientConfiguration represents an object whose purpose is to setup a Gitlab client given type ClientConfiguration interface { // SetupClient pulls the GitlabCredentials from Kubernetes and supplies the Gitlab user and access token. - SetupClient(client client.Client, credentialsName string) (gitlabClient.Client, error) + SetupClient(client client.Client, credentialsName string) (gitlabClient.Client, *gitlabAPIVer.GitlabCredentials, error) } // ClientConfigurationImpl is the default implementation for the ClientConfiguration interface @@ -33,17 +33,17 @@ type ClientConfigurationImpl struct { // SetupClient looks up the gitlab credentials to get the access token and username for connecting to Gitlab // on behalf of the reconciler. -func (c ClientConfigurationImpl) SetupClient(client client.Client, credentialsName string) (gitlabClient.Client, error) { +func (c ClientConfigurationImpl) SetupClient(client client.Client, credentialsName string) (gitlabClient.Client, *gitlabAPIVer.GitlabCredentials, error) { // Get the Gitlab Credentials var gitLabCredentialsName = types.NamespacedName{ Namespace: c.Req.Namespace, Name: credentialsName, } - var gitlabCredentials gitlabv1alpha1.GitlabCredentials + var gitlabCredentials gitlabAPIVer.GitlabCredentials if err := client.Get(c.Ctx, gitLabCredentialsName, &gitlabCredentials); err != nil { c.Log.Error(err, errorUnableToFetchGitlabCredentials) - return nil, err + return nil, nil, err } var secretName = types.NamespacedName{ @@ -55,19 +55,20 @@ func (c ClientConfigurationImpl) SetupClient(client client.Client, credentialsNa // Get the Secret if err := client.Get(c.Ctx, secretName, &secret); err != nil { c.Log.Error(err, errorUnableToFetchSecret) - return nil, err + return nil, nil, err } // Login to Gitlab var accessToken = string(secret.Data[gitlabCredentials.Spec.AccessTokenKey]) + gitlabCredentials.Spec.AccessTokenPrivate = accessToken var returnClient gitlabClient.Client var err error if returnClient, err = gitlabClient.NewClient(gitlabCredentials.Spec.URL, accessToken, nil); err != nil { c.Log.Error(err, errorUnableToCreateGitlabClient, "username", gitlabCredentials.Spec.Username, "url", gitlabCredentials.Spec.URL) - return nil, err + return nil, nil, err } - return returnClient, nil + return returnClient, &gitlabCredentials, nil } diff --git a/apis/gitlab/v1alpha1/common_types.go b/apis/gitlab/v1alpha1/common_types.go index efbbb609185c8a7b285787d631fb2a08b02c48fe..44fe221d88daf3aa85d73e4a4e2f1c82d5ce1f29 100755 --- a/apis/gitlab/v1alpha1/common_types.go +++ b/apis/gitlab/v1alpha1/common_types.go @@ -33,11 +33,11 @@ type BootstrapWithManifestRepo struct { // Language is a string key to an existing pre-built code stack // +kubebuilder:validation:Required - Services []services `json:"services"` + Services []Service `json:"services"` } -// services (private) manifest representation of a deployable container -type services struct { +// Service manifest representation of a deployable container +type Service struct { Name string `json:"name"` ContainerRegistry string `json:"containerRegistry"` Port int `json:"port"` diff --git a/apis/gitlab/v1alpha1/gitlabcredentials_types.go b/apis/gitlab/v1alpha1/gitlabcredentials_types.go index 161386a22f869b8bb6a6765c97b6cbb6adc6f567..8181c029da5ca3ddb4374ac7649d7af807738be4 100644 --- a/apis/gitlab/v1alpha1/gitlabcredentials_types.go +++ b/apis/gitlab/v1alpha1/gitlabcredentials_types.go @@ -35,6 +35,9 @@ type GitlabCredentialsSpec struct { //AccessTokenKey is the key of the secret data that contains the Gitlab Access Token for the user. AccessTokenKey string `json:"accessTokenKey"` + + //AccessTokenPrivate omit json marshalling. + AccessTokenPrivate string `json:"-"` } // GitlabCredentialsStatus defines the observed state of GitlabCredentials diff --git a/apis/gitlab/v1alpha1/project_types_test.go b/apis/gitlab/v1alpha1/project_types_test.go index 86fb853dd49c9e5e100f510d60ef1fced6558c3b..7f9014fad2d2830d1ae0281e687305730ff020b7 100644 --- a/apis/gitlab/v1alpha1/project_types_test.go +++ b/apis/gitlab/v1alpha1/project_types_test.go @@ -248,7 +248,7 @@ func TestDeepCopy_BoostrapManifest(t *testing.T) { registry := "test22" port := 9999 toCopy := BootstrapWithManifestRepo{ - Services: []services{ + Services: []Service{ { Name: name, ContainerRegistry: registry, diff --git a/apis/gitlab/v1alpha1/zz_generated.deepcopy.go b/apis/gitlab/v1alpha1/zz_generated.deepcopy.go index bf45adcbcc4c311b235bf4fbc2cd61110eb55ac7..c9e0c75d78fa4a75827dcf2afbdabe8f2761f96a 100644 --- a/apis/gitlab/v1alpha1/zz_generated.deepcopy.go +++ b/apis/gitlab/v1alpha1/zz_generated.deepcopy.go @@ -31,7 +31,7 @@ func (in *BootstrapWithManifestRepo) DeepCopyInto(out *BootstrapWithManifestRepo *out = *in if in.Services != nil { in, out := &in.Services, &out.Services - *out = make([]services, len(*in)) + *out = make([]Service, len(*in)) copy(*out, *in) } } @@ -727,3 +727,18 @@ func (in *SdElementsPipelineConfigurationStatus) DeepCopy() *SdElementsPipelineC in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Service) DeepCopyInto(out *Service) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Service. +func (in *Service) DeepCopy() *Service { + if in == nil { + return nil + } + out := new(Service) + in.DeepCopyInto(out) + return out +} diff --git a/clients/git/repo_client.go b/clients/git/repo_client.go index 2e8def062f38f77ee1151d4b6b61513bb7701492..30cb422d329040bb130504b76a55e24f5f281762 100644 --- a/clients/git/repo_client.go +++ b/clients/git/repo_client.go @@ -106,6 +106,10 @@ func (c *RepoClientImpl) CheckoutOrCreateBranch(branchName string) (err error) { RemoteName: remoteName, RefSpecs: []config.RefSpec{"refs/*:refs/*", "HEAD:refs/heads/HEAD"}, }) + //if already up-to-date, then it's no an error + if err != nil && err.Error() == "already up-to-date" { + err = nil + } if err != nil { rlog.Errorf("CheckoutOrCreateBranch - Fetch - Error %v", err) return diff --git a/clients/git/repo_client_test.go b/clients/git/repo_client_test.go index 2dc4b7965c0788cdfbd8be78db64b006381f0d67..36825401955b13d3f43ff004a0da26442b3e9444 100644 --- a/clients/git/repo_client_test.go +++ b/clients/git/repo_client_test.go @@ -1,6 +1,7 @@ package git import ( + "reflect" "testing" "github.com/go-git/go-billy/v5/memfs" @@ -34,6 +35,30 @@ func TestRepoClientImpl_CheckoutOrCreateBranch(t *testing.T) { } } +func TestRepoClientImpl_Push(t *testing.T) { + fileSystem := memfs.New() + auth := &http.BasicAuth{ + Username: "username", + Password: "token", + } + memStorage := memory.NewStorage() + repo, err := git.Init(memStorage, fileSystem) + if err != nil { + panic(err) + } + client := RepoClientImpl{ + repo: repo, + fs: &fileSystem, + auth: auth, + } + + branchName := "test_branch" + err = client.Push(branchName) + if err == nil { + t.Fatal("expected an error when creating the branch") + } +} + func TestRepoClientImpl_AddFile(t *testing.T) { fileSystem := memfs.New() auth := &http.BasicAuth{ @@ -123,3 +148,90 @@ func TestRepoClientImpl_DeleteRemoteBranch(t *testing.T) { t.Fatalf("expect an error but got %v", err) } } + +func Test_newClientFS(t *testing.T) { + type args struct { + username string + token string + repoURL string + memoryFs bool + osPath string + } + tests := []struct { + name string + args args + wantClient *RepoClientImpl + wantErr bool + }{ + {name: "test 1", args: args{username: "username", token: "token", repoURL: "http://site/path", memoryFs: true, osPath: ""}, wantClient: nil, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotClient, err := newClientFS(tt.args.username, tt.args.token, tt.args.repoURL, tt.args.memoryFs, tt.args.osPath) + if (err != nil) != tt.wantErr { + t.Errorf("newClientFS() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(gotClient, tt.wantClient) { + t.Errorf("newClientFS() = %v, want %v", gotClient, tt.wantClient) + } + }) + } +} + +func TestNewClient(t *testing.T) { + type args struct { + username string + token string + repoURL string + } + tests := []struct { + name string + args args + wantClient *RepoClientImpl + wantErr bool + }{ + {name: "test 1", args: args{username: "username", token: "token", repoURL: "http://site/path"}, wantClient: nil, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotClient, err := NewClient(tt.args.username, tt.args.token, tt.args.repoURL) + if (err != nil) != tt.wantErr { + t.Errorf("NewClient() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(gotClient, tt.wantClient) { + t.Errorf("NewClient() = %v, want %v", gotClient, tt.wantClient) + } + }) + } +} + +func TestNewClientOSFS(t *testing.T) { + type args struct { + username string + token string + repoURL string + osPath string + } + tests := []struct { + name string + args args + wantClient *RepoClientImpl + wantErr bool + }{ + {name: "test 1", args: args{username: "username", token: "token", repoURL: "http://site/path", osPath: "/tmp"}, wantClient: nil, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotClient, err := NewClientOSFS(tt.args.username, tt.args.token, tt.args.repoURL, tt.args.osPath) + if (err != nil) != tt.wantErr { + t.Errorf("NewClientOSFS() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(gotClient, tt.wantClient) { + t.Errorf("NewClientOSFS() = %v, want %v", gotClient, tt.wantClient) + } + }) + } +} diff --git a/clients/utils/datetime_utils_test.go b/clients/utils/datetime_utils_test.go index e41d5c952d278460799c653775cc141f42382376..4eadc939b427b249081afde2c05f6dff4e23eeae 100644 --- a/clients/utils/datetime_utils_test.go +++ b/clients/utils/datetime_utils_test.go @@ -23,8 +23,5 @@ func TestDateTime_Test_1(t *testing.T) { } func checkString(v *string) bool { - if v == nil { - return false - } - return true + return v != nil } diff --git a/clients/utils/file_utils.go b/clients/utils/file_utils.go index 8d6578f383be7195e652b0a4407c0e67ab0e75d4..93647e349382696d35831210c74ecfed976d5fca 100644 --- a/clients/utils/file_utils.go +++ b/clients/utils/file_utils.go @@ -1,8 +1,12 @@ package utils import ( + "fmt" + "io" "io/ioutil" "os" + "os/exec" + "strings" "github.com/romana/rlog" ) @@ -10,8 +14,12 @@ import ( //TempRoot - temp directory root for current OS var TempRoot = os.TempDir() + string(os.PathSeparator) + "valkyrie-temp" +//PathSeparator - string value of os.PathSeparator +var PathSeparator = fmt.Sprintf("%c", os.PathSeparator) + // Function pointers - allows for easy mocking of functions var osMkdirAll = os.MkdirAll +var osGetwd = os.Getwd // processError - helper func processError(logPrefix string, err error) { @@ -33,3 +41,180 @@ func CreateTempDir(prefix string, rootDir string) (tempDir string, err error) { } return tempDir, nil } + +// CopyFile - Copy a single file from locaton to another using full paths +func CopyFile(src, dst string) error { + sourceFileStat, err := os.Stat(src) + if err != nil { + return err + } + + if !sourceFileStat.Mode().IsRegular() { + return fmt.Errorf("%s is not a regular file", src) + } + + source, err := os.Open(src) + if err != nil { + return err + } + defer source.Close() + + destination, err := os.Create(dst) + if err != nil { + return err + } + defer destination.Close() + + _, err = io.Copy(destination, source) + if err != nil { + return err + } + return nil +} + +//CopyDir - copy directory contents recursively using exec +func CopyDir(srcPath string, dstPath string) error { + // copy source folder to temp directory + srcPath = strings.TrimRight(srcPath, PathSeparator) + dstPath = strings.TrimRight(dstPath, PathSeparator) + _, err := exec.Command("cp", "-r", srcPath+PathSeparator, dstPath+PathSeparator).Output() + if err != nil { + return err + } + return nil +} + +// DeleteDir - +func DeleteDir(path string) error { + err := os.RemoveAll(path) + if err != nil { + rlog.Warnf("error is RemoveAll %v", err) + return err + } + return nil +} + +// CreateTempDirWithFile - create tempdir and populate with filePath +// filePath may be a single file, a directory, or a zip or tar.gz file +func CreateTempDirWithFile(filePath string, prefix string, decompress bool) (string, error) { + tempDir, err := CreateTempDir(prefix, TempRoot) + if err != nil { + return "", err + } + + stat, err := os.Stat(filePath) + if err != nil { + // try to cleanup + DeleteDir(tempDir) + return "", err + } + + // flag if filepath already processed + var processed bool = false + + // if NOT directory, needs decompress and is a zip file + if !stat.IsDir() && decompress && strings.HasSuffix(filePath, ".zip") { + err = UnZipFile(filePath, tempDir) + if err != nil { + // try to cleanup + DeleteDir(tempDir) + return "", err + } + processed = true + } + + // if NOT directory, needs decompress and is a tar gz file + if !stat.IsDir() && decompress && strings.HasSuffix(filePath, ".tar.gz") { + err = UnTarFile(filePath, tempDir) + if err != nil { + // try to cleanup + DeleteDir(tempDir) + return "", err + } + processed = true + } + + // if NOT directory, and not yet processed + if !stat.IsDir() && !processed { + targetFullPath := tempDir + PathSeparator + stat.Name() + err = CopyFile(filePath, targetFullPath) + if err != nil { + // try to cleanup + DeleteDir(tempDir) + return "", err + } + } + + // if IS a directory + if stat.IsDir() { + err = CopyDir(filePath, tempDir) + if err != nil { + // try to cleanup + DeleteDir(tempDir) + return "", err + } + } + + return tempDir, nil +} + +// UnTarFile - decompress .tar.gz file to dstPath directory +func UnTarFile(filePath string, dstPath string) error { + if !IsFile(filePath) { + return fmt.Errorf("path is not a valid file: %s", filePath) + } + // clean up target dir path - insure has only one trailig separator + dstPath = strings.TrimRight(dstPath, PathSeparator) + PathSeparator + _, err := exec.Command("tar", "-xvf", filePath, "--directory", dstPath).Output() + if err != nil { + return err + } + return nil +} + +// UnZipFile - decompress .zip file to dstPath directory +func UnZipFile(filePath string, dstPath string) error { + if !IsFile(filePath) { + return fmt.Errorf("path is not a valid file: %s", filePath) + } + // clean up target dir path - insure has only one trailig separator + dstPath = strings.TrimRight(dstPath, PathSeparator) + PathSeparator + _, err := exec.Command("unzip", filePath, "-d", dstPath).Output() + if err != nil { + return err + } + return nil +} + +// IsFile - test if a path is a file using stat +func IsFile(filePath string) bool { + stat, err := os.Stat(filePath) + if err != nil { + return false + } + if !stat.IsDir() { + return true + } + return false +} + +// IsDir - test if a path is a directory using stat +func IsDir(filePath string) bool { + stat, err := os.Stat(filePath) + if err != nil { + return false + } + if stat.IsDir() { + return true + } + return false +} + +// CurrentDir - get current dir +func CurrentDir() (string, error) { + path, err := osGetwd() + if err != nil { + return "", err + } + return path, nil +} diff --git a/clients/utils/file_utils_test.go b/clients/utils/file_utils_test.go index 555d440e442cbe275703aac9a55ac26278236d5a..4122417a50336860723283c4850ee486df52caf2 100644 --- a/clients/utils/file_utils_test.go +++ b/clients/utils/file_utils_test.go @@ -2,6 +2,7 @@ package utils import ( "errors" + "fmt" "os" "strings" "testing" @@ -37,6 +38,7 @@ func TestFileUtils_CreateTempDir(t *testing.T) { } else { t.Errorf("createTempDir() UNEXPECTED error = %v, wantErr %v", err, tt.wantErr) } + osMkdirAll = osMkdirAllOrig return } if !strings.HasPrefix(gotTempDir, tt.wantTempDir) { @@ -50,7 +52,325 @@ func TestFileUtils_CreateTempDir(t *testing.T) { } } -// mock used to create an error in createTempDir +func TestCurrentDir(t *testing.T) { + tests := []struct { + name string + want string + wantErr bool + mockKey string + }{ + {name: "test good", want: "test_osgetwd", wantErr: false, mockKey: "osGetwdGood"}, + {name: "test bad", want: "test_osgetwd", wantErr: true, mockKey: "osGetwdBad"}, + } + for _, tt := range tests { + osGetwdOrig := osGetwd + t.Run(tt.name, func(t *testing.T) { + if tt.mockKey == "osGetwdGood" { + osGetwd = osGetwdGood + } + if tt.mockKey == "osGetwdBad" { + osGetwd = osGetwdBad + } + got, err := CurrentDir() + if (err != nil) && !tt.wantErr { + t.Errorf("CurrentDir() error = %v, wantErr %v", err, tt.wantErr) + osGetwd = osGetwdOrig + return + } + if (err != nil) && tt.wantErr { + t.Logf("CurrentDir() expected error occured. error = %v, wantErr %v", err, tt.wantErr) + osGetwd = osGetwdOrig + return + } + + if got != tt.want { + t.Errorf("CurrentDir() = %v, want %v", got, tt.want) + } + osGetwd = osGetwdOrig + + }) + } +} + +func TestIsDir(t *testing.T) { + type args struct { + filePath string + } + tests := []struct { + name string + args args + want bool + }{ + {name: "test bad path", args: args{filePath: "/tmp/bad/path"}, want: false}, + {name: "test bad type", args: args{filePath: "/tmp/testfile.txt"}, want: false}, + {name: "test good", args: args{filePath: "/tmp"}, want: true}, + } + // create a test file + _, err := os.Create("/tmp/testfile.txt") + if err != nil { + t.Errorf("Ios.Create failed %v", err) + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := IsDir(tt.args.filePath); got != tt.want { + t.Errorf("IsDir() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestIsFile(t *testing.T) { + type args struct { + filePath string + } + tests := []struct { + name string + args args + want bool + }{ + {name: "test bad", args: args{filePath: "/tmp/bad/path"}, want: false}, + {name: "test good", args: args{filePath: "/tmp/testfile.txt"}, want: true}, + {name: "test bad type", args: args{filePath: "/tmp"}, want: false}, + } + // create a test file + _, err := os.Create("/tmp/testfile.txt") + if err != nil { + t.Errorf("Ios.Create failed %v", err) + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := IsFile(tt.args.filePath); got != tt.want { + t.Errorf("IsFile() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestUnZipFile(t *testing.T) { + type args struct { + filePath string + dstPath string + } + tests := []struct { + name string + args args + wantErr bool + }{ + {name: "test bad file", args: args{filePath: "/tmp/notzipfile.zip", dstPath: "/tmp/out"}, wantErr: true}, + {name: "test bad path", args: args{filePath: "/tmp/filenotfound.zip", dstPath: "/tmp/out"}, wantErr: true}, + } + // create a test file + _, err := os.Create("/tmp/notzipfile.zip") + if err != nil { + t.Errorf("Ios.Create failed %v", err) + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := UnZipFile(tt.args.filePath, tt.args.dstPath); (err != nil) != tt.wantErr { + t.Errorf("UnZipFile() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestUnTarFile(t *testing.T) { + type args struct { + filePath string + dstPath string + } + tests := []struct { + name string + args args + wantErr bool + }{ + {name: "test bad file", args: args{filePath: "/tmp/nottarfile.tar.gz", dstPath: "/tmp/out"}, wantErr: true}, + {name: "test bad path", args: args{filePath: "/tmp/filenotfound.zip", dstPath: "/tmp/out"}, wantErr: true}, + } + // create a test file + _, err := os.Create("/tmp/nottarfile.tar.gz") + if err != nil { + t.Errorf("Ios.Create failed %v", err) + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := UnTarFile(tt.args.filePath, tt.args.dstPath); (err != nil) != tt.wantErr { + t.Errorf("UnTarFile() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestDeleteDir(t *testing.T) { + testDir := "/tmp/test-dir" + type args struct { + path string + } + tests := []struct { + name string + args args + wantErr bool + }{ + {name: "test good", args: args{path: testDir}, wantErr: false}, + {name: "test bad path", args: args{path: "/tmp/baddir"}, wantErr: false}, + } + // create a test dir + os.RemoveAll(testDir) + err := os.Mkdir(testDir, 0755) + if err != nil { + t.Errorf("os.Mkdir failed %v", err) + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := DeleteDir(tt.args.path); (err != nil) != tt.wantErr { + t.Errorf("DeleteDir() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } + os.RemoveAll(testDir) +} + +func TestCopyDir(t *testing.T) { + testDir := "/tmp/test-dir" + type args struct { + srcPath string + dstPath string + } + tests := []struct { + name string + args args + wantErr bool + }{ + {name: "test good", args: args{srcPath: testDir, dstPath: "/tmp/test-out-dir"}, wantErr: false}, + {name: "test bad path", args: args{srcPath: "/tmp/baddir", dstPath: "/tmp/test-out-dir"}, wantErr: true}, + } + // create a test dir and file + os.RemoveAll(testDir) + err := os.Mkdir(testDir, 0755) + if err != nil { + t.Errorf("os.Mkdir failed %v", err) + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := CopyDir(tt.args.srcPath, tt.args.dstPath); (err != nil) != tt.wantErr { + t.Errorf("CopyDir() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } + + os.RemoveAll(testDir) +} + +func TestCopyFile(t *testing.T) { + testDir := "/tmp/test-dir" + testFile := testDir + PathSeparator + "test-file.txt" + testDstFile := testDir + PathSeparator + "test-file.dst.txt" + + type args struct { + src string + dst string + } + tests := []struct { + name string + args args + wantErr bool + }{ + {name: "test good", args: args{src: testFile, dst: testDstFile}, wantErr: false}, + } + // create a test dir and file + os.RemoveAll(testDir) + err := os.Mkdir(testDir, 0755) + if err != nil { + t.Errorf("os.Mkdir failed %v", err) + } + // create a test file + _, err = os.Create(testFile) + if err != nil { + t.Errorf("os.Create failed %v", err) + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := CopyFile(tt.args.src, tt.args.dst); (err != nil) != tt.wantErr { + t.Errorf("CopyFile() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } + os.RemoveAll(testDir) +} + +func TestCreateTempDirWithFile(t *testing.T) { + testDir := "/tmp/test-dir" + testFile := testDir + PathSeparator + "test-file.txt" + testZipFile := testDir + PathSeparator + "test-file.zip" + testTarFile := testDir + PathSeparator + "test-file.tar.gz" + + type args struct { + filePath string + prefix string + decompress bool + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + {name: "test good 1", args: args{filePath: testDir, prefix: "unit-test-", decompress: false}, want: "unit-test-", wantErr: false}, + {name: "test good 2", args: args{filePath: testFile, prefix: "unit-test-", decompress: false}, want: "unit-test-", wantErr: false}, + {name: "test bad zip", args: args{filePath: testZipFile, prefix: "unit-test-", decompress: true}, want: "unit-test-", wantErr: true}, + {name: "test bad tar", args: args{filePath: testTarFile, prefix: "unit-test-", decompress: true}, want: "unit-test-", wantErr: true}, + } + + // create a test dir and file + os.RemoveAll(testDir) + err := os.Mkdir(testDir, 0755) + if err != nil { + t.Errorf("os.Mkdir failed %v", err) + } + // create a test file + _, err = os.Create(testFile) + if err != nil { + t.Errorf("os.Create failed %v", err) + } + _, err = os.Create(testZipFile) + if err != nil { + t.Errorf("os.Create failed %v", err) + } + + // tar file needs some bad data in it to fail + tf, err := os.Create(testTarFile) + if err != nil { + t.Errorf("os.Create failed %v", err) + } + newLine := "invalid data" + DateTimeUTC() + fmt.Fprintln(tf, newLine) + tf.Close() + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := CreateTempDirWithFile(tt.args.filePath, tt.args.prefix, tt.args.decompress) + if (err != nil) && !tt.wantErr { + t.Errorf("CreateTempDirWithFile() error = %v, wantErr %v", err, tt.wantErr) + return + } + if (err != nil) && tt.wantErr { + t.Logf("CreateTempDirWithFile() expected error = %v, wantErr %v", err, tt.wantErr) + return + } + if (err == nil) && tt.wantErr { + t.Errorf("CreateTempDirWithFile() expected error got = %v, wantErr %v", got, tt.wantErr) + return + } + if !strings.Contains(got, tt.want) { + t.Errorf("CreateTempDirWithFile() = %v, want %v", got, tt.want) + } + }) + } + os.RemoveAll(testDir) +} + +// mocks used to create an error in createTempDir var testCounter = 1 func osMkdirAllBad(path string, mode os.FileMode) error { @@ -61,3 +381,11 @@ func osMkdirAllBad(path string, mode os.FileMode) error { testCounter = testCounter - 1 return os.MkdirAll(path, mode) } + +func osGetwdGood() (string, error) { + return "test_osgetwd", nil +} + +func osGetwdBad() (string, error) { + return "", errors.New("test error") +} diff --git a/controllers/gitlab/group_controller.go b/controllers/gitlab/group_controller.go index 50ed36ec7714e58b8fe5abcdd2f05defceed0f56..2a43100e2068dd9fc2484ebd61b907221fb0d367 100644 --- a/controllers/gitlab/group_controller.go +++ b/controllers/gitlab/group_controller.go @@ -31,8 +31,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" "strings" "time" - apisGitlab "valkyrie.dso.mil/valkyrie-api/apis/gitlab" - gitlabv1alpha1 "valkyrie.dso.mil/valkyrie-api/apis/gitlab/v1alpha1" + gitlabAPI "valkyrie.dso.mil/valkyrie-api/apis/gitlab" + gitlabAPIVer "valkyrie.dso.mil/valkyrie-api/apis/gitlab/v1alpha1" gitlabClient "valkyrie.dso.mil/valkyrie-api/clients/gitlab" ) @@ -83,7 +83,8 @@ type GroupReconciler struct { Log logr.Logger Scheme *runtime.Scheme gitlabClient gitlabClient.Client - gitlabClientConfiguration apisGitlab.ClientConfiguration + gitlabCredentials *gitlabAPIVer.GitlabCredentials + gitlabClientConfiguration gitlabAPI.ClientConfiguration } //+kubebuilder:rbac:groups=gitlab.valkyrie.dso.mil,resources=groups,verbs=get;list;watch;create;update;patch;delete @@ -102,7 +103,7 @@ func (r *GroupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl log.Info("in Group Controller") var err error - group := &gitlabv1alpha1.Group{} + group := &gitlabAPIVer.Group{} // Get the Group Object if err = r.Get(ctx, req.NamespacedName, group); err != nil { @@ -116,13 +117,13 @@ func (r *GroupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl // Do a nil check here so we can use mock gitlabClients if r.gitlabClient == nil { if r.gitlabClientConfiguration == nil { - r.gitlabClientConfiguration = apisGitlab.ClientConfigurationImpl{ + r.gitlabClientConfiguration = gitlabAPI.ClientConfigurationImpl{ Log: log, Ctx: ctx, Req: req, } } - if r.gitlabClient, err = r.gitlabClientConfiguration.SetupClient(r.Client, group.Spec.GitlabCredentialsName); err != nil { + if r.gitlabClient, r.gitlabCredentials, err = r.gitlabClientConfiguration.SetupClient(r.Client, group.Spec.GitlabCredentialsName); err != nil { log.Error(err, errorUnableToSetupGitlabClient) _ = r.updateState(ctx, group, errorUnableToSetupGitlabClient, log) return ctrl.Result{Requeue: true}, err @@ -279,7 +280,7 @@ func (r *GroupReconciler) groupExistsByID(groupID int) (bool, *gitlab.Group, err return false, nil, nil } -func getPaths(group *gitlabv1alpha1.Group) (parentGroupPath string, path string) { +func getPaths(group *gitlabAPIVer.Group) (parentGroupPath string, path string) { lastSlashIndex := strings.LastIndex(group.Spec.FullPath, "/") if lastSlashIndex == -1 { return "", "" @@ -288,7 +289,7 @@ func getPaths(group *gitlabv1alpha1.Group) (parentGroupPath string, path string) } //createGroup assumes the parent group already exists, if not, it will error out -func (r *GroupReconciler) createGroup(group *gitlabv1alpha1.Group) (*gitlab.Group, int, error) { +func (r *GroupReconciler) createGroup(group *gitlabAPIVer.Group) (*gitlab.Group, int, error) { //find parent group first parentGroupPath, groupPath := getPaths(group) @@ -316,7 +317,7 @@ func (r *GroupReconciler) createGroup(group *gitlabv1alpha1.Group) (*gitlab.Grou return gitlabGroup, statusCode, err } -func (r *GroupReconciler) updateGroup(id int, group *gitlabv1alpha1.Group) (*gitlab.Group, int, error) { +func (r *GroupReconciler) updateGroup(id int, group *gitlabAPIVer.Group) (*gitlab.Group, int, error) { _, groupPath := getPaths(group) gitlabGroup, statusCode, err := r.gitlabClient.UpdateGroup(id, &gitlab.UpdateGroupOptions{ @@ -328,12 +329,12 @@ func (r *GroupReconciler) updateGroup(id int, group *gitlabv1alpha1.Group) (*git return gitlabGroup, statusCode, err } -func getDNSMergerRequestResourceName(group *gitlabv1alpha1.Group) string { +func getDNSMergerRequestResourceName(group *gitlabAPIVer.Group) string { return group.Name + "-dns-mr" } -func (r *GroupReconciler) processDNSMergeRequest(ctx context.Context, group *gitlabv1alpha1.Group, log logr.Logger) (err error) { - var dnsMr *gitlabv1alpha1.DNSRepoMergeRequest +func (r *GroupReconciler) processDNSMergeRequest(ctx context.Context, group *gitlabAPIVer.Group, log logr.Logger) (err error) { + var dnsMr *gitlabAPIVer.DNSRepoMergeRequest dnsMrResourceName := getDNSMergerRequestResourceName(group) //gets the resource @@ -363,8 +364,8 @@ func (r *GroupReconciler) processDNSMergeRequest(ctx context.Context, group *git return nil } -func (r *GroupReconciler) getDNSMergeRequestResource(ctx context.Context, group *gitlabv1alpha1.Group, log logr.Logger) (*gitlabv1alpha1.DNSRepoMergeRequest, error) { - var dnsMR gitlabv1alpha1.DNSRepoMergeRequest +func (r *GroupReconciler) getDNSMergeRequestResource(ctx context.Context, group *gitlabAPIVer.Group, log logr.Logger) (*gitlabAPIVer.DNSRepoMergeRequest, error) { + var dnsMR gitlabAPIVer.DNSRepoMergeRequest resourceName := getDNSMergerRequestResourceName(group) dnsMRName := types.NamespacedName{ Namespace: group.Namespace, @@ -380,11 +381,11 @@ func (r *GroupReconciler) getDNSMergeRequestResource(ctx context.Context, group return &dnsMR, nil } -func (r *GroupReconciler) createDNSMergeRequestResource(ctx context.Context, group *gitlabv1alpha1.Group, log logr.Logger) error { +func (r *GroupReconciler) createDNSMergeRequestResource(ctx context.Context, group *gitlabAPIVer.Group, log logr.Logger) error { blockOwnerDeletion := true managingController := true resourceName := getDNSMergerRequestResourceName(group) - err := r.Create(ctx, &gitlabv1alpha1.DNSRepoMergeRequest{ + err := r.Create(ctx, &gitlabAPIVer.DNSRepoMergeRequest{ ObjectMeta: ctrl.ObjectMeta{ Name: resourceName, Namespace: group.Namespace, @@ -408,9 +409,9 @@ func (r *GroupReconciler) createDNSMergeRequestResource(ctx context.Context, gro return err } -func (r *GroupReconciler) deleteDNSMergeRequestResource(ctx context.Context, group *gitlabv1alpha1.Group, log logr.Logger) (err error) { +func (r *GroupReconciler) deleteDNSMergeRequestResource(ctx context.Context, group *gitlabAPIVer.Group, log logr.Logger) (err error) { //check if resource exits - var dnsMrResource *gitlabv1alpha1.DNSRepoMergeRequest + var dnsMrResource *gitlabAPIVer.DNSRepoMergeRequest if dnsMrResource, err = r.getDNSMergeRequestResource(ctx, group, log); err != nil { return err } @@ -426,7 +427,7 @@ func (r *GroupReconciler) deleteDNSMergeRequestResource(ctx context.Context, gro return nil } -func (r *GroupReconciler) processProjects(ctx context.Context, group *gitlabv1alpha1.Group, log logr.Logger) error { +func (r *GroupReconciler) processProjects(ctx context.Context, group *gitlabAPIVer.Group, log logr.Logger) error { var projectResourceNames = make([]string, len(group.Spec.ProjectTemplates)) for i, projectTemplate := range group.Spec.ProjectTemplates { projectResourceNames[i] = projectTemplate.Name @@ -454,7 +455,7 @@ func (r *GroupReconciler) processProjects(ctx context.Context, group *gitlabv1al return nil } -func (r *GroupReconciler) cleanUpProjects(ctx context.Context, group *gitlabv1alpha1.Group, projectResourceNames []string, log logr.Logger) error { +func (r *GroupReconciler) cleanUpProjects(ctx context.Context, group *gitlabAPIVer.Group, projectResourceNames []string, log logr.Logger) error { projectsToDelete := getElementsNotContained(group.Status.ProjectResourceNames, projectResourceNames) for _, projectToDelete := range projectsToDelete { if project, foundProject, err := r.getProject(ctx, group.Namespace, projectToDelete); err != nil { @@ -500,7 +501,7 @@ func getElementsNotContained(a []string, b []string) []string { return keys } -func (r *GroupReconciler) deleteProjects(ctx context.Context, group *gitlabv1alpha1.Group, templates []gitlabv1alpha1.ProjectTemplate, log logr.Logger) error { +func (r *GroupReconciler) deleteProjects(ctx context.Context, group *gitlabAPIVer.Group, templates []gitlabAPIVer.ProjectTemplate, log logr.Logger) error { for _, template := range templates { // if it's not found we'll consider it gone and not worry about it. if project, foundProject, err := r.getProject(ctx, group.Namespace, template.Name); err != nil { @@ -516,8 +517,8 @@ func (r *GroupReconciler) deleteProjects(ctx context.Context, group *gitlabv1alp return nil } -func (r *GroupReconciler) getProject(ctx context.Context, namespace string, name string) (*gitlabv1alpha1.Project, bool, error) { - var project gitlabv1alpha1.Project +func (r *GroupReconciler) getProject(ctx context.Context, namespace string, name string) (*gitlabAPIVer.Project, bool, error) { + var project gitlabAPIVer.Project projectName := types.NamespacedName{ Namespace: namespace, Name: name, @@ -532,10 +533,10 @@ func (r *GroupReconciler) getProject(ctx context.Context, namespace string, name return &project, true, err } -func (r *GroupReconciler) createProject(ctx context.Context, group *gitlabv1alpha1.Group, template gitlabv1alpha1.ProjectTemplate) error { +func (r *GroupReconciler) createProject(ctx context.Context, group *gitlabAPIVer.Group, template gitlabAPIVer.ProjectTemplate) error { blockOwnerDeletion := true managingController := true - return r.Create(ctx, &gitlabv1alpha1.Project{ + return r.Create(ctx, &gitlabAPIVer.Project{ ObjectMeta: ctrl.ObjectMeta{ Name: template.Name, Namespace: group.Namespace, @@ -554,8 +555,8 @@ func (r *GroupReconciler) createProject(ctx context.Context, group *gitlabv1alph }) } -func getProjectSpec(group *gitlabv1alpha1.Group, template gitlabv1alpha1.ProjectTemplate) gitlabv1alpha1.ProjectSpec { - return gitlabv1alpha1.ProjectSpec{ +func getProjectSpec(group *gitlabAPIVer.Group, template gitlabAPIVer.ProjectTemplate) gitlabAPIVer.ProjectSpec { + return gitlabAPIVer.ProjectSpec{ GroupID: *group.Status.ID, Name: template.ProjectName, FullPath: template.FullPath, @@ -567,12 +568,12 @@ func getProjectSpec(group *gitlabv1alpha1.Group, template gitlabv1alpha1.Project } } -func (r *GroupReconciler) updateProject(ctx context.Context, group *gitlabv1alpha1.Group, template gitlabv1alpha1.ProjectTemplate, project *gitlabv1alpha1.Project) (*gitlabv1alpha1.Project, error) { +func (r *GroupReconciler) updateProject(ctx context.Context, group *gitlabAPIVer.Group, template gitlabAPIVer.ProjectTemplate, project *gitlabAPIVer.Project) (*gitlabAPIVer.Project, error) { project.Spec = getProjectSpec(group, template) return project, r.Update(ctx, project) } -func (r *GroupReconciler) updateStatus(ctx context.Context, group *gitlabv1alpha1.Group) error { +func (r *GroupReconciler) updateStatus(ctx context.Context, group *gitlabAPIVer.Group) error { group.Status.LastUpdatedTime = &metav1.Time{ Time: time.Now(), } @@ -588,14 +589,14 @@ func (r *GroupReconciler) updateStatus(ctx context.Context, group *gitlabv1alpha func (r *GroupReconciler) SetupWithManager(mgr ctrl.Manager) error { pred := predicate.GenerationChangedPredicate{} return ctrl.NewControllerManagedBy(mgr). - For(&gitlabv1alpha1.Group{}). + For(&gitlabAPIVer.Group{}). //owns cause any update to the created Project to run the reconcile method...not exactly what we need - //Owns(&gitlabv1alpha1.Project{}). + //Owns(&gitlabAPIVer.Project{}). WithEventFilter(pred). Complete(r) } -func (r *GroupReconciler) updateCiVariable(ID int, variableToUpdate gitlabv1alpha1.CICDVariable, groupVariable *gitlab.GroupVariable, log logr.Logger) error { +func (r *GroupReconciler) updateCiVariable(ID int, variableToUpdate gitlabAPIVer.CICDVariable, groupVariable *gitlab.GroupVariable, log logr.Logger) error { //if value needs to be updated, update it option := gitlab.UpdateGroupVariableOptions{ Value: &variableToUpdate.Value, @@ -614,7 +615,7 @@ func (r *GroupReconciler) updateCiVariable(ID int, variableToUpdate gitlabv1alph return nil } -func (r *GroupReconciler) updateCiVariables(ID int, group *gitlabv1alpha1.Group, log logr.Logger) error { +func (r *GroupReconciler) updateCiVariables(ID int, group *gitlabAPIVer.Group, log logr.Logger) error { var variablesToUpdate = group.Spec.CIVariables var err error var existingGroupVariable *gitlab.GroupVariable @@ -643,7 +644,7 @@ func (r *GroupReconciler) updateCiVariables(ID int, group *gitlabv1alpha1.Group, } // CreateCiVariable - -func (r *GroupReconciler) CreateCiVariable(ID int, variableToCreate gitlabv1alpha1.CICDVariable) (statusCode int, err error) { +func (r *GroupReconciler) CreateCiVariable(ID int, variableToCreate gitlabAPIVer.CICDVariable) (statusCode int, err error) { envType := gitlab.EnvVariableType protected := true masked := false @@ -660,7 +661,7 @@ func (r *GroupReconciler) CreateCiVariable(ID int, variableToCreate gitlabv1alph return statusCode, err } -func (r *GroupReconciler) updateState(ctx context.Context, group *gitlabv1alpha1.Group, state string, log logr.Logger) error { +func (r *GroupReconciler) updateState(ctx context.Context, group *gitlabAPIVer.Group, state string, log logr.Logger) error { group.Status.State = state if err := r.Status().Update(ctx, group); err != nil { log.Error(err, errorWhileUpdatingGroupState, "group", group) @@ -669,7 +670,7 @@ func (r *GroupReconciler) updateState(ctx context.Context, group *gitlabv1alpha1 return nil } -func (r *GroupReconciler) deleteProjectsAndGitlabGroup(ctx context.Context, group *gitlabv1alpha1.Group, log logr.Logger) error { +func (r *GroupReconciler) deleteProjectsAndGitlabGroup(ctx context.Context, group *gitlabAPIVer.Group, log logr.Logger) error { var err error groupID := group.Status.ID diff --git a/controllers/gitlab/group_controller_test.go b/controllers/gitlab/group_controller_test.go index 2f23e49621fb159f4e7174aee73c280a5276e3b5..00bd0902f5326ac6b742590d5640115903c34bd3 100755 --- a/controllers/gitlab/group_controller_test.go +++ b/controllers/gitlab/group_controller_test.go @@ -11,7 +11,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "time" apisGitlab "valkyrie.dso.mil/valkyrie-api/apis/gitlab" - gitlabv1alpha1 "valkyrie.dso.mil/valkyrie-api/apis/gitlab/v1alpha1" + gitlabAPIVer "valkyrie.dso.mil/valkyrie-api/apis/gitlab/v1alpha1" gitlabClient "valkyrie.dso.mil/valkyrie-api/clients/gitlab" ) @@ -49,17 +49,17 @@ func getGreenGroupControllerDeleteRequest() ctrl.Request { } } -func getGreenGroup() *gitlabv1alpha1.Group { - returnGroup := &gitlabv1alpha1.Group{ +func getGreenGroup() *gitlabAPIVer.Group { + returnGroup := &gitlabAPIVer.Group{ TypeMeta: metav1.TypeMeta{}, ObjectMeta: metav1.ObjectMeta{ Annotations: make(map[string]string), }, - Spec: gitlabv1alpha1.GroupSpec{ + Spec: gitlabAPIVer.GroupSpec{ FullPath: "/greenGroup", Name: "greenGroup", GitlabCredentialsName: "greenCredentialsName", - CIVariables: []gitlabv1alpha1.CICDVariable{ + CIVariables: []gitlabAPIVer.CICDVariable{ { Key: kustomizeProductionPathVariableName, Value: "/kustomize/prod/path", @@ -73,14 +73,14 @@ func getGreenGroup() *gitlabv1alpha1.Group { Value: manifestRepoURLVariableValue, }, }, - DNSRepoMergeRequestSpec: &gitlabv1alpha1.DNSRepoMergeRequestSpec{ + DNSRepoMergeRequestSpec: &gitlabAPIVer.DNSRepoMergeRequestSpec{ RepoCredentialName: "test", AppHostname: "greenapp", TargetEnvironment: "il2", }, - ProjectTemplates: make([]gitlabv1alpha1.ProjectTemplate, 0), + ProjectTemplates: make([]gitlabAPIVer.ProjectTemplate, 0), }, - Status: gitlabv1alpha1.GroupStatus{ + Status: gitlabAPIVer.GroupStatus{ CreatedTime: &metav1.Time{ Time: time.Now(), }, @@ -90,7 +90,7 @@ func getGreenGroup() *gitlabv1alpha1.Group { State: "", }, } - returnGroup.Spec.ProjectTemplates = append(returnGroup.Spec.ProjectTemplates, gitlabv1alpha1.ProjectTemplate{ + returnGroup.Spec.ProjectTemplates = append(returnGroup.Spec.ProjectTemplates, gitlabAPIVer.ProjectTemplate{ Name: "Project resource name", ProjectName: "A Green Project", FullPath: "full path", @@ -100,7 +100,7 @@ func getGreenGroup() *gitlabv1alpha1.Group { } func getGroupControllerWithMocksInGreenTestState() (*GroupReconciler, *MockLogger, *MockClientConfiguration, *MockClient, *MockGitlabClient) { - builder := gitlabv1alpha1.SchemeBuilder.Register(&gitlabv1alpha1.Group{}, &gitlabv1alpha1.GroupList{}) + builder := gitlabAPIVer.SchemeBuilder.Register(&gitlabAPIVer.Group{}, &gitlabAPIVer.GroupList{}) scheme, _ := builder.Build() loggerMock := MockLogger{ WithValuesKeysAndValues: nil, @@ -170,17 +170,17 @@ func getGroupControllerWithMocksInGreenTestState() (*GroupReconciler, *MockLogge return &sut, &loggerMock, &configurationMock, &clientMock, &gitlabClientMock } -func getToDeleteGroup() *gitlabv1alpha1.Group { - toDeleteGroup := &gitlabv1alpha1.Group{ +func getToDeleteGroup() *gitlabAPIVer.Group { + toDeleteGroup := &gitlabAPIVer.Group{ TypeMeta: metav1.TypeMeta{}, ObjectMeta: metav1.ObjectMeta{ Annotations: make(map[string]string), }, - Spec: gitlabv1alpha1.GroupSpec{ + Spec: gitlabAPIVer.GroupSpec{ FullPath: "/greenDeleteGroup", Name: "greenDeleteGroup", GitlabCredentialsName: "greenCredentialsName", - CIVariables: []gitlabv1alpha1.CICDVariable{ + CIVariables: []gitlabAPIVer.CICDVariable{ { Key: kustomizeProductionPathVariableName, Value: "/kustomize/prod/path", @@ -194,14 +194,14 @@ func getToDeleteGroup() *gitlabv1alpha1.Group { Value: manifestRepoURLVariableValue, }, }, - DNSRepoMergeRequestSpec: &gitlabv1alpha1.DNSRepoMergeRequestSpec{ + DNSRepoMergeRequestSpec: &gitlabAPIVer.DNSRepoMergeRequestSpec{ RepoCredentialName: "test", AppHostname: "testdelete", TargetEnvironment: "il2", }, - ProjectTemplates: make([]gitlabv1alpha1.ProjectTemplate, 0), + ProjectTemplates: make([]gitlabAPIVer.ProjectTemplate, 0), }, - Status: gitlabv1alpha1.GroupStatus{ + Status: gitlabAPIVer.GroupStatus{ CreatedTime: &metav1.Time{ Time: time.Now(), }, @@ -214,7 +214,7 @@ func getToDeleteGroup() *gitlabv1alpha1.Group { toDeleteGroup.Spec.Name = toDeleteGreenGroupName toDeleteGroup.Namespace = toDeleteGreenGroupNamespace toDeleteGroup.Spec.FullPath = "/greenDeleteGroup" - toDeleteGroup.Spec.ProjectTemplates = append(toDeleteGroup.Spec.ProjectTemplates, gitlabv1alpha1.ProjectTemplate{ + toDeleteGroup.Spec.ProjectTemplates = append(toDeleteGroup.Spec.ProjectTemplates, gitlabAPIVer.ProjectTemplate{ Name: "project-resource-name", ProjectName: "A Green Project to Delete", FullPath: "full path", @@ -267,13 +267,13 @@ var _ = Describe("Reconcile", func() { }) // Finish the Delete - group := sut.Client.(*MockClient).expectedObjects[getGreenGroupControllerDeleteRequest().NamespacedName].(*gitlabv1alpha1.Group) + group := sut.Client.(*MockClient).expectedObjects[getGreenGroupControllerDeleteRequest().NamespacedName].(*gitlabAPIVer.Group) group.SetDeletionTimestamp(&metav1.Time{Time: time.Now()}) group.Finalizers = append(group.Finalizers, groupFinalizerName) groupIDToDelete := toDeleteGreenGroupID group.Status.ID = &groupIDToDelete - sut.Client.(*MockClient).expectedObjects[types.NamespacedName{Namespace: group.Namespace, Name: getDNSMergerRequestResourceName(group)}] = &gitlabv1alpha1.DNSRepoMergeRequest{} + sut.Client.(*MockClient).expectedObjects[types.NamespacedName{Namespace: group.Namespace, Name: getDNSMergerRequestResourceName(group)}] = &gitlabAPIVer.DNSRepoMergeRequest{} result, err = sut.Reconcile(context.TODO(), getGreenGroupControllerDeleteRequest()) It("should not request a requeue", func() { @@ -317,8 +317,8 @@ var _ = Describe("Reconcile", func() { }) Context("gitlab client configuration fails", func() { sut, logger, clientConfiguration, _, _ := getGroupControllerWithMocksInGreenTestState() - clientConfiguration.SetupClientFunction = func(client client.Client, credentialsName string) (gitlabClient.Client, error) { - return nil, &MockError{message: "setupClient Failed"} + clientConfiguration.SetupClientFunction = func(client client.Client, credentialsName string) (gitlabClient.Client, *gitlabAPIVer.GitlabCredentials, error) { + return nil, nil, &MockError{message: "setupClient Failed"} } result, err := sut.Reconcile(context.TODO(), getGreenGroupControllerRequest()) It("should log the error", func() { @@ -332,7 +332,7 @@ var _ = Describe("Reconcile", func() { Expect(err).ToNot(BeNil()) }) It("should update the state of the object", func() { - Expect(sut.Status().(*MockStatusWriter).updatedObject.(*gitlabv1alpha1.Group).Status.State).To(Equal(errorUnableToSetupGitlabClient)) + Expect(sut.Status().(*MockStatusWriter).updatedObject.(*gitlabAPIVer.Group).Status.State).To(Equal(errorUnableToSetupGitlabClient)) }) }) Context("groupExistsByPath fails", func() { @@ -360,7 +360,7 @@ var _ = Describe("Reconcile", func() { Expect(err).NotTo(BeNil()) }) It("should update the state of the object", func() { - Expect(sut.Status().(*MockStatusWriter).updatedObject.(*gitlabv1alpha1.Group).Status.State).To(Equal(errorWhileCreatingGitlabGroup)) + Expect(sut.Status().(*MockStatusWriter).updatedObject.(*gitlabAPIVer.Group).Status.State).To(Equal(errorWhileCreatingGitlabGroup)) }) }) Context("createGroup fails", func() { @@ -392,7 +392,7 @@ var _ = Describe("Reconcile", func() { Expect(err).NotTo(BeNil()) }) It("should update the state of the object", func() { - Expect(sut.Status().(*MockStatusWriter).updatedObject.(*gitlabv1alpha1.Group).Status.State).To(Equal(errorWhileCreatingGitlabGroup)) + Expect(sut.Status().(*MockStatusWriter).updatedObject.(*gitlabAPIVer.Group).Status.State).To(Equal(errorWhileCreatingGitlabGroup)) }) }) Context("updateGroup fails", func() { @@ -403,7 +403,7 @@ var _ = Describe("Reconcile", func() { message: "Error", } } - group := sut.Client.(*MockClient).expectedObjects[getGreenGroupControllerRequest().NamespacedName].(*gitlabv1alpha1.Group) + group := sut.Client.(*MockClient).expectedObjects[getGreenGroupControllerRequest().NamespacedName].(*gitlabAPIVer.Group) groupID := greenGroupID group.Status.ID = &groupID @@ -419,7 +419,7 @@ var _ = Describe("Reconcile", func() { Expect(err).NotTo(BeNil()) }) It("should update the state of the object", func() { - Expect(sut.Status().(*MockStatusWriter).updatedObject.(*gitlabv1alpha1.Group).Status.State).To(Equal(errorWhileUpdatingGitlabGroup)) + Expect(sut.Status().(*MockStatusWriter).updatedObject.(*gitlabAPIVer.Group).Status.State).To(Equal(errorWhileUpdatingGitlabGroup)) }) }) Context("getGroupVariable fails", func() { @@ -439,7 +439,7 @@ var _ = Describe("Reconcile", func() { Expect(err).ToNot(BeNil()) }) It("should update the state of the object", func() { - Expect(sut.Status().(*MockStatusWriter).updatedObject.(*gitlabv1alpha1.Group).Status.State).To(Equal(errorWhileUpdatingGitlabGroupCiVariables)) + Expect(sut.Status().(*MockStatusWriter).updatedObject.(*gitlabAPIVer.Group).Status.State).To(Equal(errorWhileUpdatingGitlabGroupCiVariables)) }) }) Context("createCiVariable fails", func() { @@ -459,7 +459,7 @@ var _ = Describe("Reconcile", func() { Expect(err).ToNot(BeNil()) }) It("should update the state of the object", func() { - Expect(sut.Status().(*MockStatusWriter).updatedObject.(*gitlabv1alpha1.Group).Status.State).To(Equal(errorWhileUpdatingGitlabGroupCiVariables)) + Expect(sut.Status().(*MockStatusWriter).updatedObject.(*gitlabAPIVer.Group).Status.State).To(Equal(errorWhileUpdatingGitlabGroupCiVariables)) }) }) Context("updateGroupVariable fails", func() { @@ -467,7 +467,7 @@ var _ = Describe("Reconcile", func() { gitlabClient.updateGroupVariableFunction = func(groupID int, key string, options gitlab.UpdateGroupVariableOptions) (*gitlab.GroupVariable, int, error) { return nil, 500, &MockError{"updateGroupVariable mocked error"} } - group := sut.Client.(*MockClient).expectedObjects[getGreenGroupControllerRequest().NamespacedName].(*gitlabv1alpha1.Group) + group := sut.Client.(*MockClient).expectedObjects[getGreenGroupControllerRequest().NamespacedName].(*gitlabAPIVer.Group) groupID := greenGroupID group.Status.ID = &groupID @@ -484,7 +484,7 @@ var _ = Describe("Reconcile", func() { Expect(err).ToNot(BeNil()) }) It("should update the state of the object", func() { - Expect(sut.Status().(*MockStatusWriter).updatedObject.(*gitlabv1alpha1.Group).Status.State).To(Equal(errorWhileUpdatingGitlabGroupCiVariables)) + Expect(sut.Status().(*MockStatusWriter).updatedObject.(*gitlabAPIVer.Group).Status.State).To(Equal(errorWhileUpdatingGitlabGroupCiVariables)) }) }) Context("processProjects fails", func() { @@ -492,14 +492,14 @@ var _ = Describe("Reconcile", func() { kubernetesClient.createFunction = func(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { switch obj.(type) { - case *gitlabv1alpha1.Project: + case *gitlabAPIVer.Project: return &MockError{message: "processProjects Fails"} } return nil } kubernetesClient.UpdateFunction = func(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { switch obj.(type) { - case *gitlabv1alpha1.Project: + case *gitlabAPIVer.Project: return &MockError{message: "processProjects Fails"} } return nil @@ -620,9 +620,9 @@ var _ = Describe("createGroup", func() { sut.gitlabClient = gitlabClient group, statusCode, err := sut.createGroup( - &gitlabv1alpha1.Group{ + &gitlabAPIVer.Group{ ObjectMeta: ctrl.ObjectMeta{Annotations: make(map[string]string)}, - Spec: gitlabv1alpha1.GroupSpec{Name: "a group"}, + Spec: gitlabAPIVer.GroupSpec{Name: "a group"}, }) It("should return the gitlab group created", func() { Expect(group).To(Equal(expectedGroup)) @@ -652,10 +652,10 @@ var _ = Describe("updateGroup", func() { group, response, err := sut.updateGroup( 1, - &gitlabv1alpha1.Group{ + &gitlabAPIVer.Group{ ObjectMeta: ctrl.ObjectMeta{Annotations: make(map[string]string)}, - Spec: gitlabv1alpha1.GroupSpec{Name: "a group"}, - Status: gitlabv1alpha1.GroupStatus{ + Spec: gitlabAPIVer.GroupSpec{Name: "a group"}, + Status: gitlabAPIVer.GroupStatus{ CreatedTime: &metav1.Time{}, LastUpdatedTime: &metav1.Time{}, State: "", @@ -673,7 +673,7 @@ var _ = Describe("updateGroup", func() { }) var _ = Describe("SetupWithManager", func() { - builder := gitlabv1alpha1.SchemeBuilder.Register(&gitlabv1alpha1.Group{}, &gitlabv1alpha1.GroupList{}) + builder := gitlabAPIVer.SchemeBuilder.Register(&gitlabAPIVer.Group{}, &gitlabAPIVer.GroupList{}) scheme, _ := builder.Build() loggerMock := MockLogger{ WithValuesKeysAndValues: nil, @@ -702,7 +702,7 @@ var _ = Describe("updateStatus", func() { When("group id is not 0", func() { sut, _, _, _, _ := getGroupControllerWithMocksInGreenTestState() - group := gitlabv1alpha1.Group{Status: gitlabv1alpha1.GroupStatus{ + group := gitlabAPIVer.Group{Status: gitlabAPIVer.GroupStatus{ CreatedTime: &metav1.Time{}, LastUpdatedTime: &metav1.Time{}, State: "", @@ -716,11 +716,11 @@ var _ = Describe("updateStatus", func() { }) var _ = Describe("updateProject", func() { - template := gitlabv1alpha1.ProjectTemplate{ + template := gitlabAPIVer.ProjectTemplate{ Name: "a project", } - project := gitlabv1alpha1.Project{Spec: gitlabv1alpha1.ProjectSpec{}} - builder := gitlabv1alpha1.SchemeBuilder.Register(&gitlabv1alpha1.Group{}, &gitlabv1alpha1.GroupList{}) + project := gitlabAPIVer.Project{Spec: gitlabAPIVer.ProjectSpec{}} + builder := gitlabAPIVer.SchemeBuilder.Register(&gitlabAPIVer.Group{}, &gitlabAPIVer.GroupList{}) scheme, _ := builder.Build() loggerMock := MockLogger{ WithValuesKeysAndValues: nil, @@ -746,11 +746,11 @@ var _ = Describe("updateProject", func() { } groupID := 1 - group := gitlabv1alpha1.Group{ - Spec: gitlabv1alpha1.GroupSpec{ + group := gitlabAPIVer.Group{ + Spec: gitlabAPIVer.GroupSpec{ GitlabCredentialsName: "test", }, - Status: gitlabv1alpha1.GroupStatus{ + Status: gitlabAPIVer.GroupStatus{ ID: &groupID, }, } @@ -777,7 +777,7 @@ var _ = Describe("CreateCiVariable", func() { sut.gitlabClient = gitlabClient statusCode, err := sut.CreateCiVariable(1, - gitlabv1alpha1.CICDVariable{ + gitlabAPIVer.CICDVariable{ Key: "variable", Value: "testvalue", }, diff --git a/controllers/gitlab/mocks_test.go b/controllers/gitlab/mocks_test.go index 87905256d8c13ab5d24d39f341ea4354ca6321fb..359b8c80be90b2cce225591973a26465d0680877 100755 --- a/controllers/gitlab/mocks_test.go +++ b/controllers/gitlab/mocks_test.go @@ -21,6 +21,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/scheme" "sigs.k8s.io/controller-runtime/pkg/webhook" "time" + gitlabAPIVer "valkyrie.dso.mil/valkyrie-api/apis/gitlab/v1alpha1" gitlabClient "valkyrie.dso.mil/valkyrie-api/clients/gitlab" "valkyrie.dso.mil/valkyrie-api/controllers" ) @@ -924,18 +925,18 @@ func (m *MockGitlabClient) GetRepositoryTree(projectID int) ([]*gitlab.TreeNode, type MockClientConfiguration struct { GitlabClient gitlabClient.Client - SetupClientFunction func(client client.Client, credentialsName string) (gitlabClient.Client, error) + SetupClientFunction func(client client.Client, credentialsName string) (gitlabClient.Client, *gitlabAPIVer.GitlabCredentials, error) SetupClientCalled bool } -func (m *MockClientConfiguration) SetupClient(client client.Client, credentialsName string) (gitlabClient.Client, error) { +func (m *MockClientConfiguration) SetupClient(client client.Client, credentialsName string) (gitlabClient.Client, *gitlabAPIVer.GitlabCredentials, error) { m.SetupClientCalled = true if m.SetupClientFunction == nil { if m.GitlabClient == nil { - return &MockGitlabClient{}, nil + return &MockGitlabClient{}, nil, nil } - return m.GitlabClient, nil + return m.GitlabClient, nil, nil } return m.SetupClientFunction(client, credentialsName) diff --git a/controllers/gitlab/project_controller.go b/controllers/gitlab/project_controller.go index 01148ee37ef612ee2bfee42bedf412de28d67b0f..e749c7eb1789508bd39cea064b205bbc839a6040 100644 --- a/controllers/gitlab/project_controller.go +++ b/controllers/gitlab/project_controller.go @@ -29,10 +29,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" "strings" fortifyv1 "valkyrie.dso.mil/valkyrie-api/apis/fortify/v1alpha1" - apisGitlab "valkyrie.dso.mil/valkyrie-api/apis/gitlab" + gitlabAPI "valkyrie.dso.mil/valkyrie-api/apis/gitlab" "valkyrie.dso.mil/valkyrie-api/apis/gitlab/v1alpha1" + gitlabAPIVer "valkyrie.dso.mil/valkyrie-api/apis/gitlab/v1alpha1" twistlockv1alpha1 "valkyrie.dso.mil/valkyrie-api/apis/twistlock/v1alpha1" gitlabClient "valkyrie.dso.mil/valkyrie-api/clients/gitlab" + customImpl "valkyrie.dso.mil/valkyrie-api/custom/p1" ) // ProjectReconciler reconciles a Project object @@ -41,7 +43,8 @@ type ProjectReconciler struct { Log logr.Logger Scheme *runtime.Scheme gitlabClient gitlabClient.Client - gitlabClientConfiguration apisGitlab.ClientConfiguration + gitlabCredentials *gitlabAPIVer.GitlabCredentials + gitlabClientConfiguration gitlabAPI.ClientConfiguration } //+kubebuilder:rbac:groups=gitlab.valkyrie.dso.mil,resources=projects,verbs=get;list;watch;create;update;patch;delete @@ -104,13 +107,13 @@ func (r *ProjectReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct // Do a nil check here so we can use mock gitlabClients if r.gitlabClient == nil { if r.gitlabClientConfiguration == nil { - r.gitlabClientConfiguration = apisGitlab.ClientConfigurationImpl{ + r.gitlabClientConfiguration = gitlabAPI.ClientConfigurationImpl{ Log: log, Ctx: ctx, Req: req, } } - if r.gitlabClient, err = r.gitlabClientConfiguration.SetupClient(r.Client, project.Spec.GitlabCredentialsName); err != nil { + if r.gitlabClient, r.gitlabCredentials, err = r.gitlabClientConfiguration.SetupClient(r.Client, project.Spec.GitlabCredentialsName); err != nil { log.Error(err, errorUnableToSetupGitlabClient) _ = r.updateStatus(ctx, project, errorUnableToSetupGitlabClient, log) return ctrl.Result{Requeue: true}, err @@ -214,6 +217,15 @@ func (r *ProjectReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct return ctrl.Result{Requeue: true}, err } + // project should exists - check bootstrap manifest + if project.Spec.BootstrapWithManifestRepo != nil { + // bootstrap with manifests + err = customImpl.BootstrapProjectWithManifest(r.gitlabCredentials, gitlabProject.ID, *project.Spec.BootstrapWithManifestRepo) + if err != nil { + log.Error(err, "Error bootstrapping manifest") + } + } + if err = r.updateCiVariables(*id, project, log); err != nil { log.Error(err, errorUpdatingProjectVariableInGitlab) _ = r.updateStatus(ctx, project, errorUpdatingProjectVariableInGitlab, log) diff --git a/controllers/gitlab/project_controller_test.go b/controllers/gitlab/project_controller_test.go index 355eb702b1c987941d51e846391b26c8b8fb1b81..4ae00ba4e9a9cfe32ce09b2148a5acf8511668c7 100644 --- a/controllers/gitlab/project_controller_test.go +++ b/controllers/gitlab/project_controller_test.go @@ -12,6 +12,7 @@ import ( fortifyv1 "valkyrie.dso.mil/valkyrie-api/apis/fortify/v1alpha1" apisGitlab "valkyrie.dso.mil/valkyrie-api/apis/gitlab" "valkyrie.dso.mil/valkyrie-api/apis/gitlab/v1alpha1" + gitlabAPIVer "valkyrie.dso.mil/valkyrie-api/apis/gitlab/v1alpha1" twistlockv1 "valkyrie.dso.mil/valkyrie-api/apis/twistlock/v1alpha1" gitlabClient "valkyrie.dso.mil/valkyrie-api/clients/gitlab" ) @@ -163,8 +164,8 @@ var _ = Describe("reconcile", func() { request := getRequestWithDefaultNamespacedTestProject() sut.gitlabClient = nil sut.gitlabClientConfiguration = &MockClientConfiguration{ - SetupClientFunction: func(client client.Client, credentialsName string) (gitlabClient.Client, error) { - return nil, &MockError{message: "SetupClientFunction fails."} + SetupClientFunction: func(client client.Client, credentialsName string) (gitlabClient.Client, *gitlabAPIVer.GitlabCredentials, error) { + return nil, nil, &MockError{message: "SetupClientFunction fails."} }, } result, err := sut.Reconcile(context.TODO(), request) diff --git a/custom/p1/README.md b/custom/p1/README.md new file mode 100644 index 0000000000000000000000000000000000000000..99a3c5065413d36ad38072a493ea67127dc699f3 --- /dev/null +++ b/custom/p1/README.md @@ -0,0 +1,6 @@ +## P1 platform one interface to automation methods + +### Run tests +- go clean -testcache && go test -v ./custom/p1/... -coverprofile cover.out +- go tool cover -html=cover.out -o coverage.html +- open coverage.html diff --git a/custom/p1/group_test.go b/custom/p1/group_test.go index adddbb1b6ef2f7a2fd0aff04560bbb047781ec9e..6d115848d8c060d5393fcbbc358570a054c2db80 100644 --- a/custom/p1/group_test.go +++ b/custom/p1/group_test.go @@ -35,6 +35,7 @@ func TestCreateGroup(t *testing.T) { // test objects testAPIUrl := "https://test/api/v4/" testToken := "token" + testUsername := "username" testGroupSpec := apigitlab.GroupSpec{FullPath: testGroupFullPath, Name: "Group Name"} type args struct { @@ -47,7 +48,7 @@ func TestCreateGroup(t *testing.T) { args args wantErr bool }{ - {name: "test", args: args{gitlabCredentials: GitlabCredentials{testAPIUrl, testToken}, groupSpec: testGroupSpec, httpClient: testHTTPClient}, wantErr: false}, + {name: "test", args: args{gitlabCredentials: GitlabCredentials{testAPIUrl, testUsername, testToken}, groupSpec: testGroupSpec, httpClient: testHTTPClient}, wantErr: false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -95,6 +96,7 @@ func TestDeleteGroup(t *testing.T) { // test objects testAPIUrl := "https://test/api/v4/" testToken := "token" + testUsername := "username" testGroupSpec := apigitlab.GroupSpec{FullPath: testGroupFullPath, Name: "Group Name"} type args struct { @@ -107,7 +109,7 @@ func TestDeleteGroup(t *testing.T) { args args wantErr bool }{ - {name: "test", args: args{gitlabCredentials: GitlabCredentials{testAPIUrl, testToken}, groupSpec: testGroupSpec, httpClient: testHTTPClient}, wantErr: false}, + {name: "test", args: args{gitlabCredentials: GitlabCredentials{testAPIUrl, testUsername, testToken}, groupSpec: testGroupSpec, httpClient: testHTTPClient}, wantErr: false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -147,6 +149,7 @@ func TestUpdateGroup(t *testing.T) { // test objects testAPIUrl := "https://test/api/v4/" testToken := "token" + testUsername := "username" testGroupSpec := apigitlab.GroupSpec{FullPath: testGroupFullPath, Name: "Group Name"} type args struct { @@ -159,7 +162,7 @@ func TestUpdateGroup(t *testing.T) { args args wantErr bool }{ - {name: "test", args: args{gitlabCredentials: GitlabCredentials{testAPIUrl, testToken}, groupSpec: testGroupSpec, httpClient: testHTTPClient}, wantErr: false}, + {name: "test", args: args{gitlabCredentials: GitlabCredentials{testAPIUrl, testUsername, testToken}, groupSpec: testGroupSpec, httpClient: testHTTPClient}, wantErr: false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/custom/p1/project.go b/custom/p1/project.go index 257eb3f9c238b24f95b5217685f4d65e7960cc94..522aae60a8bdadbae42e5adaa9f83664cff7477d 100644 --- a/custom/p1/project.go +++ b/custom/p1/project.go @@ -33,20 +33,20 @@ func logProjectStart(logPrefix string, projectSpec apigitlab.ProjectSpec) { } // CreateProject - -func CreateProject(gitlabCredentials GitlabCredentials, projectSpec apigitlab.ProjectSpec, httpClient *http.Client) error { +func CreateProject(gitlabCredentials GitlabCredentials, projectSpec apigitlab.ProjectSpec, httpClient *http.Client) (int, error) { logPrefix := "CreateProject" logProjectStart(logPrefix, projectSpec) err := ValidateGitlabPath(projectSpec.FullPath) if err != nil { processProjectError(logPrefix, err) - return err + return 0, err } client, err := gitlab.NewClient(gitlabCredentials.ServerURL, gitlabCredentials.ServerToken, httpClient) if err != nil { processProjectError(logPrefix, err) - return err + return 0, err } // Identify parent Group for project @@ -57,15 +57,15 @@ func CreateProject(gitlabCredentials GitlabCredentials, projectSpec apigitlab.Pr parentGroup, statusCode, err := client.GetGroupByFullPath(&groupFullPath) if err != nil { - return fmt.Errorf("failed to find group with fullPath %s. error: %v", groupFullPath, err) + return 0, fmt.Errorf("failed to find group with fullPath %s. error: %v", groupFullPath, err) } if statusCode != http.StatusFound { - return fmt.Errorf("failed to find group with fullPath %s. status code: %d", groupFullPath, statusCode) + return 0, fmt.Errorf("failed to find group with fullPath %s. status code: %d", groupFullPath, statusCode) } ciConfigPath, err := generateCIConfigPath(projectPath, groupFullPath) if err != nil { - return fmt.Errorf("failed to generateCIConfigPath: %v", err) + return 0, fmt.Errorf("failed to generateCIConfigPath: %v", err) } projectOptions := gogitlab.CreateProjectOptions{ @@ -105,15 +105,15 @@ func CreateProject(gitlabCredentials GitlabCredentials, projectSpec apigitlab.Pr project, statusCode, err := client.AddProject(projectOptions) if err != nil { - return fmt.Errorf("failed to add project with name %s to group path %s. error: %v", *projectOptions.Name, groupFullPath, err) + return 0, fmt.Errorf("failed to add project with name %s to group path %s. error: %v", *projectOptions.Name, groupFullPath, err) } if statusCode != http.StatusCreated { - return fmt.Errorf(fmt.Sprintf("Failed to add project with name %s to group path %s. statusCode: %d", *projectOptions.Name, groupFullPath, statusCode)) + return 0, fmt.Errorf(fmt.Sprintf("Failed to add project with name %s to group path %s. statusCode: %d", *projectOptions.Name, groupFullPath, statusCode)) } rlog.Debugf("Added project %s to path %s with status code %d", project.Name, project.NameWithNamespace, statusCode) - return nil + return project.ID, nil } // UpdateProject - @@ -195,7 +195,7 @@ func DeleteProject(gitlabCredentials GitlabCredentials, projectSpec apigitlab.Pr } if statusCode != http.StatusFound { - return fmt.Errorf("failed to find project with fullPath %s", projectFullPath) + return fmt.Errorf("project not found (%d) fullPath %s", http.StatusNotFound, projectFullPath) } // delete and wait for delete to complete up to 500ms * 240 tries diff --git a/custom/p1/project_bootstrap.go b/custom/p1/project_bootstrap.go new file mode 100644 index 0000000000000000000000000000000000000000..e9fedb6bcf2d885708f60b2f684495b812f7a4a9 --- /dev/null +++ b/custom/p1/project_bootstrap.go @@ -0,0 +1,193 @@ +package p1 + +import ( + "fmt" + "os" + "strings" + + "github.com/romana/rlog" + gogitlab "github.com/xanzy/go-gitlab" + gitlabAPIVer "valkyrie.dso.mil/valkyrie-api/apis/gitlab/v1alpha1" + "valkyrie.dso.mil/valkyrie-api/clients/git" + "valkyrie.dso.mil/valkyrie-api/clients/gitlab" + "valkyrie.dso.mil/valkyrie-api/clients/utils" + templateCustom "valkyrie.dso.mil/valkyrie-api/custom/template" + templateConfig "valkyrie.dso.mil/valkyrie-api/custom/template/configurations" +) + +var directoryPrefix = "bootstrap-" +var newRepoInitializeFile = "README.md" + +// DefaultBootstrapBranch - by default bootstrap to the master branch +var DefaultBootstrapBranch = "master" + +func processProjectBootstrapError(logPrefix string, err error) { + rlog.Warnf("%s error: %v", logPrefix, err) +} + +//BootstrapProjectWithManifest - wrapper to be called from controller +func BootstrapProjectWithManifest(gitlabCredentials *gitlabAPIVer.GitlabCredentials, projectID int, bootstrapConfig gitlabAPIVer.BootstrapWithManifestRepo) (err error) { + var gitlabCreds GitlabCredentials + gitlabCreds.ServerURL = gitlabCredentials.Spec.URL + gitlabCreds.ServerUsername = gitlabCredentials.Spec.Username + gitlabCreds.ServerToken = gitlabCredentials.Spec.AccessTokenPrivate + return BootstrapProjectWithManifestCreds(gitlabCreds, projectID, bootstrapConfig) +} + +// BootstrapProjectWithManifestCreds - use simple gitlabCredentials object for credentials +func BootstrapProjectWithManifestCreds(gitlabCredentials GitlabCredentials, projectID int, bootstrapConfig gitlabAPIVer.BootstrapWithManifestRepo) (err error) { + logPrefix := "BootstrapProjectWithManfiest" + + // TODO: figure out how to convert bootstrapConfig into templateConfigObj + templateConfigObj := templateConfig.Microservice{ + ProjectName: "PROJECT_NAME", + ImpactLevel: "IL2", + MetaData: "xx", + } + + // identify cookie cutter folder location + RootDir, err := utils.CurrentDir() + if err != nil { + processProjectBootstrapError(logPrefix+" utils.CurrentDir", err) + return err + } + + var templatePath string = RootDir + "/resources/templates/manifests/mission-bootstrap/integrations" + // create a temp copy of the templatePath + scratchTemplatePath, err := utils.CreateTempDirWithFile(templatePath, "bootstrap-manifest-", false) + if err != nil { + processProjectBootstrapError(logPrefix+" utils.CreateTempDirWithFile", err) + return err + } + + templateOutputDirectory, err := templateCustom.ProcessCookieCutterFile(scratchTemplatePath, &templateConfigObj) + if err != nil { + processProjectBootstrapError(logPrefix+" templateCustom.ProcessCookieCutterFile", err) + return err + } + + // bootstrap project with contents of templateOutputDirectory + err = BootstrapProject(gitlabCredentials, projectID, templateOutputDirectory, DefaultBootstrapBranch) + + // cleanup + // rlog.Errorf("%s scratchDir %s", logPrefix,scratchTemplatePath) + // rlog.Errorf("%s ccOutputDir %s", logPrefix,templateOutputDirectory) + // remove templateOutputDirectory + os.RemoveAll(templateOutputDirectory) + // remove scratchTemplatePath + os.RemoveAll(scratchTemplatePath) + return +} + +//BootstrapProject add filepath to an existing gitlab project ID repository +func BootstrapProject(gitlabCredentials GitlabCredentials, projectID int, sourcePath string, branch string) error { + logPrefix := "BootstrapProject" + + // gitlab username token credentials to use for git ops + gitlabUsername := "oauth2" + gitlabToken := gitlabCredentials.ServerToken + + // create a gitlab client + gitlabClient, err := gitlab.NewClient(gitlabCredentials.ServerURL, gitlabCredentials.ServerToken, nil) + if err != nil { + processProjectBootstrapError(logPrefix+" gitlab.NewClient", err) + return err + } + + project, _, err := gitlabClient.GetProject(projectID) + if err != nil { + processProjectBootstrapError(logPrefix+" gitlabClient.GetProject", err) + return err + } + gitlabRepositoryURL := project.HTTPURLToRepo + + // check that repo is empty. otherwise return err + fileTree, _, err := gitlabClient.GetRepositoryTree(projectID) + if err != nil { + processProjectBootstrapError(logPrefix+" gitlabClient.GetRepositoryTree", err) + return err + } + if !repoIsEmpty(fileTree) { + err = fmt.Errorf("repository %s not empty", project.PathWithNamespace) + processProjectBootstrapError(logPrefix+" repoIsEmpty", err) + return err + } + + // be sure to cleanup temp directory + workDir, err := utils.CreateTempDir(directoryPrefix, utils.TempRoot) + if err != nil { + processProjectBootstrapError(logPrefix+" CreateTempDir", err) + return err + } + rlog.Debugf("%s - CreateTempDir - Created temp workDir %s", logPrefix, workDir) + + client, err := git.NewClientOSFS( + gitlabUsername, + gitlabToken, + gitlabRepositoryURL, + workDir, + ) + if err != nil { + processProjectBootstrapError(logPrefix+" git.NewClientOSFS", err) + return err + } + // If remote does NOT already exist, then this is a LOCAL Branch only until pushed + err = client.CheckoutOrCreateBranch(branch) + if err != nil { + processProjectBootstrapError(logPrefix+" CheckoutOrCreateBranch", err) + return err + } + + // copy files from sourcePath to root of repository + err = utils.CopyDir(sourcePath, workDir) + if err != nil { + processProjectBootstrapError(logPrefix+" utils.CopyDir", err) + return err + } + + // add everything in working tree + err = client.AddFile(".") + if err != nil { + processProjectBootstrapError(logPrefix+" client.AddFile", err) + return err + } + err = client.Commit("Valkyrie - Bootstrap") + if err != nil { + processProjectBootstrapError(logPrefix+" client.Commit", err) + return err + } + + // push to remote to create remote branch if does NOT already + err = client.Push(branch) + //if already up-to-date, then it's already deleted + if err != nil && err.Error() == "already up-to-date" { + rlog.Debugf(logPrefix+" client.Push %s - already up-to-date", branch) + err = nil + } + if err != nil { + processProjectBootstrapError(logPrefix+" client.Push", err) + return err + } + rlog.Debugf("Checkout and Push branch %s - success", branch) + + // cleanup + os.RemoveAll(workDir) + return nil +} + +// repoIsEmpty - test if repo is effectively empty +func repoIsEmpty(fileTree []*gogitlab.TreeNode) bool { + // if no files and repo is initialized + if len(fileTree) == 0 { + return true + } + // if only one file and it is the newRepoInitializeFile + if len(fileTree) == 1 { + file := fileTree[0] + if strings.EqualFold(file.Name, newRepoInitializeFile) { + return true + } + } + // repo is NOT empty + return false +} diff --git a/custom/p1/project_bootstrap_test.go b/custom/p1/project_bootstrap_test.go new file mode 100644 index 0000000000000000000000000000000000000000..33cc7f6fec80ca0a563d64b4f18868fb47231271 --- /dev/null +++ b/custom/p1/project_bootstrap_test.go @@ -0,0 +1,167 @@ +package p1 + +import ( + "fmt" + "os" + "testing" + + gogitlab "github.com/xanzy/go-gitlab" + gitlabAPIVer "valkyrie.dso.mil/valkyrie-api/apis/gitlab/v1alpha1" +) + +// force working directory to be root of the project folder +var testPath = "" + +func init() { + wd, _ := os.Getwd() + // save current directory as TestPath + testPath = wd + if err := os.Chdir(wd + "/../../"); err != nil { + panic(err) + } +} +func Test_processProjectBootstrapError(t *testing.T) { + type args struct { + logPrefix string + err error + } + tests := []struct { + name string + args args + }{ + {name: "test", args: args{logPrefix: "prefix-", err: fmt.Errorf("test error")}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + processProjectBootstrapError(tt.args.logPrefix, tt.args.err) + }) + } +} + +func TestBootstrapProjectWithManifest(t *testing.T) { + // test objects + testAPIUrl := "https://test/api/v4/" + testToken := "token" + testUsername := "username" + testCreds := gitlabAPIVer.GitlabCredentials{ + Spec: gitlabAPIVer.GitlabCredentialsSpec{URL: testAPIUrl, Username: testUsername, AccessTokenPrivate: testToken}, + } + testBootstrapConfig := gitlabAPIVer.BootstrapWithManifestRepo{} + + type args struct { + gitlabCredentials *gitlabAPIVer.GitlabCredentials + projectID int + bootstrapConfig gitlabAPIVer.BootstrapWithManifestRepo + } + tests := []struct { + name string + args args + wantErr bool + }{ + {name: "test", args: args{gitlabCredentials: &testCreds, projectID: 11, bootstrapConfig: testBootstrapConfig}, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := BootstrapProjectWithManifest(tt.args.gitlabCredentials, tt.args.projectID, tt.args.bootstrapConfig); (err != nil) != tt.wantErr { + t.Errorf("BootstrapProjectWithManifest() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestBootstrapProjectWithManifestCreds(t *testing.T) { + // test objects + testAPIUrl := "https://test/api/v4/" + testToken := "token" + testUsername := "username" + testCreds := GitlabCredentials{ + ServerURL: testAPIUrl, + ServerUsername: testUsername, + ServerToken: testToken, + } + testBootstrapConfig := gitlabAPIVer.BootstrapWithManifestRepo{} + testBootstrapConfig2 := gitlabAPIVer.BootstrapWithManifestRepo{ + Services: []gitlabAPIVer.Service{ + {Name: "name", ContainerRegistry: "registry.il2.dso.mil", Port: 8081}, + }, + } + + type args struct { + gitlabCredentials GitlabCredentials + projectID int + bootstrapConfig gitlabAPIVer.BootstrapWithManifestRepo + } + tests := []struct { + name string + args args + wantErr bool + }{ + {name: "test", args: args{gitlabCredentials: testCreds, projectID: 11, bootstrapConfig: testBootstrapConfig}, wantErr: true}, + {name: "test", args: args{gitlabCredentials: testCreds, projectID: 11, bootstrapConfig: testBootstrapConfig2}, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := BootstrapProjectWithManifestCreds(tt.args.gitlabCredentials, tt.args.projectID, tt.args.bootstrapConfig); (err != nil) != tt.wantErr { + t.Errorf("BootstrapProjectWithManifestCreds() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestBootstrapProject(t *testing.T) { + // test objects + testAPIUrl := "https://test/api/v4/" + testToken := "token" + testUsername := "username" + testCreds := GitlabCredentials{ + ServerURL: testAPIUrl, + ServerUsername: testUsername, + ServerToken: testToken, + } + testSourcePath := "/tmp/test-dir" + + type args struct { + gitlabCredentials GitlabCredentials + projectID int + sourcePath string + branch string + } + tests := []struct { + name string + args args + wantErr bool + }{ + {name: "test", args: args{gitlabCredentials: testCreds, projectID: 11, sourcePath: testSourcePath, branch: DefaultBootstrapBranch}, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := BootstrapProject(tt.args.gitlabCredentials, tt.args.projectID, tt.args.sourcePath, tt.args.branch); (err != nil) != tt.wantErr { + t.Errorf("BootstrapProject() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func Test_repoIsEmpty(t *testing.T) { + testTreeGood := []*gogitlab.TreeNode{{ID: "1", Name: newRepoInitializeFile}} + testTreeBad := []*gogitlab.TreeNode{{ID: "1", Name: "bad filename"}} + type args struct { + fileTree []*gogitlab.TreeNode + } + tests := []struct { + name string + args args + want bool + }{ + {name: "test 1", args: args{fileTree: nil}, want: true}, + {name: "test 2", args: args{fileTree: testTreeGood}, want: true}, + {name: "test 3", args: args{fileTree: testTreeBad}, want: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := repoIsEmpty(tt.args.fileTree); got != tt.want { + t.Errorf("repoIsEmpty() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/custom/p1/project_test.go b/custom/p1/project_test.go index f512615bbf47cb8f99d12d8f256748c2aeda6b2b..c7c343dc2ecbd65a92142cd1ba5d0b8df7efc0c6 100644 --- a/custom/p1/project_test.go +++ b/custom/p1/project_test.go @@ -48,6 +48,7 @@ func TestCreateProject(t *testing.T) { // test objects testAPIUrl := "https://test/api/v4/" testToken := "token" + testUsername := "username" testProjectSpec := apigitlab.ProjectSpec{Name: "Project Name", FullPath: testProjectFullPath} type args struct { @@ -60,11 +61,12 @@ func TestCreateProject(t *testing.T) { args args wantErr bool }{ - {name: "test", args: args{gitlabCredentials: GitlabCredentials{testAPIUrl, testToken}, projectSpec: testProjectSpec, httpClient: testHTTPClient}, wantErr: false}, + {name: "test", args: args{gitlabCredentials: GitlabCredentials{testAPIUrl, testUsername, testToken}, projectSpec: testProjectSpec, httpClient: testHTTPClient}, wantErr: false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if err := CreateProject(tt.args.gitlabCredentials, tt.args.projectSpec, tt.args.httpClient); (err != nil) != tt.wantErr { + _, err := CreateProject(tt.args.gitlabCredentials, tt.args.projectSpec, tt.args.httpClient) + if (err != nil) != tt.wantErr { t.Errorf("CreateProject() error = %v, wantErr %v", err, tt.wantErr) } }) @@ -109,6 +111,7 @@ func TestDeleteProject(t *testing.T) { // test objects testAPIUrl := "https://test/api/v4/" testToken := "token" + testUsername := "username" testProjectSpec := apigitlab.ProjectSpec{Name: "Project Name", FullPath: testProjectFullPath} type args struct { @@ -121,7 +124,7 @@ func TestDeleteProject(t *testing.T) { args args wantErr bool }{ - {name: "test", args: args{gitlabCredentials: GitlabCredentials{testAPIUrl, testToken}, projectSpec: testProjectSpec, httpClient: testHTTPClient}, wantErr: false}, + {name: "test", args: args{gitlabCredentials: GitlabCredentials{testAPIUrl, testUsername, testToken}, projectSpec: testProjectSpec, httpClient: testHTTPClient}, wantErr: false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -172,6 +175,7 @@ func TestUpdateProject(t *testing.T) { // test objects testAPIUrl := "https://test/api/v4/" testToken := "token" + testUsername := "username" testProjectSpec := apigitlab.ProjectSpec{Name: "Project Name", FullPath: testProjectFullPath} type args struct { @@ -184,7 +188,7 @@ func TestUpdateProject(t *testing.T) { args args wantErr bool }{ - {name: "test", args: args{gitlabCredentials: GitlabCredentials{testAPIUrl, testToken}, projectSpec: testProjectSpec, httpClient: testHTTPClient}, wantErr: false}, + {name: "test", args: args{gitlabCredentials: GitlabCredentials{testAPIUrl, testUsername, testToken}, projectSpec: testProjectSpec, httpClient: testHTTPClient}, wantErr: false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/custom/p1/types.go b/custom/p1/types.go index 705e692d72d4f022b4e114cecb600a1bb1102139..5832c050f9db90fd0a79c39f600303b9797737a7 100644 --- a/custom/p1/types.go +++ b/custom/p1/types.go @@ -6,8 +6,9 @@ import ( // GitlabCredentials - type GitlabCredentials struct { - ServerURL string - ServerToken string + ServerURL string + ServerUsername string + ServerToken string } // enumeration for pipeline yaml template paths diff --git a/custom/template/configurations/template_config.go b/custom/template/configurations/template_config.go index 3d1483c3164942f8dea10f0caf39bd207b382a6b..d312315222f8049ebc903e770f77b8226b1d64ca 100644 --- a/custom/template/configurations/template_config.go +++ b/custom/template/configurations/template_config.go @@ -1,5 +1,8 @@ package templateconfiguration +// Common Cookie Cutter Template Objects +// These Structs are converted to JSON then used as inputs to Cookie Cutter Templates + // Application - type Application struct { ProjectName string `json:"project_name"` diff --git a/custom/template/template_test.go b/custom/template/template_test.go index 913393b1517b66d5f3aee80471a702f6f6dcd6d8..5a761944031d13aacec337e1716582d51778a163 100644 --- a/custom/template/template_test.go +++ b/custom/template/template_test.go @@ -2,9 +2,13 @@ package template import ( "errors" + "fmt" + "math" "os" "strings" "testing" + + "valkyrie.dso.mil/valkyrie-api/clients/utils" ) func Test_createTempDir(t *testing.T) { @@ -27,6 +31,7 @@ func Test_createTempDir(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { osMkdirAllOrig := os.MkdirAll + // IF MOCK then MUST restore to original when exiting if tt.mockKey == "osMkdirAll" { osMkdirAll = osMkdirAllBad } @@ -37,20 +42,26 @@ func Test_createTempDir(t *testing.T) { } else { t.Errorf("createTempDir() UNEXPECTED error = %v, wantErr %v", err, tt.wantErr) } + osMkdirAll = osMkdirAllOrig return } if !strings.HasPrefix(gotTempDir, tt.wantTempDir) { t.Errorf("createTempDir() = %v, want %v", gotTempDir, tt.wantTempDir) } os.RemoveAll(gotTempDir) - if tt.mockKey == "osMkdirAll" { - osMkdirAll = osMkdirAllOrig - } + osMkdirAll = osMkdirAllOrig }) } } func TestWriteCookieCutterFile(t *testing.T) { + + type badStruct struct { + Name string `json:"name"` + Age float64 `json:"age"` + } + testObjBad := badStruct{Name: "name", Age: math.Inf(1)} + testFilePath := os.TempDir() + "cc_test.json" type args struct { filePath string @@ -62,7 +73,8 @@ func TestWriteCookieCutterFile(t *testing.T) { wantErr bool }{ {name: "test pass 1", args: args{filePath: testFilePath, obj: struct{ Test string }{Test: "test"}}, wantErr: false}, - {name: "test pass 2", args: args{filePath: "/xx/xx.json", obj: struct{ Test int }{Test: 7}}, wantErr: true}, + {name: "test fail 1", args: args{filePath: testFilePath, obj: testObjBad}, wantErr: true}, + {name: "test fail 2", args: args{filePath: "/xx/xx.json", obj: struct{ Test int }{Test: 7}}, wantErr: true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -83,7 +95,26 @@ func TestWriteCookieCutterFile(t *testing.T) { } func TestReadCookieCutterFile(t *testing.T) { - testFilePath := os.TempDir() + "cc_test.json" + testFilePathGood := os.TempDir() + "cc_test_good.json" + testFilePathBad := os.TempDir() + "cc_test_missing.json" + testFilePathMarshallError := os.TempDir() + "cc_test_error.json" + + tf1, err := os.Create(testFilePathGood) + if err != nil { + t.Errorf("os.Create failed %v", err) + } + newLine1 := "{}" + fmt.Fprintln(tf1, newLine1) + tf1.Close() + + tf2, err := os.Create(testFilePathMarshallError) + if err != nil { + t.Errorf("os.Create failed %v", err) + } + newLine2 := "{invalid data}" + fmt.Fprintln(tf2, newLine2) + tf2.Close() + type args struct { filePath string obj interface{} @@ -93,16 +124,12 @@ func TestReadCookieCutterFile(t *testing.T) { args args wantErr bool }{ - {name: "test pass 1", args: args{filePath: testFilePath, obj: struct{ Test string }{Test: "test"}}, wantErr: false}, - {name: "test pass 2", args: args{filePath: "/xx/xx.json", obj: struct{ Test int }{Test: 7}}, wantErr: true}, + {name: "test pass 1", args: args{filePath: testFilePathGood, obj: struct{ Test string }{Test: "test"}}, wantErr: true}, + {name: "test fail 1", args: args{filePath: testFilePathBad, obj: struct{ Test string }{Test: "test"}}, wantErr: true}, + {name: "test fail 2", args: args{filePath: testFilePathMarshallError, obj: struct{ Test string }{Test: "test"}}, wantErr: true}, + {name: "test fail 3", args: args{filePath: "/xx/xx.json", obj: struct{ Test int }{Test: 7}}, wantErr: true}, } for _, tt := range tests { - // wrire a Cookiecutter file first so we can read it - err := WriteCookieCutterFile(testFilePath, tt.args.obj) - if err != nil { - t.Errorf("WriteCookieCutterFile() error = %v, wantErr %v", err, tt.wantErr) - return - } t.Run(tt.name, func(t *testing.T) { err := ReadCookieCutterFile(tt.args.filePath, tt.args.obj) if err != nil { @@ -122,6 +149,7 @@ func TestReadCookieCutterFile(t *testing.T) { func TestProcessCookieCutterFile(t *testing.T) { testConfigPath := os.TempDir() + "test/" + testConfigPathExists, _ := utils.CreateTempDir("unit-test-", utils.TempRoot) type args struct { cookieCutterConfigPath string templateConfigObject interface{} @@ -134,9 +162,11 @@ func TestProcessCookieCutterFile(t *testing.T) { mockKey string testCounter int }{ + {name: "test pass 1", args: args{cookieCutterConfigPath: testConfigPathExists, templateConfigObject: struct{ Test string }{Test: "test"}}, wantOutputDirectory: "/tmp/cc", wantErr: true, mockKey: "", testCounter: 1}, {name: "test fail 1", args: args{cookieCutterConfigPath: testConfigPath, templateConfigObject: struct{ Test string }{Test: "test"}}, wantOutputDirectory: "/tmp/cc", wantErr: true, mockKey: "", testCounter: 1}, {name: "test fail 2", args: args{cookieCutterConfigPath: "/tmp", templateConfigObject: struct{ Test string }{Test: "test"}}, wantOutputDirectory: "/tmp/cc", wantErr: true, mockKey: "osMkdirAll", testCounter: 1}, {name: "test fail 3", args: args{cookieCutterConfigPath: "/tmp", templateConfigObject: struct{ Test string }{Test: "test"}}, wantOutputDirectory: "/tmp/cc", wantErr: true, mockKey: "osMkdirAll", testCounter: 2}, + {name: "test fail 4", args: args{cookieCutterConfigPath: "/tmp", templateConfigObject: struct{ Test string }{Test: "test"}}, wantOutputDirectory: "/tmp/cc", wantErr: true, mockKey: "osMkdirAll", testCounter: 3}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/integration-tests/custom/p1/data/angular-world.tar.gz b/integration-tests/custom/p1/data/angular-world.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..4e671867b8ea1083570ea3a040ff4fa6c3480185 Binary files /dev/null and b/integration-tests/custom/p1/data/angular-world.tar.gz differ diff --git a/integration-tests/custom/p1/p1_groups_test.go b/integration-tests/custom/p1/p1_groups_test.go index a0948bbd015770962d49e00fd9905697ca1ed480..0b07ec63cbc5b5647b70855b09e8ae1eae20c9a9 100644 --- a/integration-tests/custom/p1/p1_groups_test.go +++ b/integration-tests/custom/p1/p1_groups_test.go @@ -9,18 +9,13 @@ import ( custom_p1 "valkyrie.dso.mil/valkyrie-api/custom/p1" ) -/* - -p1Config is initialized in the TestMain wrapper method - -*/ const integrationTestGroupPath = "int-test-group" func Test_P1_AddGroup(t *testing.T) { logPrefix := "Test_P1_AddGroup" t.Run("test", func(t *testing.T) { groupSpec := v1alpha1.GroupSpec{ - Name: "My Test Group", + Name: "Int Test Group", FullPath: p1Config.GitlabIntTestRootGroupPath + "/" + integrationTestGroupPath, } @@ -43,7 +38,7 @@ func Test_P1_UpdateGroup(t *testing.T) { logPrefix := "Test_P1_UpdateGroup" t.Run("test", func(t *testing.T) { groupSpec := v1alpha1.GroupSpec{ - Name: "My Test Updated Group", + Name: "Int Test Updated Group", FullPath: p1Config.GitlabIntTestRootGroupPath + "/" + integrationTestGroupPath, } diff --git a/integration-tests/custom/p1/p1_main_test.go b/integration-tests/custom/p1/p1_main_test.go index 8d3339cc0e79cf263723de392feade9c030170c1..613ce611c592491028f8b5ae9d34b748d49e55a3 100644 --- a/integration-tests/custom/p1/p1_main_test.go +++ b/integration-tests/custom/p1/p1_main_test.go @@ -53,6 +53,7 @@ var p1Config P1Config // packageSetup - setup environment for integration testing func packageSetup() error { p1Config.WorkingDir, _ = os.Getwd() + var ok bool var err error @@ -139,21 +140,21 @@ func addRootGroup() error { // dont fail if group already exists fmt.Printf("Error! AddRootGroup %s error = %v %d\n", path, err, statusCode) return err - } else { - p1Config.GitlabIntTestRootGroupID = got.ID - fmt.Printf("AddGroup %s %s %s %s %d \n", got.Name, got.Path, got.FullPath, got.WebURL, statusCode) } + p1Config.GitlabIntTestRootGroupID = got.ID + fmt.Printf("AddGroup %s %s %s %s %d \n", got.Name, got.Path, got.FullPath, got.WebURL, statusCode) + } return nil } // Insure the integration test project exists -func addIntTestProject(projectUrl string) (int, error) { +func addIntTestProject(projectURL string) (int, error) { var project *gogitlab.Project = nil var responseCode int - url, err := url.Parse(projectUrl) + url, err := url.Parse(projectURL) if err != nil { return 0, err } @@ -166,29 +167,29 @@ func addIntTestProject(projectUrl string) (int, error) { if responseCode == http.StatusFound { return project.ID, nil - } else { - path := p1Config.GitlabIntTestProjectPath - name := p1Config.GitlabIntTestProjectPath - description := "Integration Test Project" - configPath := "a/b/config.yml@group1/group2/project" - initializeWithReadme := true - - projectOpts := gogitlab.CreateProjectOptions{ - Name: &name, - Path: &path, - Description: &description, - NamespaceID: &p1Config.GitlabIntTestRootGroupID, - CIConfigPath: &configPath, - InitializeWithReadme: &initializeWithReadme, - } - got, statusCode, err := p1Config.GitlabClient.AddProject(projectOpts) - - if err != nil { - fmt.Printf("ClientImpl.AddProject() error = %v %d \n", err, statusCode) - return 0, err - } + } + path := p1Config.GitlabIntTestProjectPath + name := p1Config.GitlabIntTestProjectPath + description := "Integration Test Project" + configPath := "a/b/config.yml@group1/group2/project" + initializeWithReadme := true + + projectOpts := gogitlab.CreateProjectOptions{ + Name: &name, + Path: &path, + Description: &description, + NamespaceID: &p1Config.GitlabIntTestRootGroupID, + CIConfigPath: &configPath, + InitializeWithReadme: &initializeWithReadme, + } + got, statusCode, err := p1Config.GitlabClient.AddProject(projectOpts) - fmt.Printf("AddProject %s %s %s %s %d", got.Name, got.Path, got.PathWithNamespace, got.WebURL, statusCode) - return got.ID, nil + if err != nil { + fmt.Printf("ClientImpl.AddProject() error = %v %d \n", err, statusCode) + return 0, err } + + fmt.Printf("AddProject %s %s %s %s %d", got.Name, got.Path, got.PathWithNamespace, got.WebURL, statusCode) + return got.ID, nil + } diff --git a/integration-tests/custom/p1/p1_projects_bootstrap_test.go b/integration-tests/custom/p1/p1_projects_bootstrap_test.go new file mode 100644 index 0000000000000000000000000000000000000000..c599c6be69dc958bdad8d05c94a221c7ede567e8 --- /dev/null +++ b/integration-tests/custom/p1/p1_projects_bootstrap_test.go @@ -0,0 +1,191 @@ +// +build integration + +package integration + +import ( + "os" + "strings" + "testing" + + gitlabAPIVer "valkyrie.dso.mil/valkyrie-api/apis/gitlab/v1alpha1" + "valkyrie.dso.mil/valkyrie-api/clients/utils" + customP1 "valkyrie.dso.mil/valkyrie-api/custom/p1" +) + +// go clean -testcache && go test -tags "integration" ./integration-tests/custom/p1/p1_main_test.go ./integration-tests/custom/p1/p1_projects_bootstrap_test.go + +const integrationTestProjectBootstrapPath = "int-test-bootstrap-project" +const integrationTestProjectDataFile = "angular-world.tar.gz" + +// force working directory to be root of the project folder +var testPath = "" +func init() { + wd, _ := os.Getwd() + // save current directory as TestPath + testPath = wd + if err := os.Chdir(wd + "/../../../"); err != nil { + panic(err) + } +} +// Delete bootstrap test project if exists and re-create empty clean project +func Test_P1_ProjectBootstrap_DataFile(t *testing.T) { + logPrefix := "Test_P1_AddProject" + t.Run("test", func(t *testing.T) { + projectSpec := gitlabAPIVer.ProjectSpec{ + Name: "Int Test Bootstrap Project 1", + FullPath: p1Config.GitlabIntTestRootGroupPath + "/" + integrationTestProjectBootstrapPath, + } + + creds := customP1.GitlabCredentials{ + ServerURL: p1Config.GitlabAPIURL, + ServerToken: p1Config.GitlabAPIToken, + } + + var err error + + err = customP1.DeleteProject(creds, projectSpec, nil) + if err != nil && strings.Contains(err.Error(), "not found") { + t.Logf("DeleteProject- project not found %s", projectSpec.FullPath) + err = nil + } + if err != nil { + t.Logf("DeleteProject() error = %v", err) + } + t.Logf("%s deleted project %s", logPrefix, projectSpec.FullPath) + + projectID, err := customP1.CreateProject(creds, projectSpec, nil) + if err != nil { + t.Errorf("CreateProject() error = %v", err) + return + } + + t.Logf("%s created project %v", logPrefix, projectSpec.FullPath) + + // be sure to cleanup temp directory + tempDir, err := utils.CreateTempDir("int-test-", utils.TempRoot) + if err != nil { + t.Errorf("Error - CreateTempDir %v", err) + return + } + + testDataPath := testPath + utils.PathSeparator + "data" + err = utils.CopyDir(testDataPath, tempDir) + if err != nil { + t.Errorf("Error - CopyDir %v", err) + return + } + t.Logf("temp directory %s", tempDir) + err = customP1.BootstrapProject(creds, projectID, tempDir, customP1.DefaultBootstrapBranch) + if err != nil { + t.Errorf("Error - BootstrapProject %v", err) + return + } + + // cleanup + os.RemoveAll(tempDir) + t.Logf("Bootstrap completed") + }) +} + +func Test_P1_ProjectBootstrap_DataFile_Expanded(t *testing.T) { + logPrefix := "Test_P1_AddProject" + t.Run("test", func(t *testing.T) { + projectSpec := gitlabAPIVer.ProjectSpec{ + Name: "Int Test Bootstrap Project 2", + FullPath: p1Config.GitlabIntTestRootGroupPath + "/" + integrationTestProjectBootstrapPath + "-2", + } + + creds := customP1.GitlabCredentials{ + ServerURL: p1Config.GitlabAPIURL, + ServerToken: p1Config.GitlabAPIToken, + } + + var err error + + err = customP1.DeleteProject(creds, projectSpec, nil) + if err != nil && strings.Contains(err.Error(), "not found") { + t.Logf("DeleteProject- project not found %s", projectSpec.FullPath) + err = nil + } + if err != nil { + t.Logf("DeleteProject() error = %v", err) + } + t.Logf("%s deleted project %s", logPrefix, projectSpec.FullPath) + + projectID, err := customP1.CreateProject(creds, projectSpec, nil) + if err != nil { + t.Errorf("CreateProject() error = %v", err) + return + } + + t.Logf("%s created project %v", logPrefix, projectSpec.FullPath) + + testDataPath := testPath + utils.PathSeparator + "data" + tempDir, err := utils.CreateTempDirWithFile(testDataPath+utils.PathSeparator+integrationTestProjectDataFile, "int-test-", true) + if err != nil { + t.Errorf("CreateTempDirWithFile() error = %v", err) + return + } + + t.Logf("temp directory %s", tempDir) + err = customP1.BootstrapProject(creds, projectID, tempDir, customP1.DefaultBootstrapBranch) + if err != nil { + t.Errorf("Error - BootstrapProject %v", err) + return + } + + // cleanup + os.RemoveAll(tempDir) + t.Logf("Bootstrap completed") + }) +} + +func Test_P1_ProjectBootstrap_CookieCutter(t *testing.T) { + logPrefix := "Test_P1_AddProject" + t.Run("test", func(t *testing.T) { + projectSpec := gitlabAPIVer.ProjectSpec{ + Name: "Int Test Bootstrap Project CC", + FullPath: p1Config.GitlabIntTestRootGroupPath + "/" + integrationTestProjectBootstrapPath + "-cc", + } + + creds := customP1.GitlabCredentials{ + ServerURL: p1Config.GitlabAPIURL, + ServerToken: p1Config.GitlabAPIToken, + ServerUsername: "oauth2", + } + + var err error + + err = customP1.DeleteProject(creds, projectSpec, nil) + if err != nil && strings.Contains(err.Error(), "not found") { + t.Logf("DeleteProject- project not found %s", projectSpec.FullPath) + err = nil + } + if err != nil { + t.Logf("DeleteProject() error = %v", err) + } + t.Logf("%s deleted project %s", logPrefix, projectSpec.FullPath) + + projectID, err := customP1.CreateProject(creds, projectSpec, nil) + if err != nil { + t.Errorf("CreateProject() error = %v", err) + return + } + + t.Logf("%s created project %v", logPrefix, projectSpec.FullPath) + + // err = customP1.BootstrapProject(creds, projectID, tempDir, customP1.DefaultBootstrapBranch) + bootstrapConfig := gitlabAPIVer.BootstrapWithManifestRepo{ + Services: []gitlabAPIVer.Service{ + {Name: "name", ContainerRegistry: "registry.il2.dso.mil", Port: 8081}, + }, + } + err = customP1.BootstrapProjectWithManifestCreds(creds, projectID, bootstrapConfig) + if err != nil { + t.Errorf("Error - BootstrapProject %v", err) + return + } + + t.Logf("Bootstrap completed") + }) +} diff --git a/integration-tests/custom/p1/p1_projects_test.go b/integration-tests/custom/p1/p1_projects_test.go index 45ff8948d21ca44f21ee8f7b997f91784f96c3b8..cc501b405bf1e5bb41ee1f09ec5df16283642244 100644 --- a/integration-tests/custom/p1/p1_projects_test.go +++ b/integration-tests/custom/p1/p1_projects_test.go @@ -9,18 +9,13 @@ import ( custom_p1 "valkyrie.dso.mil/valkyrie-api/custom/p1" ) -/* - -P1Config is initialized in the TestMain wrapper method - -*/ const integrationTestProjectPath = "int-test-project" func Test_P1_AddProject(t *testing.T) { logPrefix := "Test_P1_AddProject" t.Run("test", func(t *testing.T) { projectSpec := v1alpha1.ProjectSpec{ - Name: "My Test Project", + Name: "Int Test Add Project", FullPath: p1Config.GitlabIntTestRootGroupPath + "/" + integrationTestProjectPath, } @@ -35,15 +30,15 @@ func Test_P1_AddProject(t *testing.T) { if err != nil { t.Logf("DeleteProject() error = %v", err) } - t.Logf("%s deleted project %v", logPrefix, projectSpec) + t.Logf("%s deleted project with fullpath %v", logPrefix, projectSpec) - err = custom_p1.CreateProject(creds, projectSpec, nil) + projectID, err := custom_p1.CreateProject(creds, projectSpec, nil) if err != nil { t.Errorf("CreateProject() error = %v", err) return } - t.Logf("%s created project %v", logPrefix, projectSpec) + t.Logf("%s created project with id %d fullpath %v", logPrefix, projectID, projectSpec) }) } @@ -51,7 +46,7 @@ func Test_P1_UpdateProject(t *testing.T) { logPrefix := "Test_P1_UpdateProject" t.Run("test", func(t *testing.T) { projectSpec := v1alpha1.ProjectSpec{ - Name: "My Test Updated Project", + Name: "Int Test Updated Project", FullPath: p1Config.GitlabIntTestRootGroupPath + "/" + integrationTestProjectPath, } @@ -68,3 +63,26 @@ func Test_P1_UpdateProject(t *testing.T) { t.Logf("%s update project %v", logPrefix, projectSpec) }) } + +func Test_P1_DeleteProject(t *testing.T) { + logPrefix := "Test_P1_UpdateProject" + t.Run("test", func(t *testing.T) { + projectSpec := v1alpha1.ProjectSpec{ + Name: "Int Test Updated Project", + FullPath: p1Config.GitlabIntTestRootGroupPath + "/" + integrationTestProjectPath, + } + + creds := custom_p1.GitlabCredentials{ + ServerURL: p1Config.GitlabAPIURL, + ServerToken: p1Config.GitlabAPIToken, + } + + var err error + + err = custom_p1.DeleteProject(creds, projectSpec, nil) + if err != nil { + t.Logf("DeleteProject() error = %v", err) + } + t.Logf("%s deleted project with fullpath %v", logPrefix, projectSpec) + }) +} \ No newline at end of file diff --git a/integration-tests/gitclient/data/angular-world.tar.gz b/integration-tests/gitclient/data/angular-world.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..4e671867b8ea1083570ea3a040ff4fa6c3480185 Binary files /dev/null and b/integration-tests/gitclient/data/angular-world.tar.gz differ diff --git a/integration-tests/gitclient/data/angular-world.zip b/integration-tests/gitclient/data/angular-world.zip new file mode 100644 index 0000000000000000000000000000000000000000..57354662cdf876acdf21c0b8671f19db8a473d2e Binary files /dev/null and b/integration-tests/gitclient/data/angular-world.zip differ diff --git a/integration-tests/gitclient/data/dso-dns-staging.tar.gz b/integration-tests/gitclient/data/dso-dns-staging.tar.gz index 768fee64da64a51547ffcbf4c4a96f6f84b53e97..dbcb1c0d89afa9104ec58f07b5c7b0463709bfd4 100644 Binary files a/integration-tests/gitclient/data/dso-dns-staging.tar.gz and b/integration-tests/gitclient/data/dso-dns-staging.tar.gz differ diff --git a/integration-tests/gitclient/gitclient_addpath_test.go b/integration-tests/gitclient/gitclient_addpath_test.go index c8a0a88c5206b37371289acbb13cde3850921799..61d4ab4e79a04d159b1efa5b0c9423c26c95f8cb 100644 --- a/integration-tests/gitclient/gitclient_addpath_test.go +++ b/integration-tests/gitclient/gitclient_addpath_test.go @@ -11,7 +11,9 @@ import ( "valkyrie.dso.mil/valkyrie-api/clients/git" "valkyrie.dso.mil/valkyrie-api/clients/utils" ) + var directoryPrefix = "git-" + func TestGit_AddPath_Create_Branch(t *testing.T) { var branch = "testCreateBranchOSFS" @@ -157,23 +159,18 @@ func TestGit_AddPath_Checkout_AddNewTree_Push(t *testing.T) { os.Chdir(tempDir) os.Mkdir(newPath, 0755) os.Chdir(newPath) - f, err := os.Create("adam.txt") // creates a file at current directory + f, err := os.Create("adam.txt") // creates a file at current directory if err != nil { t.Errorf("%v", err) return } - - // f, err := os.OpenFile("test.txt", os.O_APPEND|os.O_WRONLY, 0644) - // if err != nil { - // fmt.Println(err) - // return - // } - newLine := "This file made by Adam. \nUpdated" + utils.DateTimeUTC() - _, err = fmt.Fprintln(f, newLine) - if err != nil { + + newLine := "This file made by Adam. \nUpdated" + utils.DateTimeUTC() + _, err = fmt.Fprintln(f, newLine) + if err != nil { t.Errorf("%v", err) return - } + } f.Close() err = client.AddFile(newPath) diff --git a/integration-tests/gitclient/gitclient_branch_test.go b/integration-tests/gitclient/gitclient_branch_test.go index aa83843aeed106f537afa86c5a487ebb4cc828af..d595a26c75ef8e8b92a0ec9780f597fdf6335d04 100644 --- a/integration-tests/gitclient/gitclient_branch_test.go +++ b/integration-tests/gitclient/gitclient_branch_test.go @@ -3,9 +3,7 @@ package gitclient import ( - "bufio" "fmt" - "strings" "testing" "valkyrie.dso.mil/valkyrie-api/clients/git" @@ -156,114 +154,3 @@ func TestGit_Checkout_AddNewFile_Push(t *testing.T) { return } } -func TestGit_Checkout_UpdateFile_Push(t *testing.T) { - branch := "testDnsUpdate" - // create the git client - client, err := git.NewClient( - gitclientTestConfig.GitlabIntTestUsername, - gitclientTestConfig.GitlabIntTestToken, - gitclientTestConfig.GitlabIntTestRepositoryURL, - ) - if err != nil { - t.Errorf("%v", err) - return - } - t.Logf("Created git client for repo %s", gitclientTestConfig.GitlabIntTestRepositoryURL) - - // checkout and delete the branch - err = client.CheckoutOrCreateBranch(branch) - if err != nil { - t.Errorf("%v", err) - return - } - t.Logf("Checked out branch %s", branch) - - err = client.DeleteRemoteBranch(branch) - if err != nil { - t.Errorf("Error %v", err) - return - } - t.Logf("Delete remote branch %s", branch) - - err = client.CheckoutOrCreateBranch(branch) - if err != nil { - t.Errorf("Error %v", err) - return - } - t.Logf("Checked out and Create branch %s", branch) - - // Add - fs := *client.GetFileSystem() - - dnsFilePath := "staging/dns/configuration/db.staging.dso.mil" - dnsFile, err := fs.Open(dnsFilePath) - if err != nil { - t.Errorf("%v", err) - return - } - - tempFile, err := fs.Create("temp_file") - if err != nil { - t.Errorf("%v", err) - return - } - - scanner := bufio.NewScanner(dnsFile) - var done bool - for scanner.Scan() { - line := scanner.Text() - count, err := tempFile.Write(scanner.Bytes()) - tempFile.Write([]byte("\n")) - if err != nil { - t.Errorf("%v", err) - } - t.Logf("written %d bytes", count) - - if !done && strings.Contains(line, "-il2 ") { - tempFile.Write([]byte("I am here...\n")) - done = true - } - } - - if err := scanner.Err(); err != nil { - t.Errorf("%v", err) - } - - tempFile.Close() - dnsFile.Close() - - err = fs.Rename("temp_file", dnsFilePath) - if err != nil { - t.Errorf("%v", err) - } - - tempFile, err = fs.Open(dnsFilePath) - if err != nil { - t.Errorf("%v", err) - } - - scanner2 := bufio.NewScanner(tempFile) - for scanner2.Scan() { - line := scanner2.Text() - t.Logf(line) - } - - tempFile.Close() - - err = client.AddFile(dnsFilePath) - if err != nil { - fmt.Printf("%v", err) - return - } - - err = client.Commit("Add new entry for new site - Valkyrie") - if err != nil { - fmt.Printf("%v", err) - return - } - - err = client.Push(branch) - if err != nil { - return - } -} diff --git a/integration-tests/gitclient/gitclient_dnsupdate_test.go b/integration-tests/gitclient/gitclient_dnsupdate_test.go new file mode 100644 index 0000000000000000000000000000000000000000..8bfbe094b7abcf610490c86a1bc8c0e684045151 --- /dev/null +++ b/integration-tests/gitclient/gitclient_dnsupdate_test.go @@ -0,0 +1,197 @@ +// +build integration + +package gitclient + +import ( + "bufio" + "fmt" + "strings" + "testing" + + "valkyrie.dso.mil/valkyrie-api/clients/git" + "valkyrie.dso.mil/valkyrie-api/clients/utils" +) + +var dnsTestBranch string = "integrationTestDNS" + +// go clean -testcache && go test -v -tags "integration" ./integration-tests/gitclient/gitclient_main_test.go ./integration-tests/gitclient/gitclient_dnsupdate_test.go +// setup environment for this test. create a branch with test files already in the branch +func TestGit_DNS_Setup(t *testing.T) { + branch := dnsTestBranch + // create the git client + workingDir, err := utils.CreateTempDir("dns-test-", utils.TempRoot) + if err != nil { + t.Errorf("%v", err) + return + } + // create client using OS FS + client, err := git.NewClientOSFS( + gitclientTestConfig.GitlabIntTestUsername, + gitclientTestConfig.GitlabIntTestToken, + gitclientTestConfig.GitlabIntTestRepositoryURL, + workingDir, + ) + if err != nil { + t.Errorf("%v", err) + return + } + t.Logf("Created git client for repo %s", gitclientTestConfig.GitlabIntTestRepositoryURL) + + // SETUP initial STATE - checkout and delete the branch + err = client.CheckoutOrCreateBranch(branch) + if err != nil { + t.Errorf("%v", err) + return + } + t.Logf("Checked out branch %s", branch) + + err = client.DeleteRemoteBranch(branch) + if err != nil { + t.Errorf("Error %v", err) + return + } + t.Logf("Deleted remote branch %s", branch) + + // Create a NEW fresh branch + err = client.CheckoutOrCreateBranch(branch) + if err != nil { + t.Errorf("Error %v", err) + return + } + t.Logf("Checked out and created a NEW branch %s", branch) + + // ADD test files to new branch + t.Logf("git working dir %s", gitclientTestConfig.WorkingDir) + + dataFileName := "dso-dns-staging.tar.gz" + dataFile := gitclientTestConfig.WorkingDir + utils.PathSeparator + "data" + utils.PathSeparator + dataFileName + // create a tempdir and decompress dataFile into tempDir + tempDataDir, err := utils.CreateTempDirWithFile(dataFile, "prefix-", true) + if err != nil { + t.Errorf("Error %v", err) + return + } + err = utils.CopyDir(tempDataDir, workingDir) + if err != nil { + t.Errorf("Error %v", err) + return + } + + // Add everything in the workingDir to the git index + err = client.AddFile(".") + if err != nil { + t.Errorf("Error %v", err) + return + } + err = client.Commit(fmt.Sprintf("Add integration test data %s", utils.DateTimeUTC())) + if err != nil { + t.Errorf("Error %v", err) + return + } + err = client.Push(branch) + if err != nil { + t.Errorf("Error %v", err) + return + } + + // cleanup + utils.DeleteDir(tempDataDir) +} +func TestGit_Checkout_UpdateDNSFile_Push(t *testing.T) { + branch := dnsTestBranch + + // create the git client with in memory FS + client, err := git.NewClient( + gitclientTestConfig.GitlabIntTestUsername, + gitclientTestConfig.GitlabIntTestToken, + gitclientTestConfig.GitlabIntTestRepositoryURL, + ) + if err != nil { + t.Errorf("%v", err) + return + } + t.Logf("Created git client for repo %s", gitclientTestConfig.GitlabIntTestRepositoryURL) + + // Create a branch + err = client.CheckoutOrCreateBranch(branch) + if err != nil { + t.Errorf("Error %v", err) + return + } + t.Logf("Checked out and Create branch %s", branch) + + // process dns file + fs := *client.GetFileSystem() + + dnsFilePath := "dso-dns-staging/staging/dns/configuration/db.staging.dso.mil" + dnsFile, err := fs.Open(dnsFilePath) + if err != nil { + t.Errorf("%v", err) + return + } + + tempFile, err := fs.Create("temp_file") + if err != nil { + t.Errorf("%v", err) + return + } + + scanner := bufio.NewScanner(dnsFile) + var done bool + for scanner.Scan() { + line := scanner.Text() + count, err := tempFile.Write(scanner.Bytes()) + tempFile.Write([]byte("\n")) + if err != nil { + t.Errorf("%v", err) + } + t.Logf("written %d bytes", count) + + if !done && strings.Contains(line, "-il2 ") { + tempFile.Write([]byte("int-test-a-record\n")) + done = true + } + } + + if err := scanner.Err(); err != nil { + t.Errorf("%v", err) + } + + tempFile.Close() + dnsFile.Close() + + err = fs.Rename("temp_file", dnsFilePath) + if err != nil { + t.Errorf("%v", err) + } + + tempFile, err = fs.Open(dnsFilePath) + if err != nil { + t.Errorf("%v", err) + } + + scanner2 := bufio.NewScanner(tempFile) + for scanner2.Scan() { + line := scanner2.Text() + t.Logf(line) + } + + tempFile.Close() + + err = client.AddFile(dnsFilePath) + if err != nil { + fmt.Printf("%v", err) + return + } + + err = client.Commit("Add new entry for new site - Valkyrie") + if err != nil { + fmt.Printf("%v", err) + return + } + + err = client.Push(branch) + if err != nil { + return + } +} diff --git a/integration-tests/gitclient/gitclient_main_test.go b/integration-tests/gitclient/gitclient_main_test.go index d10e1f3945fdf8df8089bc32b1bd7925e40d0393..9192ebc18b81ede6e575030b70a8684446e0916d 100644 --- a/integration-tests/gitclient/gitclient_main_test.go +++ b/integration-tests/gitclient/gitclient_main_test.go @@ -5,16 +5,31 @@ package gitclient import ( "errors" "fmt" + "net/http" + "net/url" "os" + "strings" "testing" + + "github.com/romana/rlog" + gogitlab "github.com/xanzy/go-gitlab" "valkyrie.dso.mil/valkyrie-api/clients/git" + "valkyrie.dso.mil/valkyrie-api/clients/gitlab" ) type gitclientTestConfigType struct { WorkingDir string GitlabIntTestUsername string GitlabIntTestToken string + GitlabAPIToken string + GitlabAPIURL string + GitlabClient gitlab.ClientImpl + GitlabIntTestProjectPath string + GitlabIntTestProjectURL string GitlabIntTestRepositoryURL string + GitlabIntTestRepositoryID int + GitlabIntTestRootGroupPath string + GitlabIntTestRootGroupID int } var gitclientTestConfig gitclientTestConfigType @@ -39,25 +54,79 @@ func TestMain(m *testing.M) { // packageSetup - setup environment for integration testing func packageSetup() error { - gitclientTestConfig.WorkingDir, _ = os.Getwd() var ok bool - gitclientTestConfig.GitlabIntTestUsername, ok = os.LookupEnv("GITLAB_INT_TEST_USERNAME") + var err error + + configObject := &gitclientTestConfig + configObject.WorkingDir, _ = os.Getwd() + + configObject.GitlabAPIURL, ok = os.LookupEnv("GITLAB_API_URL") if !ok { - err := errors.New("env variable GITLAB_INT_TEST_USERNAME undefined") + err = errors.New("env variable GITLAB_API_URL undefined") return err } - gitclientTestConfig.GitlabIntTestToken, ok = os.LookupEnv("GITLAB_INT_TEST_TOKEN") + configObject.GitlabAPIToken, ok = os.LookupEnv("GITLAB_API_TOKEN") if !ok { - err := errors.New("env variable GITLAB_INT_TEST_TOKEN undefined") + err := errors.New("env variable GITLAB_API_TOKEN undefined") + return err + } + + configObject.GitlabIntTestProjectURL, ok = os.LookupEnv("GITLAB_INT_TEST_PROJECT_URL") + if !ok { + err := errors.New("env variable GITLAB_INT_TEST_PROJECT_URL undefined") return err } - gitclientTestConfig.GitlabIntTestRepositoryURL, ok = os.LookupEnv("GITLAB_INT_TEST_REPOSITORY_URL") + configObject.GitlabIntTestProjectPath, ok = os.LookupEnv("GITLAB_INT_TEST_PROJECT_PATH") + if !ok { + err := errors.New("env variable GITLAB_INT_TEST_PROJECT_PATH undefined") + return err + } + + configObject.GitlabIntTestRepositoryURL, ok = os.LookupEnv("GITLAB_INT_TEST_REPOSITORY_URL") if !ok { err := errors.New("env variable GITLAB_INT_TEST_REPOSITORY_URL undefined") return err } + + configObject.GitlabIntTestRootGroupPath, ok = os.LookupEnv("GITLAB_INT_TEST_ROOT_GROUP_PATH") + if !ok { + err := errors.New("env variable GITLAB_INT_TEST_ROOT_GROUP_PATH undefined") + return err + } + + configObject.GitlabIntTestUsername, ok = os.LookupEnv("GITLAB_INT_TEST_USERNAME") + if !ok { + err := errors.New("env variable GITLAB_INT_TEST_USERNAME undefined") + return err + } + + configObject.GitlabIntTestToken, ok = os.LookupEnv("GITLAB_INT_TEST_TOKEN") + if !ok { + err := errors.New("env variable GITLAB_INT_TEST_TOKEN undefined") + return err + } + + configObject.GitlabClient, err = gitlab.NewClient(configObject.GitlabAPIURL, configObject.GitlabAPIToken, nil) + if err != nil { + return err + } + + rlog.Debugf("configObject %v", configObject) + + // insure a root group exists in gitlab + err = addRootGroup() + if err != nil { + return err + } + + // insure a test project exists in root group + configObject.GitlabIntTestRepositoryID, err = addIntTestProject(configObject.GitlabIntTestProjectURL) + if err != nil { + return err + } + return nil } @@ -74,3 +143,86 @@ func logGitStatus(t *testing.T, client *git.RepoClientImpl) { t.Logf("working tree status: %s", status) } + +// Insure the integration test root level group exists +func addRootGroup() error { + configObject := &gitclientTestConfig + c := configObject.GitlabClient + + path := configObject.GitlabIntTestRootGroupPath + name := configObject.GitlabIntTestRootGroupPath + description := "Integration Test Root Level Group" + + group, statusCode, err := c.GetGroupByFullPath(&path) + if err != nil { + // dont fail if group already exists + fmt.Printf("AddRootGroup GetGroupByFullPath error = %v %d\n", err, statusCode) + return err + } + + if statusCode == http.StatusFound { + // root group found + configObject.GitlabIntTestRootGroupID = group.ID + fmt.Printf("AddRootGroup Group Found: %s %s %s %s %d \n", group.Name, group.Path, group.FullPath, group.WebURL, statusCode) + } else { + // try to create root group + groupOpts := gogitlab.CreateGroupOptions{Name: &name, Path: &path, Description: &description} + got, statusCode, err := c.AddGroup(groupOpts) + + if err != nil { + // dont fail if group already exists + fmt.Printf("Error! AddRootGroup %s error = %v %d\n", path, err, statusCode) + return err + } + configObject.GitlabIntTestRootGroupID = got.ID + fmt.Printf("AddGroup %s %s %s %s %d \n", got.Name, got.Path, got.FullPath, got.WebURL, statusCode) + + } + return nil +} + +// Insure the integration test project exists +func addIntTestProject(projectURL string) (int, error) { + configObject := &gitclientTestConfig + var project *gogitlab.Project = nil + var responseCode int + + url, err := url.Parse(projectURL) + if err != nil { + return 0, err + } + + var urlPath = strings.TrimLeft(url.Path, "/") + project, responseCode, err = configObject.GitlabClient.GetProjectByFullPath(&urlPath) + if err != nil { + return 0, err + } + + if responseCode == http.StatusFound { + return project.ID, nil + } + path := configObject.GitlabIntTestProjectPath + name := configObject.GitlabIntTestProjectPath + description := "Integration Test Project" + configPath := "a/b/config.yml@group1/group2/project" + initializeWithReadme := true + + projectOpts := gogitlab.CreateProjectOptions{ + Name: &name, + Path: &path, + Description: &description, + NamespaceID: &configObject.GitlabIntTestRootGroupID, + CIConfigPath: &configPath, + InitializeWithReadme: &initializeWithReadme, + } + got, statusCode, err := configObject.GitlabClient.AddProject(projectOpts) + + if err != nil { + fmt.Printf("ClientImpl.AddProject() error = %v %d \n", err, statusCode) + return 0, err + } + + fmt.Printf("AddProject %s %s %s %s %d", got.Name, got.Path, got.PathWithNamespace, got.WebURL, statusCode) + return got.ID, nil + +} diff --git a/integration-tests/gitlab/api/gitlab_api_mr_test.go b/integration-tests/gitlab/api/gitlab_api_mr_test.go index 88dfec576ed7b3af31d99939d85c455f3f954eef..f1d10e8a4de2d65b40f8f017ff9e2200620c36d2 100644 --- a/integration-tests/gitlab/api/gitlab_api_mr_test.go +++ b/integration-tests/gitlab/api/gitlab_api_mr_test.go @@ -35,8 +35,8 @@ func TestCreateMR(t *testing.T) { title := "Valkyrie Test" description := "Created by Valkyrie" - sourceBranch := "josh-branch" - targetBranch := "master" + sourceBranch := sourceBranch + targetBranch := targetBranch removeSourceBranch := true squash := true got, err := c.CreateMergeRequest(gitlabAPITestConfig.GitlabIntTestRepositoryID, &gitlab2.CreateMergeRequestOptions{ diff --git a/integration-tests/gitlab/api/gitlab_main_test.go b/integration-tests/gitlab/api/gitlab_main_test.go index 05d1e221dd4bc28ec7da3033ad628211ada3acc4..30c9b15049359b9a166fe43451b95fe1e8d6a518 100644 --- a/integration-tests/gitlab/api/gitlab_main_test.go +++ b/integration-tests/gitlab/api/gitlab_main_test.go @@ -54,59 +54,62 @@ func TestMain(m *testing.M) { // packageSetup - setup environment for integration testing func packageSetup() error { - gitlabAPITestConfig.WorkingDir, _ = os.Getwd() var ok bool var err error - gitlabAPITestConfig.GitlabAPIURL, ok = os.LookupEnv("GITLAB_API_URL") + configObject := &gitlabAPITestConfig + configObject.WorkingDir, _ = os.Getwd() + + + configObject.GitlabAPIURL, ok = os.LookupEnv("GITLAB_API_URL") if !ok { err = errors.New("env variable GITLAB_API_URL undefined") return err } - gitlabAPITestConfig.GitlabAPIToken, ok = os.LookupEnv("GITLAB_API_TOKEN") + configObject.GitlabAPIToken, ok = os.LookupEnv("GITLAB_API_TOKEN") if !ok { err := errors.New("env variable GITLAB_API_TOKEN undefined") return err } - gitlabAPITestConfig.GitlabIntTestProjectURL, ok = os.LookupEnv("GITLAB_INT_TEST_PROJECT_URL") + configObject.GitlabIntTestProjectURL, ok = os.LookupEnv("GITLAB_INT_TEST_PROJECT_URL") if !ok { err := errors.New("env variable GITLAB_INT_TEST_PROJECT_URL undefined") return err } - gitlabAPITestConfig.GitlabIntTestProjectPath, ok = os.LookupEnv("GITLAB_INT_TEST_PROJECT_PATH") + configObject.GitlabIntTestProjectPath, ok = os.LookupEnv("GITLAB_INT_TEST_PROJECT_PATH") if !ok { err := errors.New("env variable GITLAB_INT_TEST_PROJECT_PATH undefined") return err } - gitlabAPITestConfig.GitlabIntTestRepositoryURL, ok = os.LookupEnv("GITLAB_INT_TEST_REPOSITORY_URL") + configObject.GitlabIntTestRepositoryURL, ok = os.LookupEnv("GITLAB_INT_TEST_REPOSITORY_URL") if !ok { err := errors.New("env variable GITLAB_INT_TEST_REPOSITORY_URL undefined") return err } - gitlabAPITestConfig.GitlabIntTestRootGroupPath, ok = os.LookupEnv("GITLAB_INT_TEST_ROOT_GROUP_PATH") + configObject.GitlabIntTestRootGroupPath, ok = os.LookupEnv("GITLAB_INT_TEST_ROOT_GROUP_PATH") if !ok { err := errors.New("env variable GITLAB_INT_TEST_ROOT_GROUP_PATH undefined") return err } - gitlabAPITestConfig.GitlabIntTestUsername, ok = os.LookupEnv("GITLAB_INT_TEST_USERNAME") + configObject.GitlabIntTestUsername, ok = os.LookupEnv("GITLAB_INT_TEST_USERNAME") if !ok { err := errors.New("env variable GITLAB_INT_TEST_USERNAME undefined") return err } - gitlabAPITestConfig.GitlabIntTestToken, ok = os.LookupEnv("GITLAB_INT_TEST_TOKEN") + configObject.GitlabIntTestToken, ok = os.LookupEnv("GITLAB_INT_TEST_TOKEN") if !ok { err := errors.New("env variable GITLAB_INT_TEST_TOKEN undefined") return err } - gitlabAPITestConfig.GitlabClient, err = gitlab.NewClient(gitlabAPITestConfig.GitlabAPIURL, gitlabAPITestConfig.GitlabAPIToken, nil) + configObject.GitlabClient, err = gitlab.NewClient(gitlabAPITestConfig.GitlabAPIURL, gitlabAPITestConfig.GitlabAPIToken, nil) if err != nil { return err } @@ -118,7 +121,7 @@ func packageSetup() error { } // insure a test project exists in root group - gitlabAPITestConfig.GitlabIntTestRepositoryID, err = addIntTestProject(gitlabAPITestConfig.GitlabIntTestProjectURL) + configObject.GitlabIntTestRepositoryID, err = addIntTestProject(gitlabAPITestConfig.GitlabIntTestProjectURL) if err != nil { return err }