UNCLASSIFIED

Commit 9a2d3ec3 authored by Jason van Brackel's avatar Jason van Brackel
Browse files

Merge remote-tracking branch 'origin/master' into dev-jvb-629

parents 933b7370 599d0558
# NOTE: USING THIS IMAGE UNTIL registry1 credentials are added to IL2 runners
# Build the manager binary
FROM registry.il2.dso.mil/platform-one/devops/pipeline-templates/pipeline-job/golang-builder-1.6:1.0 as builder
FROM registry.il2.dso.mil/platform-one/devops/pipeline-templates/valkyrie/golang-builder-1.6:1.0 as builder
WORKDIR /workspace
# Copy the Go Modules manifests
......
## <img src="valkyrie.png" width="60"> Valkyrie Automated CICD Provisioning Tool
## Bump
### Golang Frameworks
- kubebuilder
- https://book.kubebuilder.io/introduction.html
......
......@@ -18,7 +18,6 @@ package v1alpha1
import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"valkyrie.dso.mil/valkyrie-api/apis/gitlab/v1alpha1"
)
// CustomerSpec defines the desired state of Customer
......@@ -26,9 +25,6 @@ type CustomerSpec struct {
// Organization is the organizational information for this customer based upon the organization record in AutoBot
Organization OrganizationSpec `json:"Organization,omitempty"`
// Group is the Gitlab Group that belongs to this Customer
Group v1alpha1.GroupSpec `json:"Group,omitempty"`
// test dummy - do not remove
Dummy string `json:"dummy,omitempty"`
}
......
......@@ -207,7 +207,7 @@ func (in *Customer) DeepCopyInto(out *Customer) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
in.Spec.DeepCopyInto(&out.Spec)
out.Spec = in.Spec
out.Status = in.Status
}
......@@ -265,7 +265,6 @@ func (in *CustomerList) DeepCopyObject() runtime.Object {
func (in *CustomerSpec) DeepCopyInto(out *CustomerSpec) {
*out = *in
out.Organization = in.Organization
in.Group.DeepCopyInto(&out.Group)
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CustomerSpec.
......
......@@ -23,7 +23,7 @@ import (
// GroupSpec defines the desired state of Group
type GroupSpec struct {
// FullPath is the gitlab path for this Project
// +kubebuilder:validation:Required
// +kubebuilder:validation:required
FullPath string `json:"path"`
// Name is the name of the Group and will be used as part of the URL in Gitlab
......@@ -31,21 +31,17 @@ type GroupSpec struct {
Name string `json:"name"`
// Description is the Gitlab Description for the group
// +kubebuilder:validation:Optional
// +kubebuilder:validation:optional
Description string `json:"description"`
// GitlabCredentialsName is the name of the object in this namespace that contains authentication
// information for logging into the
GitlabCredentialsName string `json:"gitlabCredentialsName"`
// GroupSpecs are for the GitLab Subgroups managed by this Group.
// +kubebuilder:validation:Optional
// SubGroups []GroupSpec `json:"subGroups:omitempty"`
// ProjectSpecs are for the GitLab projects managed by this Group. It's expected that Projects
// will be managed at the group level rather than be adjusted themselves.
// +kubebuilder:validation:Optional
ProjectSpecs []ProjectSpec `json:"projects:omitempty"`
// +kubebuilder:validation:Required
ProjectSpecs []ProjectSpec `json:"projects,omitempty"`
}
// GroupStatus defines the observed state of Group
......
......@@ -31,17 +31,17 @@ type ProjectSpec struct {
Name string `json:"name"`
// FullPath is the gitlab path for this Project
// +kubebuilder:validation:Required
// +kubebuilder:validation:required
FullPath string `json:"path"`
// ImpactLevel is the RMF Impact Level for this Project
// +kubebuilder:validation:Enum=2;4;5;6
// +kubebuilder:validation:Required
// +kubebuilder:validation:required
ImpactLevel string `json:"impactLevel"`
// StorageTypes is the storage types that will be added to the Project
// +kubebuilder:validation:Optional
StorageTypes []string `json:"storageTypes:omitempty"`
// +kubebuilder:validation:optional
StorageTypes []string `json:"storageTypes,omitempty"`
// VirtualService is the type of virtual service that will be created for this Project
// +kubebuilder:default="Default"
......@@ -51,7 +51,7 @@ type ProjectSpec struct {
ServicePort int32 `json:"servicePort,omitempty"`
// Language is the programming language of the Project for this application.
// +kubebuilder:validation:Required
// +kubebuilder:validation:required
Language string `json:"language"` //TODO: Consider a custom type / annotation
}
......
......@@ -45,6 +45,7 @@ import (
const itemsPerPage = 50
// Client -
type Client interface {
GetUser(userID int) (*gogitlab.User, int, error)
GetUsers(search *string) ([]*gogitlab.User, error)
......@@ -289,7 +290,7 @@ func (r ClientImpl) DeleteUser(userID int, waitInterval int, waitCount int) (int
// DeleteUserByUsername - convenience method
func (r ClientImpl) DeleteUserByUsername(username string, waitInterval int, waitCount int) (int, error) {
// waiting will be skilled if waitCount is 0
// waiting will be skipped if waitCount is 0
// setting wait will use a loop to wait until resource has completed deletion
logPrefix := "DeleteUserByUsername"
......@@ -308,6 +309,7 @@ func (r ClientImpl) DeleteUserByUsername(username string, waitInterval int, wait
return statusCode, nil
}
// the user return array should never have more than one entry
statusCode, err := r.DeleteUser(users[0].ID, 0, 0)
if err != nil {
......@@ -497,7 +499,7 @@ func (r ClientImpl) UpdateGroup(groupID int, updateGroupOptions *gogitlab.Update
// DeleteGroup -
func (r ClientImpl) DeleteGroup(groupID int, waitInterval int, waitCount int) (int, error) {
// waiting will be skilled if waitCount is 0
// waiting will be skipped if waitCount is 0
// setting wait will use a loop to wait until resource has completed deletion
logPrefix := "DeleteGroup"
......
......@@ -534,7 +534,7 @@ func TestClient_DeleteUserByUsername(t *testing.T) {
httpmock.RegisterResponder("DELETE",
`=~^https://test/api/v4/users.*`,
func(req *http.Request) (*http.Response, error) {
return httpmock.NewJsonResponse(201, testUser)
return httpmock.NewJsonResponse(http.StatusAccepted, testUser)
},
)
......@@ -564,7 +564,7 @@ func TestClient_DeleteUserByUsername(t *testing.T) {
name: "DeleteUserByUsername Success",
fields: fields{client: testGitlabClient, token: testToken, apiURL: testAPIUrl},
args: args{username: testUsername},
want: http.StatusOK,
want: http.StatusAccepted,
wantErr: false,
},
{
......@@ -1121,7 +1121,7 @@ func TestClient_DeleteGroup(t *testing.T) {
return
}
if got != tt.want {
t.Errorf("ClientImpl.ç() = %v, want %v", got, tt.want)
t.Errorf("ClientImpl.DeleteGroup() = %v, want %v", got, tt.want)
return
}
t.Logf("ClientImpl.DeleteGroup() statusCode = %d ", got)
......
......@@ -36,84 +36,6 @@ spec:
spec:
description: CustomerSpec defines the desired state of Customer
properties:
Group:
description: Group is the Gitlab Group that belongs to this Customer
properties:
description:
description: Description is the Gitlab Description for the group
type: string
gitlabCredentialsName:
description: GitlabCredentialsName is the name of the object in
this namespace that contains authentication information for
logging into the
type: string
name:
description: Name is the name of the Group and will be used as
part of the URL in Gitlab
type: string
path:
description: FullPath is the gitlab path for this Project
type: string
projects:omitempty:
description: ProjectSpecs are for the GitLab projects managed
by this Group. It's expected that Projects will be managed
at the group level rather than be adjusted themselves.
items:
description: ProjectSpec defines the desired state of Project
properties:
groupId:
description: GroupID is the id of the GitLab Group id that
owns this project
type: integer
impactLevel:
description: ImpactLevel is the RMF Impact Level for this
Project
enum:
- 2
- 4
- 5
- 6
type: string
language:
description: Language is the programming language of the
Project for this application.
type: string
name:
description: Name is the name of this Project
type: string
path:
description: FullPath is the gitlab path for this Project
type: string
servicePort:
description: ServicePort is the port for the virtual service
for the application served by this Project
format: int32
type: integer
storageTypes:omitempty:
description: StorageTypes is the storage types that will
be added to the Project
items:
type: string
type: array
virtualService:
default: Default
description: VirtualService is the type of virtual service
that will be created for this Project
type: string
required:
- groupId
- impactLevel
- language
- name
- path
- virtualService
type: object
type: array
required:
- gitlabCredentialsName
- name
- path
type: object
Organization:
description: Organization is the organizational information for this
customer based upon the organization record in AutoBot
......
......@@ -51,7 +51,7 @@ spec:
path:
description: FullPath is the gitlab path for this Project
type: string
projects:omitempty:
projects:
description: ProjectSpecs are for the GitLab projects managed by this
Group. It's expected that Projects will be managed at the group
level rather than be adjusted themselves.
......@@ -85,7 +85,7 @@ spec:
for the application served by this Project
format: int32
type: integer
storageTypes:omitempty:
storageTypes:
description: StorageTypes is the storage types that will be
added to the Project
items:
......@@ -106,6 +106,7 @@ spec:
type: object
type: array
required:
- description
- gitlabCredentialsName
- name
- path
......
......@@ -63,7 +63,7 @@ spec:
application served by this Project
format: int32
type: integer
storageTypes:omitempty:
storageTypes:
description: StorageTypes is the storage types that will be added
to the Project
items:
......
......@@ -28,30 +28,34 @@ func CreateProject(projectCredentials ProjectCredentials, projectSpec apigitlab.
logPrefix := "CreateProject"
logStart(logPrefix, projectSpec)
client, _ := gitlab.NewClient(projectCredentials.ServerURL, projectCredentials.ServerToken, httpClient)
// confirm parent group by ID
group, statusCode, err := client.GetGroup(int(projectSpec.GroupID))
client, err := gitlab.NewClient(projectCredentials.ServerURL, projectCredentials.ServerToken, httpClient)
if err != nil {
return fmt.Errorf(fmt.Sprintf("Failed to find group with ID %d. error: %v", int(projectSpec.GroupID), err))
}
if statusCode != http.StatusOK {
return fmt.Errorf(fmt.Sprintf("Failed to find group with ID %d. status code: %d", int(projectSpec.GroupID), statusCode))
processError(logPrefix, err)
return err
}
// confirm parent group full path
parentGroupFullPath := group.FullPath
parentGroup, statusCode, err := client.GetGroupByFullPath(&parentGroupFullPath)
// Identify parent Group for project
projectFullPath := projectSpec.FullPath
groupFullPath, projectPath := ParseGitlabPath(projectFullPath)
// strip any trailing /
groupFullPath = StripLastChar(groupFullPath, "/")
parentGroup, statusCode, err := client.GetGroupByFullPath(&groupFullPath)
if err != nil {
return fmt.Errorf(fmt.Sprintf("Failed to find group with ID %d. error: %v", int(projectSpec.GroupID), err))
return fmt.Errorf("failed to find group with fullPath %s. error: %v", groupFullPath, err)
}
if statusCode != http.StatusFound {
return fmt.Errorf(fmt.Sprintf("Failed to find group with ID %d. status code: %d", int(projectSpec.GroupID), statusCode))
return fmt.Errorf("failed to find group with fullPath %s. status code: %d", groupFullPath, statusCode)
}
ciConfigPath, err := generateCIConfigPath(projectPath, groupFullPath)
if err != nil {
return fmt.Errorf("failed to generateCIConfigPath: %v", err)
}
ciConfigPath := generateCIConfigPath(projectSpec.Name, parentGroupFullPath)
projectOptions := gogitlab.CreateProjectOptions{
Name: &projectSpec.Name,
Path: &projectPath,
NamespaceID: &parentGroup.ID,
CIConfigPath: &ciConfigPath,
}
......@@ -62,10 +66,10 @@ func CreateProject(projectCredentials ProjectCredentials, projectSpec apigitlab.
project, statusCode, err := client.AddProject(projectOptions)
if err != nil {
return fmt.Errorf(fmt.Sprintf("Failed to add project with name %s to group path %s. error: %v", *projectOptions.Name, parentGroupFullPath, err))
return fmt.Errorf("failed to add project with name %s to group path %s. error: %v", *projectOptions.Name, groupFullPath, err)
}
if statusCode != http.StatusCreated {
return fmt.Errorf(fmt.Sprintf("Failed to add project with name %s to group path %s. statusCode: %d", *projectOptions.Name, parentGroupFullPath, statusCode))
return fmt.Errorf(fmt.Sprintf("Failed to add project with name %s to group path %s. statusCode: %d", *projectOptions.Name, groupFullPath, statusCode))
}
rlog.Debugf("Added project %s to path %s with status code %d", project.Name, project.NameWithNamespace, statusCode)
......@@ -78,28 +82,28 @@ func DeleteProject(projectCredentials ProjectCredentials, projectSpec apigitlab.
logPrefix := "DeleteProject"
logStart(logPrefix, projectSpec)
client, _ := gitlab.NewClient(projectCredentials.ServerURL, projectCredentials.ServerToken, httpClient)
group, _, err := client.GetGroup(int(projectSpec.GroupID))
client, err := gitlab.NewClient(projectCredentials.ServerURL, projectCredentials.ServerToken, httpClient)
if err != nil {
return fmt.Errorf(fmt.Sprintf("Faild to find group with ID %d", int(projectSpec.GroupID)))
processError(logPrefix, err)
return err
}
parentGroupFullPath := group.FullPath
// get scrubbed pathname based on actual name
projectPathString, _ := GenerateGitlabPath(projectSpec.Name)
projectFullPath := parentGroupFullPath + "/" + projectPathString
projectFullPath := projectSpec.FullPath
project, statusCode, err := client.GetProjectByFullPath(&projectFullPath)
if err != nil {
processError(logPrefix, err)
return err
}
if statusCode != http.StatusFound {
return fmt.Errorf(fmt.Sprintf("Failed to find project with full path %s", projectFullPath))
return fmt.Errorf("failed to find project with fullPath %s", projectFullPath)
}
_, err = client.DeleteProject(project.ID, 0, 0)
// delete and wait for delete to complete up to 500ms * 240 tries
_, err = client.DeleteProject(project.ID, 500, 240)
if err != nil {
return fmt.Errorf(fmt.Sprintf("Faild to delete project with ID %d", int(project.ID)))
return fmt.Errorf("failed to delete project with fullPath: %s", projectFullPath)
}
return nil
}
......@@ -109,28 +113,40 @@ func UpdateProject(projectCredentials ProjectCredentials, projectSpec apigitlab.
logPrefix := "UpdateProject"
logStart(logPrefix, projectSpec)
client, _ := gitlab.NewClient(projectCredentials.ServerURL, projectCredentials.ServerToken, httpClient)
client, err := gitlab.NewClient(projectCredentials.ServerURL, projectCredentials.ServerToken, httpClient)
if err != nil {
processError(logPrefix, err)
return err
}
// Identify parent Group for project
group, _, err := client.GetGroup(int(projectSpec.GroupID))
projectFullPath := projectSpec.FullPath
groupFullPath, projectPath := ParseGitlabPath(projectSpec.FullPath)
// strip any trailing /
groupFullPath = StripLastChar(groupFullPath, "/")
group, _, err := client.GetGroupByFullPath(&groupFullPath)
if err != nil {
processError(logPrefix, err)
return fmt.Errorf(fmt.Sprintf("Faild to find group with ID %d", int(projectSpec.GroupID)))
return fmt.Errorf("failed to find group with fullPath %s", groupFullPath)
}
parentGroupFullPath := group.FullPath
// get scrubbed pathname based on actual name
projectPathString, _ := GenerateGitlabPath(projectSpec.Name)
projectFullPath := parentGroupFullPath + "/" + projectPathString
project, statusCode, err := client.GetProjectByFullPath(&projectFullPath)
if err != nil {
processError(logPrefix, err)
return err
}
if statusCode != http.StatusFound {
return fmt.Errorf(fmt.Sprintf("Failed to find project with full path %s", projectFullPath))
return fmt.Errorf("failed to find project with fullPath %s", projectFullPath)
}
ciConfigPath, err := generateCIConfigPath(projectPath, parentGroupFullPath)
if err != nil {
processError(logPrefix, err)
return err
}
ciConfigPath := generateCIConfigPath(projectSpec.Name, parentGroupFullPath)
projectOptions := gogitlab.EditProjectOptions{
Name: &projectSpec.Name,
CIConfigPath: &ciConfigPath,
......@@ -146,8 +162,13 @@ func UpdateProject(projectCredentials ProjectCredentials, projectSpec apigitlab.
return nil
}
func generateCIConfigPath(projectName string, parentGroupFullPath string) string {
projectPathString, _ := GenerateGitlabPath(projectName)
// generateCIConfigPath - helper function. create CI configuration path for P1
func generateCIConfigPath(projectName string, parentGroupFullPath string) (string, error) {
// scrub the project name
projectPathString, err := GenerateGitlabPath(projectName)
if err != nil {
return "", fmt.Errorf("failed to GenerateGitlabPath with name %s error: %v", projectName, err)
}
ciConfigPath := fmt.Sprintf("%s/%s-ci.yml@platform-one/devops/pipeline-products", parentGroupFullPath, projectPathString)
return ciConfigPath
return ciConfigPath, nil
}
......@@ -21,6 +21,12 @@ func TestGetProjectLanguage(t *testing.T) {
want: LangTypes[LangTypeCpp],
wantErr: false,
},
{
name: "project language test",
args: args{langTypeName: "BAD"},
want: LangType{},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
......
package p1
import (
"path"
"regexp"
"strings"
)
......@@ -26,3 +27,19 @@ func GenerateGitlabPath(name string) (string, error) {
return pathLower, nil
}
// ParseGitlabPath - extract the basename from a path
func ParseGitlabPath(fullpath string) (parent string, name string) {
parent, name = path.Split(fullpath)
return parent, name
}
// StripLastChar -
func StripLastChar(value string, char string) string {
var re *regexp.Regexp
// remove any trailing dash or whitespace
re, _ = regexp.Compile(`[` + char + `]{1}$`)
newValue := re.ReplaceAllString(value, "")
return newValue
}
package p1
import "testing"
import (
"testing"
)
func TestGenerateGitlabPath(t *testing.T) {
type args struct {
......@@ -34,3 +36,35 @@ func TestGenerateGitlabPath(t *testing.T) {
})
}
}
func TestParseGitlabPath(t *testing.T) {
type args struct {
fullpath string
}
tests := []struct {
name string
args args
wantParent string
wantName string
}{
{name: "test 1", args: args{fullpath: ""}, wantParent: "", wantName: ""},
{name: "test 1", args: args{fullpath: "test-path"}, wantParent: "", wantName: "test-path"},
{name: "test 1", args: args{fullpath: "parent-path/base-name"}, wantParent: "parent-path/", wantName: "base-name"},
{name: "test 1", args: args{fullpath: "/parent-path/sub-group-path/base-name"}, wantParent: "/parent-path/sub-group-path/", wantName: "base-name"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotParent, gotName := ParseGitlabPath(tt.args.fullpath)
if gotParent != tt.wantParent {
t.Errorf("ParseGitlabPath() gotParent = %v, want %v", gotParent, tt.wantParent)
return
}
if gotName != tt.wantName {
t.Errorf("ParseGitlabPath() gotName = %v, want %v", gotName, tt.wantName)
return
}
t.Logf("TestParseGitlabPath() PASS input '%s', want '%s' '%s'", tt.args.fullpath, gotParent, gotName)
})
}
}
// +build integration
package integration
......@@ -15,8 +14,10 @@ import (
gitlab "valkyrie.dso.mil/valkyrie-api/clients/gitlab"
)
// pathname for root group used in integration testing
const p1IntegrationRootGroupPath = "integration-root-group"
// configuration obect to be shared for integration testing
var P1Config struct {
gitlabAPIURL string
gitlabAPIToken string
......@@ -26,24 +27,35 @@ var P1Config struct {
// TestMain - setup and teardown reusable code. wraps around other tests
func TestMain(m *testing.M) {
// setup environment for testing
err := packageSetup()
if err != nil {
fmt.Printf("packageSetup error: %v", err)
os.Exit(1)
}
// run tests
resultCode := m.Run()
packageTeardown()
os.Exit(resultCode)
}
// packageSetup - setup environment for integration testing
func packageSetup() error {
var err error
// initialize
P1Config.testProjectGroupPath = p1IntegrationRootGroupPath
// read env variables
err = getEnvVariables()
if err != nil {
return err
}
// add gitlab root group if does not exist
err = addRootGroup()
if err != nil {
return err
......@@ -66,8 +78,11 @@ func getEnvVariables() error {
if !ok {
return errors.New("env variable GITLAB_API_TOKEN undefinded")
}
// set values in package scoped config object
P1Config.gitlabAPIToken = gitlabAPIToken
P1Config.gitlabAPIURL = gitlabAPIURL
return nil
}
......@@ -107,16 +122,20 @@ func addRootGroup() error {
return nil
}
// getClientGroups -
func getClient() (gitlab.Client, error) {
// getClient - create a gitlab client object
func getClient() (gitlab.ClientImpl, error) {
gitlabAPIURL, ok := os.LookupEnv("GITLAB_API_URL")
if !ok {
return gitlab.Client{}, errors.New("env variable GITLAB_API_URL undefinded")
return gitlab.ClientImpl{}, errors.New("env variable GITLAB_API_URL undefinded")
}
gitlabAPIToken, ok := os.LookupEnv("GITLAB_API_TOKEN")
if !ok {
return gitlab.Client{}, errors.New("env variable GITLAB_API_TOKEN undefinded")
return gitlab.ClientImpl{}, errors.New("env variable GITLAB_API_TOKEN undefinded")
}
client, err := gitlab.NewClient(gitlabAPIURL, gitlabAPIToken, nil)
if err != nil {
return gitlab.ClientImpl{}, err
}
var client, _ = gitlab.NewClient(gitlabAPIURL, gitlabAPIToken, nil)
return client, nil
}
......@@ -14,13 +14,14 @@ import (
P1Config is initialized in the TestMain wrapper method
*/
const integrationTestProjectPath = "int-test-project"
func Test_P1_AddProject(t *testing.T) {
logPrefix := "Test_P1_AddProject"
t.Run("test", func(t *testing.T) {
projectSpec := v1alpha1.ProjectSpec{
Name: "My Test Project",
GroupID: int64(P1Config.integrationRootGroupID),
Name: "My Test Project",
FullPath: p1IntegrationRootGroupPath + "/" + integrationTestProjectPath,
Language: custom_p1.LangTypeAngular,
}
......@@ -43,9 +44,9 @@ func Test_P1_UpdateProject(t *testing.T) {
logPrefix := "Test_P1_UpdateProject"
t.Run("test", func(t *testing.T) {
projectSpec := v1alpha1.ProjectSpec{
Name: "My Test Project",
GroupID: int64(P1Config.integrationRootGroupID),
Language: custom_p1.LangTypeJavaMaven,
Name: "My Test Updated Project",
FullPath: p1IntegrationRootGroupPath + "/" + integrationTestProjectPath,
Language: custom_p1.LangTypeJavaMaven,
}
creds := custom_p1.ProjectCredentials{
......@@ -66,8 +67,7 @@ func Test_P1_DeleteProject(t *testing.T) {
logPrefix := "Test_P1_DeleteProject"
t.Run("test", func(t *testing.T) {
projectSpec := v1alpha1.ProjectSpec{
Name: "My Test Project",
GroupID: int64(P1Config.integrationRootGroupID),
FullPath: p1IntegrationRootGroupPath + "/" + integrationTestProjectPath,
}
creds := custom_p1.ProjectCredentials{
......
......@@ -6,7 +6,6 @@ import (
"testing"
"errors"
"os"
"testing"
gitlab "valkyrie.dso.mil/valkyrie-api/clients/gitlab"
)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment