diff --git a/README.md b/README.md
index 35143b0af6ff2fa07d73f14cb46972657faad171..5d003f515f927e656b454c9e8229c396784f0b4d 100644
--- a/README.md
+++ b/README.md
@@ -24,11 +24,11 @@ the name of your go project in `go.mod` (don't need to do this step unless you w
First off you will need to install benchstat for go. Then you
can do the following:
1. Write the benchmark test in a file, and run the command
-2. `go test -bench=<BenchmarkFunctionName> -benchmem -count 5 -run=^# | tee old.txt ./...` for
+2. `go test -bench=<BenchmarkFunctionName> -benchmem -count 5 -run=^# ./... | tee performance/old.txt` for
your old implementation, and
-`go test -bench=<BenchmarkFunctionName> -benchmem -count 5 -run=^# | tee new.txt ./...` for your new implementation where only the implementation called in the
+`go test -bench=<BenchmarkFunctionName> -benchmem -count 5 -run=^# ./... | tee performance/new.txt ` for your new implementation where only the implementation called in the
benchmark is changed.
-3. run `/path/to/benchstat path/to/old.txt path/to/new.txt` (running this command from the
+3. run `/path/to/benchstat performance/old.txt performance/new.txt > performance/last-diff.txt` (running this command from the
main repo path), and you have a decreasing delta on the new version,
you are doing better than in performance than the old version for
(speed, memory etc.)
diff --git a/performance/last-diff.txt b/performance/last-diff.txt
new file mode 100644
index 0000000000000000000000000000000000000000..51a52d8262c6ca9efc4bc9f4fd59442d60ca3f00
--- /dev/null
+++ b/performance/last-diff.txt
@@ -0,0 +1,8 @@
+name old time/op new time/op delta
+Collect/it_should_collect_jira_related_data-12 4.76ms ± 5% 4.72ms ± 7% ~ (p=1.000 n=5+5)
+
+name old alloc/op new alloc/op delta
+Collect/it_should_collect_jira_related_data-12 12.7MB ±11% 12.9MB ± 9% ~ (p=0.690 n=5+5)
+
+name old allocs/op new allocs/op delta
+Collect/it_should_collect_jira_related_data-12 4.22k ± 0% 4.11k ± 0% -2.71% (p=0.029 n=4+4)
diff --git a/performance/new.txt b/performance/new.txt
new file mode 100644
index 0000000000000000000000000000000000000000..f57eebba43ec5f41d364379ae70d3dd6c7f0b73c
--- /dev/null
+++ b/performance/new.txt
@@ -0,0 +1,81 @@
+PASS
+ok holocron/collector-jira-workflow/cmd 0.011s
+PASS
+ok holocron/collector-jira-workflow/config 0.010s
+goos: darwin
+goarch: amd64
+pkg: holocron/collector-jira-workflow/pkg/collector
+cpu: Intel(R) Core(TM) i7-8700B CPU @ 3.20GHz
+BenchmarkCollect/it_should_collect_jira_related_data-12 632 4752403 ns/op 12506353 B/op 4123 allocs/op
+--- BENCH: BenchmarkCollect/it_should_collect_jira_related_data-12
+ main_test.go:205: PASS: GetBoards()
+ main_test.go:205: PASS: DeleteBoard(mock.argumentMatcher)
+ main_test.go:205: PASS: UpsertBoard(mock.argumentMatcher)
+ main_test.go:205: PASS: UpsertBoard(mock.argumentMatcher)
+ main_test.go:205: PASS: GetActiveBoards()
+ main_test.go:205: PASS: UpsertBoard(mock.argumentMatcher)
+ main_test.go:205: PASS: GetTickets(mock.argumentMatcher)
+ main_test.go:205: PASS: DeleteTicket(mock.argumentMatcher)
+ main_test.go:205: PASS: UpsertTicket(mock.argumentMatcher)
+ main_test.go:205: PASS: UpsertTicket(mock.argumentMatcher)
+ ... [output truncated]
+BenchmarkCollect/it_should_collect_jira_related_data-12 632 4510787 ns/op 12475593 B/op 4108 allocs/op
+--- BENCH: BenchmarkCollect/it_should_collect_jira_related_data-12
+ main_test.go:205: PASS: GetBoards()
+ main_test.go:205: PASS: DeleteBoard(mock.argumentMatcher)
+ main_test.go:205: PASS: UpsertBoard(mock.argumentMatcher)
+ main_test.go:205: PASS: UpsertBoard(mock.argumentMatcher)
+ main_test.go:205: PASS: GetActiveBoards()
+ main_test.go:205: PASS: UpsertBoard(mock.argumentMatcher)
+ main_test.go:205: PASS: GetTickets(mock.argumentMatcher)
+ main_test.go:205: PASS: DeleteTicket(mock.argumentMatcher)
+ main_test.go:205: PASS: UpsertTicket(mock.argumentMatcher)
+ main_test.go:205: PASS: UpsertTicket(mock.argumentMatcher)
+ ... [output truncated]
+BenchmarkCollect/it_should_collect_jira_related_data-12 709 4977852 ns/op 13939821 B/op 4109 allocs/op
+--- BENCH: BenchmarkCollect/it_should_collect_jira_related_data-12
+ main_test.go:205: PASS: GetBoards()
+ main_test.go:205: PASS: DeleteBoard(mock.argumentMatcher)
+ main_test.go:205: PASS: UpsertBoard(mock.argumentMatcher)
+ main_test.go:205: PASS: UpsertBoard(mock.argumentMatcher)
+ main_test.go:205: PASS: GetActiveBoards()
+ main_test.go:205: PASS: UpsertBoard(mock.argumentMatcher)
+ main_test.go:205: PASS: GetTickets(mock.argumentMatcher)
+ main_test.go:205: PASS: DeleteTicket(mock.argumentMatcher)
+ main_test.go:205: PASS: UpsertTicket(mock.argumentMatcher)
+ main_test.go:205: PASS: UpsertTicket(mock.argumentMatcher)
+ ... [output truncated]
+BenchmarkCollect/it_should_collect_jira_related_data-12 708 4987159 ns/op 13922303 B/op 4110 allocs/op
+--- BENCH: BenchmarkCollect/it_should_collect_jira_related_data-12
+ main_test.go:205: PASS: GetBoards()
+ main_test.go:205: PASS: DeleteBoard(mock.argumentMatcher)
+ main_test.go:205: PASS: UpsertBoard(mock.argumentMatcher)
+ main_test.go:205: PASS: UpsertBoard(mock.argumentMatcher)
+ main_test.go:205: PASS: GetActiveBoards()
+ main_test.go:205: PASS: UpsertBoard(mock.argumentMatcher)
+ main_test.go:205: PASS: GetTickets(mock.argumentMatcher)
+ main_test.go:205: PASS: DeleteTicket(mock.argumentMatcher)
+ main_test.go:205: PASS: UpsertTicket(mock.argumentMatcher)
+ main_test.go:205: PASS: UpsertTicket(mock.argumentMatcher)
+ ... [output truncated]
+BenchmarkCollect/it_should_collect_jira_related_data-12 597 4365503 ns/op 11816312 B/op 4110 allocs/op
+--- BENCH: BenchmarkCollect/it_should_collect_jira_related_data-12
+ main_test.go:205: PASS: GetBoards()
+ main_test.go:205: PASS: DeleteBoard(mock.argumentMatcher)
+ main_test.go:205: PASS: UpsertBoard(mock.argumentMatcher)
+ main_test.go:205: PASS: UpsertBoard(mock.argumentMatcher)
+ main_test.go:205: PASS: GetActiveBoards()
+ main_test.go:205: PASS: UpsertBoard(mock.argumentMatcher)
+ main_test.go:205: PASS: GetTickets(mock.argumentMatcher)
+ main_test.go:205: PASS: DeleteTicket(mock.argumentMatcher)
+ main_test.go:205: PASS: UpsertTicket(mock.argumentMatcher)
+ main_test.go:205: PASS: UpsertTicket(mock.argumentMatcher)
+ ... [output truncated]
+PASS
+ok holocron/collector-jira-workflow/pkg/collector 16.484s
+PASS
+ok holocron/collector-jira-workflow/pkg/database 0.012s
+PASS
+ok holocron/collector-jira-workflow/pkg/httpClient 0.012s
+PASS
+ok holocron/collector-jira-workflow/pkg/utils 0.010s
diff --git a/performance/old.txt b/performance/old.txt
new file mode 100644
index 0000000000000000000000000000000000000000..f600a6c0568a13e67cacf386046b6cd89c4a02cd
--- /dev/null
+++ b/performance/old.txt
@@ -0,0 +1,81 @@
+PASS
+ok holocron/collector-jira-workflow/cmd 0.011s
+PASS
+ok holocron/collector-jira-workflow/config 0.011s
+goos: darwin
+goarch: amd64
+pkg: holocron/collector-jira-workflow/pkg/collector
+cpu: Intel(R) Core(TM) i7-8700B CPU @ 3.20GHz
+BenchmarkCollect/it_should_collect_jira_related_data-12 571 4507704 ns/op 11354847 B/op 4235 allocs/op
+--- BENCH: BenchmarkCollect/it_should_collect_jira_related_data-12
+ main_test.go:205: PASS: GetBoards()
+ main_test.go:205: PASS: DeleteBoard(mock.argumentMatcher)
+ main_test.go:205: PASS: UpsertBoard(mock.argumentMatcher)
+ main_test.go:205: PASS: UpsertBoard(mock.argumentMatcher)
+ main_test.go:205: PASS: GetActiveBoards()
+ main_test.go:205: PASS: UpsertBoard(mock.argumentMatcher)
+ main_test.go:205: PASS: GetTickets(mock.argumentMatcher)
+ main_test.go:205: PASS: DeleteTicket(mock.argumentMatcher)
+ main_test.go:205: PASS: UpsertTicket(mock.argumentMatcher)
+ main_test.go:205: PASS: UpsertTicket(mock.argumentMatcher)
+ ... [output truncated]
+BenchmarkCollect/it_should_collect_jira_related_data-12 685 4916627 ns/op 13496644 B/op 4224 allocs/op
+--- BENCH: BenchmarkCollect/it_should_collect_jira_related_data-12
+ main_test.go:205: PASS: GetBoards()
+ main_test.go:205: PASS: DeleteBoard(mock.argumentMatcher)
+ main_test.go:205: PASS: UpsertBoard(mock.argumentMatcher)
+ main_test.go:205: PASS: UpsertBoard(mock.argumentMatcher)
+ main_test.go:205: PASS: GetActiveBoards()
+ main_test.go:205: PASS: UpsertBoard(mock.argumentMatcher)
+ main_test.go:205: PASS: GetTickets(mock.argumentMatcher)
+ main_test.go:205: PASS: DeleteTicket(mock.argumentMatcher)
+ main_test.go:205: PASS: UpsertTicket(mock.argumentMatcher)
+ main_test.go:205: PASS: UpsertTicket(mock.argumentMatcher)
+ ... [output truncated]
+BenchmarkCollect/it_should_collect_jira_related_data-12 685 4917054 ns/op 13491501 B/op 4223 allocs/op
+--- BENCH: BenchmarkCollect/it_should_collect_jira_related_data-12
+ main_test.go:205: PASS: GetBoards()
+ main_test.go:205: PASS: DeleteBoard(mock.argumentMatcher)
+ main_test.go:205: PASS: UpsertBoard(mock.argumentMatcher)
+ main_test.go:205: PASS: UpsertBoard(mock.argumentMatcher)
+ main_test.go:205: PASS: GetActiveBoards()
+ main_test.go:205: PASS: UpsertBoard(mock.argumentMatcher)
+ main_test.go:205: PASS: GetTickets(mock.argumentMatcher)
+ main_test.go:205: PASS: DeleteTicket(mock.argumentMatcher)
+ main_test.go:205: PASS: UpsertTicket(mock.argumentMatcher)
+ main_test.go:205: PASS: UpsertTicket(mock.argumentMatcher)
+ ... [output truncated]
+BenchmarkCollect/it_should_collect_jira_related_data-12 608 4513139 ns/op 12032696 B/op 4223 allocs/op
+--- BENCH: BenchmarkCollect/it_should_collect_jira_related_data-12
+ main_test.go:205: PASS: GetBoards()
+ main_test.go:205: PASS: DeleteBoard(mock.argumentMatcher)
+ main_test.go:205: PASS: UpsertBoard(mock.argumentMatcher)
+ main_test.go:205: PASS: UpsertBoard(mock.argumentMatcher)
+ main_test.go:205: PASS: GetActiveBoards()
+ main_test.go:205: PASS: UpsertBoard(mock.argumentMatcher)
+ main_test.go:205: PASS: GetTickets(mock.argumentMatcher)
+ main_test.go:205: PASS: DeleteTicket(mock.argumentMatcher)
+ main_test.go:205: PASS: UpsertTicket(mock.argumentMatcher)
+ main_test.go:205: PASS: UpsertTicket(mock.argumentMatcher)
+ ... [output truncated]
+BenchmarkCollect/it_should_collect_jira_related_data-12 674 4930024 ns/op 13285131 B/op 4224 allocs/op
+--- BENCH: BenchmarkCollect/it_should_collect_jira_related_data-12
+ main_test.go:205: PASS: GetBoards()
+ main_test.go:205: PASS: DeleteBoard(mock.argumentMatcher)
+ main_test.go:205: PASS: UpsertBoard(mock.argumentMatcher)
+ main_test.go:205: PASS: UpsertBoard(mock.argumentMatcher)
+ main_test.go:205: PASS: GetActiveBoards()
+ main_test.go:205: PASS: UpsertBoard(mock.argumentMatcher)
+ main_test.go:205: PASS: GetTickets(mock.argumentMatcher)
+ main_test.go:205: PASS: DeleteTicket(mock.argumentMatcher)
+ main_test.go:205: PASS: UpsertTicket(mock.argumentMatcher)
+ main_test.go:205: PASS: UpsertTicket(mock.argumentMatcher)
+ ... [output truncated]
+PASS
+ok holocron/collector-jira-workflow/pkg/collector 16.355s
+PASS
+ok holocron/collector-jira-workflow/pkg/database 0.013s
+PASS
+ok holocron/collector-jira-workflow/pkg/httpClient 0.013s
+PASS
+ok holocron/collector-jira-workflow/pkg/utils 0.015s
diff --git a/pkg/collector/collectBoards.go b/pkg/collector/collectBoards.go
index 807a4b8d62b0f790fc22f722c2efc5c528dc590a..6820e78df54bea323a8ac63d3f1fb76339ce7b9e 100644
--- a/pkg/collector/collectBoards.go
+++ b/pkg/collector/collectBoards.go
@@ -1,27 +1,32 @@
package collector
import (
- "fmt"
"time"
- "holocron/collector-jira-workflow/config"
"holocron/collector-jira-workflow/pkg/database"
"holocron/collector-jira-workflow/pkg/httpClient"
+ "holocron/collector-jira-workflow/pkg/utils"
)
-func collectBoards(db database.DB) *[]database.BoardJSON {
- currentTime := time.Now()
- boards := httpClient.GetBoards()
- for i, b := range *boards {
- (*boards)[i].API_URL = fmt.Sprintf("%s/board/%d", config.API_URL, b.ID)
- (*boards)[i].LastCollected = currentTime
+func saveBoards(boards *[]database.BoardJSON, db database.DB) {
+ utils.Logger.Info("Saving boards...")
+ for _, board := range *boards {
+ db.UpsertBoard(board)
}
- pruneBoards(boards, db)
+}
+
+func setLastUpdated(activeBoards []database.CollectorTarget, time time.Time,
+ db database.DB) {
- return boards
+ for _, board := range activeBoards {
+ currentBoard := board.ToBoardJSON()
+ currentBoard.LastCollected = time
+ db.UpsertBoard(currentBoard)
+ }
}
-func pruneBoards(collectedBoards *[]database.BoardJSON, db database.DB) {
+func pruneBoards(collectedBoards *[]database.BoardJSON, db database.DB) (delCount int) {
+ utils.Logger.Info("Deleting boards...")
savedBoards := db.GetBoards()
var collectedHashTable map[uint]bool = make(map[uint]bool, len(*collectedBoards))
for _, board := range *collectedBoards {
@@ -30,6 +35,25 @@ func pruneBoards(collectedBoards *[]database.BoardJSON, db database.DB) {
for _, board := range savedBoards {
if _, boardInAPIResp := collectedHashTable[board.BoardId]; !boardInAPIResp {
db.DeleteBoard(board)
+ delCount++
}
}
+
+ return delCount
+}
+
+func collectBoards(db database.DB) {
+ utils.Logger.Info("Collecting boards...")
+ startTime := time.Now()
+ boards := httpClient.GetBoards()
+ delCount := pruneBoards(boards, db)
+ saveBoards(boards, db)
+
+ statsFormattedStr := "\n---Finished collecting boards---\n" +
+ "---Time Elapsed - %.0f seconds---\n" +
+ "%d\t-\tBoards Collected\n" +
+ "%d\t-\tBoards Removed\n"
+ utils.Logger.Infof(statsFormattedStr,
+ time.Since(startTime).Seconds(),
+ len(*boards), delCount)
}
diff --git a/pkg/collector/collectBoards_test.go b/pkg/collector/collectBoards_test.go
index 98773bf3c23a92db3e168dfd4f9930b6f89eeac5..b5d8da1d4f13a3bc6337ef9917974518c23a06ea 100644
--- a/pkg/collector/collectBoards_test.go
+++ b/pkg/collector/collectBoards_test.go
@@ -1,6 +1,7 @@
package collector
import (
+ "bytes"
"net/http"
"net/http/httptest"
"testing"
@@ -8,10 +9,12 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
+ "holocron/collector-jira-workflow/pkg/database"
db "holocron/collector-jira-workflow/pkg/database"
+ "holocron/collector-jira-workflow/pkg/utils"
)
-//nolint: funlen // This is just a test.
+//nolint: funlen, gocyclo // This is just a test.
func TestCollectBoards(t *testing.T) {
var testCases = []struct {
AddAPIMocks func(*mockAPI)
@@ -33,18 +36,22 @@ func TestCollectBoards(t *testing.T) {
mdb.T().Helper()
mdb.On("GetBoards").Return([]db.CollectorTarget{}).Once()
+
+ mdb.On("UpsertBoard", mock.MatchedBy(func(b database.BoardJSON) bool {
+ return b.ID == uint(1) && b.Name == "test1" && b.URL == "test.com/1"
+ })).Once()
},
AssertExpectations: func(mdb *mockDB, api *mockAPI, _ *httptest.Server) {
defer func() {
mdb.AssertExpectations(mdb.T())
api.AssertExpectations(api.T())
}()
- result := *collectBoards(mdb)
-
- assert.Equal(mdb.T(), 1, len(result))
- assert.Equal(mdb.T(), uint(1), result[0].ID)
- assert.Equal(mdb.T(), "test1", result[0].Name)
- assert.Equal(mdb.T(), "test.com/1", result[0].URL)
+ // setting output recorder
+ buffer := bytes.Buffer{}
+ utils.Logger.InfoLogger.SetOutput(&buffer)
+ // testing
+ collectBoards(mdb)
+ assert.Contains(mdb.T(), buffer.String(), "1\t-\tBoards Collected")
},
},
{
@@ -63,24 +70,30 @@ func TestCollectBoards(t *testing.T) {
getMock("testdata/db/collector_targets/boards-to-delete.json",
mdb.T(), &boardsToDelete)
mdb.On("GetBoards").Return(boardsToDelete).Once()
+
for _, board := range boardsToDelete {
board := board
mdb.On("DeleteBoard", mock.MatchedBy(func(b db.CollectorTarget) bool {
return b.Name == board.Name
})).Once()
}
+
+ mdb.On("UpsertBoard", mock.MatchedBy(func(b database.BoardJSON) bool {
+ return b.ID == uint(5) && b.Name == "test" && b.URL == "test.com"
+ })).Once()
},
AssertExpectations: func(mdb *mockDB, api *mockAPI, _ *httptest.Server) {
defer func() {
mdb.AssertExpectations(mdb.T())
api.AssertExpectations(api.T())
}()
- result := *collectBoards(mdb)
-
- assert.Equal(mdb.T(), 1, len(result))
- assert.Equal(mdb.T(), uint(5), result[0].ID)
- assert.Equal(mdb.T(), "test", result[0].Name)
- assert.Equal(mdb.T(), "test.com", result[0].URL)
+ // setting output recorder
+ buffer := bytes.Buffer{}
+ utils.Logger.InfoLogger.SetOutput(&buffer)
+ // testing
+ collectBoards(mdb)
+ assert.Contains(mdb.T(), buffer.String(), "1\t-\tBoards Collected")
+ assert.Contains(mdb.T(), buffer.String(), "1\t-\tBoards Removed")
},
},
}
diff --git a/pkg/collector/collectTickets.go b/pkg/collector/collectTickets.go
index 209dc2b4ea594abdefc0a7e3b2a97df05c1a9e9b..c20c38cfc92fb6e12a4dbb340bbaee56de041c5e 100644
--- a/pkg/collector/collectTickets.go
+++ b/pkg/collector/collectTickets.go
@@ -1,43 +1,25 @@
package collector
import (
+ "sync"
+ "time"
+
"holocron/collector-jira-workflow/pkg/database"
"holocron/collector-jira-workflow/pkg/httpClient"
+ "holocron/collector-jira-workflow/pkg/utils"
)
-func collectTickets(
- boards []database.CollectorTarget, db database.DB, ch chan *[]database.TicketJSON,
-) {
-
- ticketChannels := make(chan *[]database.TicketJSON, len(boards))
- for _, board := range boards {
- go collectTicketsForBoard(board, db, ticketChannels)
- }
- result := &[]database.TicketJSON{}
- for range boards {
- *result = append(*result, *<-ticketChannels...)
- }
- ch <- result
-}
-
-func collectTicketsForBoard(
- board database.CollectorTarget, db database.DB, ch chan *[]database.TicketJSON,
-) {
-
- tickets := httpClient.GetTickets(board)
- pruneTickets(tickets, board, db)
- for i := range *tickets {
- (*tickets)[i].BoardId = board.ID
- (*tickets)[i].BuildStatusHistory()
+func saveTickets(tickets []database.TicketJSON, db database.DB) {
+ for _, ticket := range tickets {
+ db.UpsertTicket(ticket)
}
- ch <- tickets
}
func pruneTickets(
collectedTickets *[]database.TicketJSON,
board database.CollectorTarget,
db database.DB,
-) {
+) (delCount int) {
savedTickets := db.GetTickets(board)
var collectedHashTable map[string]bool = make(map[string]bool, len(*collectedTickets))
@@ -47,6 +29,45 @@ func pruneTickets(
for _, ticket := range savedTickets {
if _, ticketInAPIResp := collectedHashTable[ticket.TicketID]; !ticketInAPIResp {
db.DeleteTicket(ticket)
+ delCount++
}
}
+
+ return delCount
+}
+
+func collectTickets(activeBoards []database.CollectorTarget, db database.DB) {
+ utils.Logger.Info("Collecting workflow data...")
+ var (
+ mtx sync.Mutex
+ wg sync.WaitGroup
+ delCount = 0
+ saveCount = 0
+ startTime = time.Now()
+ )
+ wg.Add(len(activeBoards))
+ for _, board := range activeBoards {
+ go func(board database.CollectorTarget) {
+ // let the fetching happen concurrently
+ defer wg.Done()
+ ticketsForBoard := httpClient.GetTickets(board)
+ // but only one go routine can write to db, and count variables
+ // at a time
+ mtx.Lock()
+ defer mtx.Unlock()
+ deleted := pruneTickets(ticketsForBoard, board, db)
+ saveTickets(*ticketsForBoard, db)
+ saveCount += len(*ticketsForBoard)
+ delCount += deleted
+ }(board)
+ }
+ wg.Wait()
+
+ statsFormattedStr := "\n---Finished collecting tickets---\n" +
+ "---Time Elapsed - %.0f seconds---\n" +
+ "%d\t-\tTickets Collected\n" +
+ "%d\t-\tTickets Removed\n"
+ utils.Logger.Infof(statsFormattedStr,
+ time.Since(startTime).Seconds(),
+ saveCount, delCount)
}
diff --git a/pkg/collector/collectTickets_test.go b/pkg/collector/collectTickets_test.go
index 8573ee3b8fe638b9fef52b69ab4df3353bf39cfb..9fba3d6d9559d0fa177cfb8063c930f1fbbea026 100644
--- a/pkg/collector/collectTickets_test.go
+++ b/pkg/collector/collectTickets_test.go
@@ -1,6 +1,7 @@
package collector
import (
+ "bytes"
"net/http"
"net/http/httptest"
"testing"
@@ -9,9 +10,10 @@ import (
"github.com/stretchr/testify/mock"
db "holocron/collector-jira-workflow/pkg/database"
+ "holocron/collector-jira-workflow/pkg/utils"
)
-//nolint: funlen // This is just a test.
+//nolint: funlen, gocyclo // This is just a test.
func TestCollectTickets(t *testing.T) {
var testCases = []struct {
AddAPIMocks func(*mockAPI)
@@ -39,6 +41,12 @@ func TestCollectTickets(t *testing.T) {
mdb.On("GetTickets", mock.MatchedBy(func(b db.CollectorTarget) bool {
return b.Name == activeBoards[0].Name
})).Return([]db.Ticket{})
+
+ mdb.On("UpsertTicket", mock.MatchedBy(func(t db.TicketJSON) bool {
+ return t.ID == "1" && t.Fields.Status.Name == "testStatus1" &&
+ t.Fields.Assignee.Email == "testEmail1" &&
+ t.Fields.Type.Name == "testType1"
+ })).Once()
},
//nolint:dupl // This is just a test.
AssertExpectations: func(mdb *mockDB, api *mockAPI, s *httptest.Server) {
@@ -46,19 +54,17 @@ func TestCollectTickets(t *testing.T) {
mdb.AssertExpectations(mdb.T())
api.AssertExpectations(api.T())
}()
+ // saving logs
+ buffer := bytes.Buffer{}
+ utils.Logger.InfoLogger.SetOutput(&buffer)
+ // testing
var boards []db.CollectorTarget
getMock("testdata/db/collector_targets/collector-targets.json", mdb.T(), &boards)
var activeBoards = []db.CollectorTarget{boards[0]}
activeBoards[0].API_URL = s.URL
- resultChannel := make(chan *[]db.TicketJSON)
- go collectTickets(activeBoards, mdb, resultChannel)
- result := *<-resultChannel
+ collectTickets(activeBoards, mdb)
- assert.Equal(mdb.T(), 1, len(result))
- assert.Equal(mdb.T(), "1", result[0].ID)
- assert.Equal(mdb.T(), "testStatus1", result[0].Fields.Status.Name)
- assert.Equal(mdb.T(), "testEmail1", result[0].Fields.Assignee.Email)
- assert.Equal(mdb.T(), "testType1", result[0].Fields.Type.Name)
+ assert.Contains(mdb.T(), buffer.String(), "1\t-\tTickets Collected")
},
},
{
@@ -83,6 +89,12 @@ func TestCollectTickets(t *testing.T) {
return b.Name == activeBoards[0].Name
})).Return(ticketsToDelete).Once()
+ mdb.On("UpsertTicket", mock.MatchedBy(func(t db.TicketJSON) bool {
+ return t.ID == "5" && t.Fields.Status.Name == "testStatus1" &&
+ t.Fields.Assignee.Email == "testEmail1" &&
+ t.Fields.Type.Name == "testType1"
+ })).Once()
+
mdb.On("DeleteTicket", mock.MatchedBy(func(t db.Ticket) bool {
return t.Assignee == ticketsToDelete[0].Assignee
})).Once()
@@ -93,20 +105,19 @@ func TestCollectTickets(t *testing.T) {
mdb.AssertExpectations(mdb.T())
api.AssertExpectations(api.T())
}()
+ // saving logs
+ buffer := bytes.Buffer{}
+ utils.Logger.InfoLogger.SetOutput(&buffer)
+ // testing
var boards []db.CollectorTarget
getMock("testdata/db/collector_targets/collector-targets.json",
mdb.T(), &boards)
var activeBoards = []db.CollectorTarget{boards[0]}
activeBoards[0].API_URL = s.URL
- resultChannel := make(chan *[]db.TicketJSON)
- go collectTickets(activeBoards, mdb, resultChannel)
- result := *<-resultChannel
+ collectTickets(activeBoards, mdb)
- assert.Equal(mdb.T(), 1, len(result))
- assert.Equal(mdb.T(), "5", result[0].ID)
- assert.Equal(mdb.T(), "testStatus1", result[0].Fields.Status.Name)
- assert.Equal(mdb.T(), "testEmail1", result[0].Fields.Assignee.Email)
- assert.Equal(mdb.T(), "testType1", result[0].Fields.Type.Name)
+ assert.Contains(mdb.T(), buffer.String(), "1\t-\tTickets Collected")
+ assert.Contains(mdb.T(), buffer.String(), "1\t-\tTickets Removed")
},
},
}
diff --git a/pkg/collector/main.go b/pkg/collector/main.go
index c81df98d6ba2b9e6d1e3dabe994cd6940fdd05b0..442bbf3f6ad9aee2896996978e03c4977eb70c36 100644
--- a/pkg/collector/main.go
+++ b/pkg/collector/main.go
@@ -4,59 +4,11 @@ import (
"time"
"holocron/collector-jira-workflow/pkg/database"
- "holocron/collector-jira-workflow/pkg/utils"
)
func Collect(db database.DB) {
- startTime := time.Now()
-
- utils.Logger.Info("Collecting boards...")
- boards := collectBoards(db)
- utils.Logger.Info("Saving boards...")
- saveBoards(boards, db)
+ collectBoards(db)
activeBoards := db.GetActiveBoards()
-
- utils.Logger.Info("Collecting workflow data...")
- ticketsChannel := make(chan *[]database.TicketJSON)
-
- go collectTickets(activeBoards, db, ticketsChannel)
-
- tickets := <-ticketsChannel
-
- utils.Logger.Info("Saving workflow data...")
- saveTickets(tickets, db)
- utils.Logger.Info("Workflow data saved.")
-
- // set LastCollected after collection is done
+ collectTickets(activeBoards, db)
setLastUpdated(activeBoards, time.Now(), db)
-
- utils.Logger.Infof("--Finished Collecting---\n"+
- "%d\t-\tBoards\n"+
- "%d\t-\tTickets\n"+
- "---Time Elapsed - %.0f seconds---",
- len(*boards),
- len(*tickets),
- time.Since(startTime).Seconds(),
- )
-}
-
-func saveTickets(tickets *[]database.TicketJSON, db database.DB) {
- for _, ticket := range *tickets {
- db.UpsertTicket(ticket)
- }
-}
-
-func saveBoards(boards *[]database.BoardJSON, db database.DB) {
- for _, board := range *boards {
- db.UpsertBoard(board)
- }
-}
-func setLastUpdated(activeBoards []database.CollectorTarget, time time.Time,
- db database.DB) {
-
- for _, board := range activeBoards {
- currentBoard := board.ToBoardJSON()
- currentBoard.LastCollected = time
- db.UpsertBoard(currentBoard)
- }
}
diff --git a/pkg/httpClient/request.go b/pkg/httpClient/request.go
index d13bd21d3f492d852fc1fa3419d28c81ca31f3a9..692ffa07b21f7c005e45f6c709c775009b83a4be 100644
--- a/pkg/httpClient/request.go
+++ b/pkg/httpClient/request.go
@@ -73,8 +73,17 @@ func getItemsInPage(url string, itemsPtr interface{}) error {
}
func GetBoards() *[]db.BoardJSON {
- boards := &db.BoardsJSON{Boards: []db.BoardJSON{}}
const perPage = 50
+ var (
+ currentTime = time.Now()
+ boards = &db.BoardsJSON{Boards: []db.BoardJSON{}}
+ setCustomFields = func(boards []db.BoardJSON) {
+ for i, b := range boards {
+ boards[i].API_URL = fmt.Sprintf("%s/board/%d", config.API_URL, b.ID)
+ boards[i].LastCollected = currentTime
+ }
+ }
+ )
for startAt, hasNext := 0, true; hasNext; startAt += perPage {
var pageBoards db.BoardsJSON
if err := getItemsInPage(fmt.Sprintf("%s/board?startAt=%d", config.API_URL, startAt),
@@ -84,13 +93,22 @@ func GetBoards() *[]db.BoardJSON {
boards.Boards = append(boards.Boards, pageBoards.Boards...)
hasNext = !pageBoards.IsLast
}
+ setCustomFields(boards.Boards)
return &boards.Boards
}
func GetTickets(board db.CollectorTarget) *[]db.TicketJSON {
- tickets := &db.TicketsJSON{Tickets: []db.TicketJSON{}}
const perPage = 100
+ var (
+ tickets = &db.TicketsJSON{Tickets: []db.TicketJSON{}}
+ setCustomFields = func(tickets []db.TicketJSON) {
+ for i := range tickets {
+ tickets[i].BoardId = board.ID
+ tickets[i].BuildStatusHistory()
+ }
+ }
+ )
for startAt, next := 0, true; next; startAt += perPage {
var pageTickets db.TicketsJSON
if err := getItemsInPage(fmt.Sprintf(
@@ -103,6 +121,7 @@ func GetTickets(board db.CollectorTarget) *[]db.TicketJSON {
tickets.Tickets = append(tickets.Tickets, pageTickets.Tickets...)
next = pageTickets.Total > startAt+perPage
}
+ setCustomFields(tickets.Tickets)
return &tickets.Tickets
}