Go sync

README.md

@@ -24,11 +24,11 @@ the name of your go project in `go.mod` (don't need to do this step unless you w
 First off you will need to install benchstat for go. Then you
 can do the following:
 1. Write the benchmark test in a file, and run the command
-2. `go test -bench=<BenchmarkFunctionName> -benchmem -count 5 -run=^# | tee old.txt ./...` for
+2. `go test -bench=<BenchmarkFunctionName> -benchmem -count 5 -run=^#  ./... | tee performance/old.txt` for
 your old implementation, and 
-`go test -bench=<BenchmarkFunctionName> -benchmem -count 5 -run=^# | tee new.txt ./...` for your new implementation where only the implementation called in the
+`go test -bench=<BenchmarkFunctionName> -benchmem -count 5 -run=^# ./... | tee performance/new.txt ` for your new implementation where only the implementation called in the
 benchmark is changed.
-3. run `/path/to/benchstat path/to/old.txt path/to/new.txt` (running this command from the
+3. run `/path/to/benchstat performance/old.txt performance/new.txt > performance/last-diff.txt` (running this command from the
 main repo path), and you have a decreasing delta on the new version,
 you are doing better than in performance than the old version for
 (speed, memory etc.)

performance/last-diff.txt

+name                                            old time/op    new time/op    delta
+Collect/it_should_collect_jira_related_data-12    4.76ms ± 5%    4.72ms ± 7%    ~     (p=1.000 n=5+5)
+
+name                                            old alloc/op   new alloc/op   delta
+Collect/it_should_collect_jira_related_data-12    12.7MB ±11%    12.9MB ± 9%    ~     (p=0.690 n=5+5)
+
+name                                            old allocs/op  new allocs/op  delta
+Collect/it_should_collect_jira_related_data-12     4.22k ± 0%     4.11k ± 0%  -2.71%  (p=0.029 n=4+4)

performance/new.txt

+PASS
+ok  	holocron/collector-jira-workflow/cmd	0.011s
+PASS
+ok  	holocron/collector-jira-workflow/config	0.010s
+goos: darwin
+goarch: amd64
+pkg: holocron/collector-jira-workflow/pkg/collector
+cpu: Intel(R) Core(TM) i7-8700B CPU @ 3.20GHz
+BenchmarkCollect/it_should_collect_jira_related_data-12         	     632	   4752403 ns/op	12506353 B/op	    4123 allocs/op
+--- BENCH: BenchmarkCollect/it_should_collect_jira_related_data-12
+    main_test.go:205: PASS:	GetBoards()
+    main_test.go:205: PASS:	DeleteBoard(mock.argumentMatcher)
+    main_test.go:205: PASS:	UpsertBoard(mock.argumentMatcher)
+    main_test.go:205: PASS:	UpsertBoard(mock.argumentMatcher)
+    main_test.go:205: PASS:	GetActiveBoards()
+    main_test.go:205: PASS:	UpsertBoard(mock.argumentMatcher)
+    main_test.go:205: PASS:	GetTickets(mock.argumentMatcher)
+    main_test.go:205: PASS:	DeleteTicket(mock.argumentMatcher)
+    main_test.go:205: PASS:	UpsertTicket(mock.argumentMatcher)
+    main_test.go:205: PASS:	UpsertTicket(mock.argumentMatcher)
+	... [output truncated]
+BenchmarkCollect/it_should_collect_jira_related_data-12         	     632	   4510787 ns/op	12475593 B/op	    4108 allocs/op
+--- BENCH: BenchmarkCollect/it_should_collect_jira_related_data-12
+    main_test.go:205: PASS:	GetBoards()
+    main_test.go:205: PASS:	DeleteBoard(mock.argumentMatcher)
+    main_test.go:205: PASS:	UpsertBoard(mock.argumentMatcher)
+    main_test.go:205: PASS:	UpsertBoard(mock.argumentMatcher)
+    main_test.go:205: PASS:	GetActiveBoards()
+    main_test.go:205: PASS:	UpsertBoard(mock.argumentMatcher)
+    main_test.go:205: PASS:	GetTickets(mock.argumentMatcher)
+    main_test.go:205: PASS:	DeleteTicket(mock.argumentMatcher)
+    main_test.go:205: PASS:	UpsertTicket(mock.argumentMatcher)
+    main_test.go:205: PASS:	UpsertTicket(mock.argumentMatcher)
+	... [output truncated]
+BenchmarkCollect/it_should_collect_jira_related_data-12         	     709	   4977852 ns/op	13939821 B/op	    4109 allocs/op
+--- BENCH: BenchmarkCollect/it_should_collect_jira_related_data-12
+    main_test.go:205: PASS:	GetBoards()
+    main_test.go:205: PASS:	DeleteBoard(mock.argumentMatcher)
+    main_test.go:205: PASS:	UpsertBoard(mock.argumentMatcher)
+    main_test.go:205: PASS:	UpsertBoard(mock.argumentMatcher)
+    main_test.go:205: PASS:	GetActiveBoards()
+    main_test.go:205: PASS:	UpsertBoard(mock.argumentMatcher)
+    main_test.go:205: PASS:	GetTickets(mock.argumentMatcher)
+    main_test.go:205: PASS:	DeleteTicket(mock.argumentMatcher)
+    main_test.go:205: PASS:	UpsertTicket(mock.argumentMatcher)
+    main_test.go:205: PASS:	UpsertTicket(mock.argumentMatcher)
+	... [output truncated]
+BenchmarkCollect/it_should_collect_jira_related_data-12         	     708	   4987159 ns/op	13922303 B/op	    4110 allocs/op
+--- BENCH: BenchmarkCollect/it_should_collect_jira_related_data-12
+    main_test.go:205: PASS:	GetBoards()
+    main_test.go:205: PASS:	DeleteBoard(mock.argumentMatcher)
+    main_test.go:205: PASS:	UpsertBoard(mock.argumentMatcher)
+    main_test.go:205: PASS:	UpsertBoard(mock.argumentMatcher)
+    main_test.go:205: PASS:	GetActiveBoards()
+    main_test.go:205: PASS:	UpsertBoard(mock.argumentMatcher)
+    main_test.go:205: PASS:	GetTickets(mock.argumentMatcher)
+    main_test.go:205: PASS:	DeleteTicket(mock.argumentMatcher)
+    main_test.go:205: PASS:	UpsertTicket(mock.argumentMatcher)
+    main_test.go:205: PASS:	UpsertTicket(mock.argumentMatcher)
+	... [output truncated]
+BenchmarkCollect/it_should_collect_jira_related_data-12         	     597	   4365503 ns/op	11816312 B/op	    4110 allocs/op
+--- BENCH: BenchmarkCollect/it_should_collect_jira_related_data-12
+    main_test.go:205: PASS:	GetBoards()
+    main_test.go:205: PASS:	DeleteBoard(mock.argumentMatcher)
+    main_test.go:205: PASS:	UpsertBoard(mock.argumentMatcher)
+    main_test.go:205: PASS:	UpsertBoard(mock.argumentMatcher)
+    main_test.go:205: PASS:	GetActiveBoards()
+    main_test.go:205: PASS:	UpsertBoard(mock.argumentMatcher)
+    main_test.go:205: PASS:	GetTickets(mock.argumentMatcher)
+    main_test.go:205: PASS:	DeleteTicket(mock.argumentMatcher)
+    main_test.go:205: PASS:	UpsertTicket(mock.argumentMatcher)
+    main_test.go:205: PASS:	UpsertTicket(mock.argumentMatcher)
+	... [output truncated]
+PASS
+ok  	holocron/collector-jira-workflow/pkg/collector	16.484s
+PASS
+ok  	holocron/collector-jira-workflow/pkg/database	0.012s
+PASS
+ok  	holocron/collector-jira-workflow/pkg/httpClient	0.012s
+PASS
+ok  	holocron/collector-jira-workflow/pkg/utils	0.010s

performance/old.txt

+PASS
+ok  	holocron/collector-jira-workflow/cmd	0.011s
+PASS
+ok  	holocron/collector-jira-workflow/config	0.011s
+goos: darwin
+goarch: amd64
+pkg: holocron/collector-jira-workflow/pkg/collector
+cpu: Intel(R) Core(TM) i7-8700B CPU @ 3.20GHz
+BenchmarkCollect/it_should_collect_jira_related_data-12         	     571	   4507704 ns/op	11354847 B/op	    4235 allocs/op
+--- BENCH: BenchmarkCollect/it_should_collect_jira_related_data-12
+    main_test.go:205: PASS:	GetBoards()
+    main_test.go:205: PASS:	DeleteBoard(mock.argumentMatcher)
+    main_test.go:205: PASS:	UpsertBoard(mock.argumentMatcher)
+    main_test.go:205: PASS:	UpsertBoard(mock.argumentMatcher)
+    main_test.go:205: PASS:	GetActiveBoards()
+    main_test.go:205: PASS:	UpsertBoard(mock.argumentMatcher)
+    main_test.go:205: PASS:	GetTickets(mock.argumentMatcher)
+    main_test.go:205: PASS:	DeleteTicket(mock.argumentMatcher)
+    main_test.go:205: PASS:	UpsertTicket(mock.argumentMatcher)
+    main_test.go:205: PASS:	UpsertTicket(mock.argumentMatcher)
+	... [output truncated]
+BenchmarkCollect/it_should_collect_jira_related_data-12         	     685	   4916627 ns/op	13496644 B/op	    4224 allocs/op
+--- BENCH: BenchmarkCollect/it_should_collect_jira_related_data-12
+    main_test.go:205: PASS:	GetBoards()
+    main_test.go:205: PASS:	DeleteBoard(mock.argumentMatcher)
+    main_test.go:205: PASS:	UpsertBoard(mock.argumentMatcher)
+    main_test.go:205: PASS:	UpsertBoard(mock.argumentMatcher)
+    main_test.go:205: PASS:	GetActiveBoards()
+    main_test.go:205: PASS:	UpsertBoard(mock.argumentMatcher)
+    main_test.go:205: PASS:	GetTickets(mock.argumentMatcher)
+    main_test.go:205: PASS:	DeleteTicket(mock.argumentMatcher)
+    main_test.go:205: PASS:	UpsertTicket(mock.argumentMatcher)
+    main_test.go:205: PASS:	UpsertTicket(mock.argumentMatcher)
+	... [output truncated]
+BenchmarkCollect/it_should_collect_jira_related_data-12         	     685	   4917054 ns/op	13491501 B/op	    4223 allocs/op
+--- BENCH: BenchmarkCollect/it_should_collect_jira_related_data-12
+    main_test.go:205: PASS:	GetBoards()
+    main_test.go:205: PASS:	DeleteBoard(mock.argumentMatcher)
+    main_test.go:205: PASS:	UpsertBoard(mock.argumentMatcher)
+    main_test.go:205: PASS:	UpsertBoard(mock.argumentMatcher)
+    main_test.go:205: PASS:	GetActiveBoards()
+    main_test.go:205: PASS:	UpsertBoard(mock.argumentMatcher)
+    main_test.go:205: PASS:	GetTickets(mock.argumentMatcher)
+    main_test.go:205: PASS:	DeleteTicket(mock.argumentMatcher)
+    main_test.go:205: PASS:	UpsertTicket(mock.argumentMatcher)
+    main_test.go:205: PASS:	UpsertTicket(mock.argumentMatcher)
+	... [output truncated]
+BenchmarkCollect/it_should_collect_jira_related_data-12         	     608	   4513139 ns/op	12032696 B/op	    4223 allocs/op
+--- BENCH: BenchmarkCollect/it_should_collect_jira_related_data-12
+    main_test.go:205: PASS:	GetBoards()
+    main_test.go:205: PASS:	DeleteBoard(mock.argumentMatcher)
+    main_test.go:205: PASS:	UpsertBoard(mock.argumentMatcher)
+    main_test.go:205: PASS:	UpsertBoard(mock.argumentMatcher)
+    main_test.go:205: PASS:	GetActiveBoards()
+    main_test.go:205: PASS:	UpsertBoard(mock.argumentMatcher)
+    main_test.go:205: PASS:	GetTickets(mock.argumentMatcher)
+    main_test.go:205: PASS:	DeleteTicket(mock.argumentMatcher)
+    main_test.go:205: PASS:	UpsertTicket(mock.argumentMatcher)
+    main_test.go:205: PASS:	UpsertTicket(mock.argumentMatcher)
+	... [output truncated]
+BenchmarkCollect/it_should_collect_jira_related_data-12         	     674	   4930024 ns/op	13285131 B/op	    4224 allocs/op
+--- BENCH: BenchmarkCollect/it_should_collect_jira_related_data-12
+    main_test.go:205: PASS:	GetBoards()
+    main_test.go:205: PASS:	DeleteBoard(mock.argumentMatcher)
+    main_test.go:205: PASS:	UpsertBoard(mock.argumentMatcher)
+    main_test.go:205: PASS:	UpsertBoard(mock.argumentMatcher)
+    main_test.go:205: PASS:	GetActiveBoards()
+    main_test.go:205: PASS:	UpsertBoard(mock.argumentMatcher)
+    main_test.go:205: PASS:	GetTickets(mock.argumentMatcher)
+    main_test.go:205: PASS:	DeleteTicket(mock.argumentMatcher)
+    main_test.go:205: PASS:	UpsertTicket(mock.argumentMatcher)
+    main_test.go:205: PASS:	UpsertTicket(mock.argumentMatcher)
+	... [output truncated]
+PASS
+ok  	holocron/collector-jira-workflow/pkg/collector	16.355s
+PASS
+ok  	holocron/collector-jira-workflow/pkg/database	0.013s
+PASS
+ok  	holocron/collector-jira-workflow/pkg/httpClient	0.013s
+PASS
+ok  	holocron/collector-jira-workflow/pkg/utils	0.015s

pkg/collector/collectBoards.go

 package collector
 
 import (
-	"fmt"
 	"time"
 
-	"holocron/collector-jira-workflow/config"
 	"holocron/collector-jira-workflow/pkg/database"
 	"holocron/collector-jira-workflow/pkg/httpClient"
+	"holocron/collector-jira-workflow/pkg/utils"
 )
 
-func collectBoards(db database.DB) *[]database.BoardJSON {
-	currentTime := time.Now()
-	boards := httpClient.GetBoards()
-	for i, b := range *boards {
-		(*boards)[i].API_URL = fmt.Sprintf("%s/board/%d", config.API_URL, b.ID)
-		(*boards)[i].LastCollected = currentTime
+func saveBoards(boards *[]database.BoardJSON, db database.DB) {
+	utils.Logger.Info("Saving boards...")
+	for _, board := range *boards {
+		db.UpsertBoard(board)
 	}
-	pruneBoards(boards, db)
+}
+
+func setLastUpdated(activeBoards []database.CollectorTarget, time time.Time,
+	db database.DB) {
 
-	return boards
+	for _, board := range activeBoards {
+		currentBoard := board.ToBoardJSON()
+		currentBoard.LastCollected = time
+		db.UpsertBoard(currentBoard)
+	}
 }
 
-func pruneBoards(collectedBoards *[]database.BoardJSON, db database.DB) {
+func pruneBoards(collectedBoards *[]database.BoardJSON, db database.DB) (delCount int) {
+	utils.Logger.Info("Deleting boards...")
 	savedBoards := db.GetBoards()
 	var collectedHashTable map[uint]bool = make(map[uint]bool, len(*collectedBoards))
 	for _, board := range *collectedBoards {
@@ -30,6 +35,25 @@ func pruneBoards(collectedBoards *[]database.BoardJSON, db database.DB) {
 	for _, board := range savedBoards {
 		if _, boardInAPIResp := collectedHashTable[board.BoardId]; !boardInAPIResp {
 			db.DeleteBoard(board)
+			delCount++
 		}
 	}
+
+	return delCount
+}
+
+func collectBoards(db database.DB) {
+	utils.Logger.Info("Collecting boards...")
+	startTime := time.Now()
+	boards := httpClient.GetBoards()
+	delCount := pruneBoards(boards, db)
+	saveBoards(boards, db)
+
+	statsFormattedStr := "\n---Finished collecting boards---\n" +
+		"---Time Elapsed - %.0f seconds---\n" +
+		"%d\t-\tBoards Collected\n" +
+		"%d\t-\tBoards Removed\n"
+	utils.Logger.Infof(statsFormattedStr,
+		time.Since(startTime).Seconds(),
+		len(*boards), delCount)
 }

pkg/collector/collectBoards_test.go

 package collector
 
 import (
+	"bytes"
 	"net/http"
 	"net/http/httptest"
 	"testing"
@@ -8,10 +9,12 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
 
+	"holocron/collector-jira-workflow/pkg/database"
 	db "holocron/collector-jira-workflow/pkg/database"
+	"holocron/collector-jira-workflow/pkg/utils"
 )
 
-//nolint: funlen // This is just a test.
+//nolint: funlen, gocyclo // This is just a test.
 func TestCollectBoards(t *testing.T) {
 	var testCases = []struct {
 		AddAPIMocks        func(*mockAPI)
@@ -33,18 +36,22 @@ func TestCollectBoards(t *testing.T) {
 				mdb.T().Helper()
 
 				mdb.On("GetBoards").Return([]db.CollectorTarget{}).Once()
+
+				mdb.On("UpsertBoard", mock.MatchedBy(func(b database.BoardJSON) bool {
+					return b.ID == uint(1) && b.Name == "test1" && b.URL == "test.com/1"
+				})).Once()
 			},
 			AssertExpectations: func(mdb *mockDB, api *mockAPI, _ *httptest.Server) {
 				defer func() {
 					mdb.AssertExpectations(mdb.T())
 					api.AssertExpectations(api.T())
 				}()
-				result := *collectBoards(mdb)
-
-				assert.Equal(mdb.T(), 1, len(result))
-				assert.Equal(mdb.T(), uint(1), result[0].ID)
-				assert.Equal(mdb.T(), "test1", result[0].Name)
-				assert.Equal(mdb.T(), "test.com/1", result[0].URL)
+				// setting output recorder
+				buffer := bytes.Buffer{}
+				utils.Logger.InfoLogger.SetOutput(&buffer)
+				// testing
+				collectBoards(mdb)
+				assert.Contains(mdb.T(), buffer.String(), "1\t-\tBoards Collected")
 			},
 		},
 		{
@@ -63,24 +70,30 @@ func TestCollectBoards(t *testing.T) {
 				getMock("testdata/db/collector_targets/boards-to-delete.json",
 					mdb.T(), &boardsToDelete)
 				mdb.On("GetBoards").Return(boardsToDelete).Once()
+
 				for _, board := range boardsToDelete {
 					board := board
 					mdb.On("DeleteBoard", mock.MatchedBy(func(b db.CollectorTarget) bool {
 						return b.Name == board.Name
 					})).Once()
 				}
+
+				mdb.On("UpsertBoard", mock.MatchedBy(func(b database.BoardJSON) bool {
+					return b.ID == uint(5) && b.Name == "test" && b.URL == "test.com"
+				})).Once()
 			},
 			AssertExpectations: func(mdb *mockDB, api *mockAPI, _ *httptest.Server) {
 				defer func() {
 					mdb.AssertExpectations(mdb.T())
 					api.AssertExpectations(api.T())
 				}()
-				result := *collectBoards(mdb)
-
-				assert.Equal(mdb.T(), 1, len(result))
-				assert.Equal(mdb.T(), uint(5), result[0].ID)
-				assert.Equal(mdb.T(), "test", result[0].Name)
-				assert.Equal(mdb.T(), "test.com", result[0].URL)
+				// setting output recorder
+				buffer := bytes.Buffer{}
+				utils.Logger.InfoLogger.SetOutput(&buffer)
+				// testing
+				collectBoards(mdb)
+				assert.Contains(mdb.T(), buffer.String(), "1\t-\tBoards Collected")
+				assert.Contains(mdb.T(), buffer.String(), "1\t-\tBoards Removed")
 			},
 		},
 	}

pkg/collector/collectTickets.go

 package collector
 
 import (
+	"sync"
+	"time"
+
 	"holocron/collector-jira-workflow/pkg/database"
 	"holocron/collector-jira-workflow/pkg/httpClient"
+	"holocron/collector-jira-workflow/pkg/utils"
 )
 
-func collectTickets(
-	boards []database.CollectorTarget, db database.DB, ch chan *[]database.TicketJSON,
-) {
-
-	ticketChannels := make(chan *[]database.TicketJSON, len(boards))
-	for _, board := range boards {
-		go collectTicketsForBoard(board, db, ticketChannels)
-	}
-	result := &[]database.TicketJSON{}
-	for range boards {
-		*result = append(*result, *<-ticketChannels...)
-	}
-	ch <- result
-}
-
-func collectTicketsForBoard(
-	board database.CollectorTarget, db database.DB, ch chan *[]database.TicketJSON,
-) {
-
-	tickets := httpClient.GetTickets(board)
-	pruneTickets(tickets, board, db)
-	for i := range *tickets {
-		(*tickets)[i].BoardId = board.ID
-		(*tickets)[i].BuildStatusHistory()
+func saveTickets(tickets []database.TicketJSON, db database.DB) {
+	for _, ticket := range tickets {
+		db.UpsertTicket(ticket)
 	}
-	ch <- tickets
 }
 
 func pruneTickets(
 	collectedTickets *[]database.TicketJSON,
 	board database.CollectorTarget,
 	db database.DB,
-) {
+) (delCount int) {
 
 	savedTickets := db.GetTickets(board)
 	var collectedHashTable map[string]bool = make(map[string]bool, len(*collectedTickets))
@@ -47,6 +29,45 @@ func pruneTickets(
 	for _, ticket := range savedTickets {
 		if _, ticketInAPIResp := collectedHashTable[ticket.TicketID]; !ticketInAPIResp {
 			db.DeleteTicket(ticket)
+			delCount++
 		}
 	}
+
+	return delCount
+}
+
+func collectTickets(activeBoards []database.CollectorTarget, db database.DB) {
+	utils.Logger.Info("Collecting workflow data...")
+	var (
+		mtx       sync.Mutex
+		wg        sync.WaitGroup
+		delCount  = 0
+		saveCount = 0
+		startTime = time.Now()
+	)
+	wg.Add(len(activeBoards))
+	for _, board := range activeBoards {
+		go func(board database.CollectorTarget) {
+			// let the fetching happen concurrently
+			defer wg.Done()
+			ticketsForBoard := httpClient.GetTickets(board)
+			// but only one go routine can write to db, and count variables
+			// at a time
+			mtx.Lock()
+			defer mtx.Unlock()
+			deleted := pruneTickets(ticketsForBoard, board, db)
+			saveTickets(*ticketsForBoard, db)
+			saveCount += len(*ticketsForBoard)
+			delCount += deleted
+		}(board)
+	}
+	wg.Wait()
+
+	statsFormattedStr := "\n---Finished collecting tickets---\n" +
+		"---Time Elapsed - %.0f seconds---\n" +
+		"%d\t-\tTickets Collected\n" +
+		"%d\t-\tTickets Removed\n"
+	utils.Logger.Infof(statsFormattedStr,
+		time.Since(startTime).Seconds(),
+		saveCount, delCount)
 }

pkg/collector/collectTickets_test.go

 package collector
 
 import (
+	"bytes"
 	"net/http"
 	"net/http/httptest"
 	"testing"
@@ -9,9 +10,10 @@ import (
 	"github.com/stretchr/testify/mock"
 
 	db "holocron/collector-jira-workflow/pkg/database"
+	"holocron/collector-jira-workflow/pkg/utils"
 )
 
-//nolint: funlen // This is just a test.
+//nolint: funlen, gocyclo // This is just a test.
 func TestCollectTickets(t *testing.T) {
 	var testCases = []struct {
 		AddAPIMocks        func(*mockAPI)
@@ -39,6 +41,12 @@ func TestCollectTickets(t *testing.T) {
 				mdb.On("GetTickets", mock.MatchedBy(func(b db.CollectorTarget) bool {
 					return b.Name == activeBoards[0].Name
 				})).Return([]db.Ticket{})
+
+				mdb.On("UpsertTicket", mock.MatchedBy(func(t db.TicketJSON) bool {
+					return t.ID == "1" && t.Fields.Status.Name == "testStatus1" &&
+						t.Fields.Assignee.Email == "testEmail1" &&
+						t.Fields.Type.Name == "testType1"
+				})).Once()
 			},
 			//nolint:dupl // This is just a test.
 			AssertExpectations: func(mdb *mockDB, api *mockAPI, s *httptest.Server) {
@@ -46,19 +54,17 @@ func TestCollectTickets(t *testing.T) {
 					mdb.AssertExpectations(mdb.T())
 					api.AssertExpectations(api.T())
 				}()
+				// saving logs
+				buffer := bytes.Buffer{}
+				utils.Logger.InfoLogger.SetOutput(&buffer)
+				// testing
 				var boards []db.CollectorTarget
 				getMock("testdata/db/collector_targets/collector-targets.json", mdb.T(), &boards)
 				var activeBoards = []db.CollectorTarget{boards[0]}
 				activeBoards[0].API_URL = s.URL
-				resultChannel := make(chan *[]db.TicketJSON)
-				go collectTickets(activeBoards, mdb, resultChannel)
-				result := *<-resultChannel
+				collectTickets(activeBoards, mdb)
 
-				assert.Equal(mdb.T(), 1, len(result))
-				assert.Equal(mdb.T(), "1", result[0].ID)
-				assert.Equal(mdb.T(), "testStatus1", result[0].Fields.Status.Name)
-				assert.Equal(mdb.T(), "testEmail1", result[0].Fields.Assignee.Email)
-				assert.Equal(mdb.T(), "testType1", result[0].Fields.Type.Name)
+				assert.Contains(mdb.T(), buffer.String(), "1\t-\tTickets Collected")
 			},
 		},
 		{
@@ -83,6 +89,12 @@ func TestCollectTickets(t *testing.T) {
 					return b.Name == activeBoards[0].Name
 				})).Return(ticketsToDelete).Once()
 
+				mdb.On("UpsertTicket", mock.MatchedBy(func(t db.TicketJSON) bool {
+					return t.ID == "5" && t.Fields.Status.Name == "testStatus1" &&
+						t.Fields.Assignee.Email == "testEmail1" &&
+						t.Fields.Type.Name == "testType1"
+				})).Once()
+
 				mdb.On("DeleteTicket", mock.MatchedBy(func(t db.Ticket) bool {
 					return t.Assignee == ticketsToDelete[0].Assignee
 				})).Once()
@@ -93,20 +105,19 @@ func TestCollectTickets(t *testing.T) {
 					mdb.AssertExpectations(mdb.T())
 					api.AssertExpectations(api.T())
 				}()
+				// saving logs
+				buffer := bytes.Buffer{}
+				utils.Logger.InfoLogger.SetOutput(&buffer)
+				// testing
 				var boards []db.CollectorTarget
 				getMock("testdata/db/collector_targets/collector-targets.json",
 					mdb.T(), &boards)
 				var activeBoards = []db.CollectorTarget{boards[0]}
 				activeBoards[0].API_URL = s.URL
-				resultChannel := make(chan *[]db.TicketJSON)
-				go collectTickets(activeBoards, mdb, resultChannel)
-				result := *<-resultChannel
+				collectTickets(activeBoards, mdb)
 
-				assert.Equal(mdb.T(), 1, len(result))
-				assert.Equal(mdb.T(), "5", result[0].ID)
-				assert.Equal(mdb.T(), "testStatus1", result[0].Fields.Status.Name)
-				assert.Equal(mdb.T(), "testEmail1", result[0].Fields.Assignee.Email)
-				assert.Equal(mdb.T(), "testType1", result[0].Fields.Type.Name)
+				assert.Contains(mdb.T(), buffer.String(), "1\t-\tTickets Collected")
+				assert.Contains(mdb.T(), buffer.String(), "1\t-\tTickets Removed")
 			},
 		},
 	}

pkg/collector/main.go

@@ -4,59 +4,11 @@ import (
 	"time"
 
 	"holocron/collector-jira-workflow/pkg/database"
-	"holocron/collector-jira-workflow/pkg/utils"
 )
 
 func Collect(db database.DB) {
-	startTime := time.Now()
-
-	utils.Logger.Info("Collecting boards...")
-	boards := collectBoards(db)
-	utils.Logger.Info("Saving boards...")
-	saveBoards(boards, db)
+	collectBoards(db)
 	activeBoards := db.GetActiveBoards()
-
-	utils.Logger.Info("Collecting workflow data...")
-	ticketsChannel := make(chan *[]database.TicketJSON)
-
-	go collectTickets(activeBoards, db, ticketsChannel)
-
-	tickets := <-ticketsChannel
-
-	utils.Logger.Info("Saving workflow data...")
-	saveTickets(tickets, db)
-	utils.Logger.Info("Workflow data saved.")
-
-	// set LastCollected after collection is done
+	collectTickets(activeBoards, db)
 	setLastUpdated(activeBoards, time.Now(), db)
-
-	utils.Logger.Infof("--Finished Collecting---\n"+
-		"%d\t-\tBoards\n"+
-		"%d\t-\tTickets\n"+
-		"---Time Elapsed - %.0f seconds---",
-		len(*boards),
-		len(*tickets),
-		time.Since(startTime).Seconds(),
-	)
-}
-
-func saveTickets(tickets *[]database.TicketJSON, db database.DB) {
-	for _, ticket := range *tickets {
-		db.UpsertTicket(ticket)
-	}
-}
-
-func saveBoards(boards *[]database.BoardJSON, db database.DB) {
-	for _, board := range *boards {
-		db.UpsertBoard(board)
-	}
-}
-func setLastUpdated(activeBoards []database.CollectorTarget, time time.Time,
-	db database.DB) {
-
-	for _, board := range activeBoards {
-		currentBoard := board.ToBoardJSON()
-		currentBoard.LastCollected = time
-		db.UpsertBoard(currentBoard)
-	}
 }

pkg/httpClient/request.go

@@ -73,8 +73,17 @@ func getItemsInPage(url string, itemsPtr interface{}) error {
 }
 
 func GetBoards() *[]db.BoardJSON {
-	boards := &db.BoardsJSON{Boards: []db.BoardJSON{}}
 	const perPage = 50
+	var (
+		currentTime     = time.Now()
+		boards          = &db.BoardsJSON{Boards: []db.BoardJSON{}}
+		setCustomFields = func(boards []db.BoardJSON) {
+			for i, b := range boards {
+				boards[i].API_URL = fmt.Sprintf("%s/board/%d", config.API_URL, b.ID)
+				boards[i].LastCollected = currentTime
+			}
+		}
+	)
 	for startAt, hasNext := 0, true; hasNext; startAt += perPage {
 		var pageBoards db.BoardsJSON
 		if err := getItemsInPage(fmt.Sprintf("%s/board?startAt=%d", config.API_URL, startAt),
@@ -84,13 +93,22 @@ func GetBoards() *[]db.BoardJSON {
 		boards.Boards = append(boards.Boards, pageBoards.Boards...)
 		hasNext = !pageBoards.IsLast
 	}
+	setCustomFields(boards.Boards)
 
 	return &boards.Boards
 }
 
 func GetTickets(board db.CollectorTarget) *[]db.TicketJSON {
-	tickets := &db.TicketsJSON{Tickets: []db.TicketJSON{}}
 	const perPage = 100
+	var (
+		tickets         = &db.TicketsJSON{Tickets: []db.TicketJSON{}}
+		setCustomFields = func(tickets []db.TicketJSON) {
+			for i := range tickets {
+				tickets[i].BoardId = board.ID
+				tickets[i].BuildStatusHistory()
+			}
+		}
+	)
 	for startAt, next := 0, true; next; startAt += perPage {
 		var pageTickets db.TicketsJSON
 		if err := getItemsInPage(fmt.Sprintf(
@@ -103,6 +121,7 @@ func GetTickets(board db.CollectorTarget) *[]db.TicketJSON {
 		tickets.Tickets = append(tickets.Tickets, pageTickets.Tickets...)
 		next = pageTickets.Total > startAt+perPage
 	}
+	setCustomFields(tickets.Tickets)
 
 	return &tickets.Tickets
 }