Merge branch 'lastCollectedNew' into 'master'

Last collected new

See merge request platform-one/big-bang/apps/sandbox/holocron/collector-jira-workflow!13

Dockerfile.dev

@@ -7,4 +7,4 @@ RUN go mod download && go mod verify
 
 COPY . .
 
-CMD ["go", "run", "./cmd/main.go"]
\ No newline at end of file
+CMD ["go", "run", "./cmd/main.go"]

pkg/collector/collectBoards.go

@@ -2,6 +2,7 @@ package collector
 
 import (
 	"fmt"
+	"time"
 
 	"holocron/collector-jira-workflow/config"
 	"holocron/collector-jira-workflow/pkg/database"
@@ -9,9 +10,11 @@ import (
 )
 
 func collectBoards(db database.DB) *[]database.BoardJSON {
+	currentTime := time.Now()
 	boards := httpClient.GetBoards()
 	for i, b := range *boards {
 		(*boards)[i].API_URL = fmt.Sprintf("%s/board/%d", config.API_URL, b.ID)
+		(*boards)[i].LastCollected = currentTime
 	}
 	pruneBoards(boards, db)
 

pkg/collector/main.go

@@ -27,6 +27,9 @@ func Collect(db database.DB) {
 	saveTickets(tickets, db)
 	utils.Logger.Info("Workflow data saved.")
 
+	// set LastCollected after collection is done
+	setLastUpdated(activeBoards, time.Now(), db)
+
 	utils.Logger.Infof("--Finished Collecting---\n"+
 		"%d\t-\tBoards\n"+
 		"%d\t-\tTickets\n"+
@@ -48,3 +51,12 @@ func saveBoards(boards *[]database.BoardJSON, db database.DB) {
 		db.UpsertBoard(board)
 	}
 }
+func setLastUpdated(activeBoards []database.CollectorTarget, time time.Time,
+	db database.DB) {
+
+	for _, board := range activeBoards {
+		currentBoard := board.ToBoardJSON()
+		currentBoard.LastCollected = time
+		db.UpsertBoard(currentBoard)
+	}
+}

pkg/collector/main_test.go

@@ -58,6 +58,14 @@ func TestCollect(t *testing.T) {
 				var activeBoards = []db.CollectorTarget{boards[0]}
 				activeBoards[0].API_URL = s.URL
 				mdb.On("GetActiveBoards").Return(activeBoards).Once()
+
+				for _, board := range activeBoards {
+					func(board db.BoardJSON) {
+						mdb.On("UpsertBoard", mock.MatchedBy(func(b db.BoardJSON) bool {
+							return board.Name == b.Name
+						})).Once()
+					}(board.ToBoardJSON())
+				}
 			},
 			AddTicketMocks: func(mdb *mockDB, _ *httptest.Server) {
 				mdb.T().Helper()
@@ -157,6 +165,13 @@ func BenchmarkCollect(b *testing.B) {
 				var activeBoards = []db.CollectorTarget{boards[0]}
 				activeBoards[0].API_URL = s.URL
 				mdb.On("GetActiveBoards").Return(activeBoards)
+				for _, board := range activeBoards {
+					func(board db.BoardJSON) {
+						mdb.On("UpsertBoard", mock.MatchedBy(func(b db.BoardJSON) bool {
+							return board.Name == b.Name
+						}))
+					}(board.ToBoardJSON())
+				}
 			},
 			AddTicketMocks: func(mdb *mockDB, s *httptest.Server) {
 				var (

pkg/database/models.go

@@ -13,10 +13,11 @@ type Collector struct {
 }
 
 type BoardJSON struct {
-	API_URL string ``
-	Name    string `json:"name"`
-	URL     string `json:"self"`
-	ID      uint   `json:"id"`
+	LastCollected time.Time
+	API_URL       string ``
+	Name          string `json:"name"`
+	URL           string `json:"self"`
+	ID            uint   `json:"id"`
 }
 
 type BoardsJSON struct {
@@ -25,17 +26,18 @@ type BoardsJSON struct {
 }
 
 type CollectorTarget struct {
-	CreatedAt   time.Time
-	UpdatedAt   time.Time
-	TeamID      *uint
-	API_URL     string
-	URL         string
-	Name        string
-	Tickets     []*Ticket `gorm:"foreignKey:BoardID;references:ID;"`
-	IsDeleted   bool
-	CollectorId uint
-	BoardId     uint
-	ID          uint
+	CreatedAt     time.Time
+	UpdatedAt     time.Time
+	LastCollected time.Time
+	TeamID        *uint
+	API_URL       string
+	URL           string
+	Name          string
+	Tickets       []*Ticket `gorm:"foreignKey:BoardID;references:ID;"`
+	IsDeleted     bool
+	CollectorId   uint
+	BoardId       uint
+	ID            uint
 }
 
 type TicketSubTaskJSON struct {
@@ -156,3 +158,13 @@ func (t *TicketJSON) BuildStatusHistory() {
 		[]TicketStatusHistoryJSON{createdHistory}, existingHistory...,
 	)
 }
+
+func (c *CollectorTarget) ToBoardJSON() BoardJSON {
+	return BoardJSON{
+		LastCollected: c.LastCollected,
+		API_URL:       c.API_URL,
+		Name:          c.Name,
+		URL:           c.URL,
+		ID:            c.BoardId,
+	}
+}

pkg/database/queryBoards.go

@@ -39,5 +39,6 @@ func (d *Database) UpsertBoard(board BoardJSON) {
 	ticketBoard.URL = board.URL
 	ticketBoard.Name = board.Name
 	ticketBoard.API_URL = board.API_URL
+	ticketBoard.LastCollected = board.LastCollected
 	d.DB.Clauses(clause.OnConflict{UpdateAll: true}).Create(&ticketBoard)
 }

pkg/database/queryBoards_test.go

@@ -51,6 +51,7 @@ func (s *boardTestSuite) TestUpsertBoard() {
 		mock.ExpectQuery(
 			regexp.QuoteMeta(`INSERT INTO "collector_targets"`)).
 			WithArgs(
+				AnyTime{},
 				AnyTime{},
 				AnyTime{},
 				nil,
@@ -65,10 +66,11 @@ func (s *boardTestSuite) TestUpsertBoard() {
 		mock.ExpectCommit()
 
 		db.UpsertBoard(BoardJSON{
-			ID:      uint(1),
-			Name:    "test",
-			URL:     "test.com",
-			API_URL: "test.com",
+			LastCollected: time.Now(),
+			ID:            uint(1),
+			Name:          "test",
+			URL:           "test.com",
+			API_URL:       "test.com",
 		})
 
 		err := mock.ExpectationsWereMet()
@@ -113,9 +115,7 @@ func (s *boardTestSuite) TestGetActiveBoards() {
 func (s *boardTestSuite) TestDeleteBoard() {
 	s.Run("it should delete a board if TeamID is nil", func() {
 		db, mock := s.m.DB, s.m.Mock
-		board := CollectorTarget{
-			ID: uint(1),
-		}
+		board := sampleBoardNoTeamID()
 		mock.MatchExpectationsInOrder(false)
 		mock.ExpectBegin()
 		mock.ExpectExec(
@@ -131,16 +131,13 @@ func (s *boardTestSuite) TestDeleteBoard() {
 	})
 	s.Run("it should mark a board as deleted if TeamID is not nil", func() {
 		db, mock := s.m.DB, s.m.Mock
-		teamID := uint(1)
-		board := CollectorTarget{
-			ID:     uint(1),
-			TeamID: &teamID,
-		}
+		board := sampleBoard()
 		mock.MatchExpectationsInOrder(false)
 		mock.ExpectBegin()
 		mock.ExpectExec(
 			regexp.QuoteMeta(`UPDATE "collector_targets" SET`)).
 			WithArgs(
+				AnyTime{},
 				AnyTime{},
 				AnyTime{},
 				1,

pkg/database/setup_test.go

@@ -102,3 +102,16 @@ func (c *mockDBConfig) tearnDownMockDB() error {
 		return fmt.Errorf("PANIC, TEST SUITE HAS FAILED TO CLEAN UP")
 	}
 }
+func sampleBoard() CollectorTarget {
+	teamID := uint(1)
+
+	return CollectorTarget{
+		ID:     uint(1),
+		TeamID: &teamID,
+	}
+}
+func sampleBoardNoTeamID() CollectorTarget {
+	return CollectorTarget{
+		ID: uint(1),
+	}
+}