add Singleton job rescheduled metric (#763)

executor.go

@@ -196,7 +196,7 @@ func (e *executor) start() {
 							default:
 								// runner is busy, reschedule the work for later
 								// which means we just skip it here and do nothing
-								// TODO when metrics are added, this should increment a rescheduled metric
+								e.incrementJobCounter(*j, SingletonRescheduled)
 								e.sendOutForRescheduling(&jIn)
 							}
 						} else {
@@ -397,9 +397,7 @@ func (e *executor) runJob(j internalJob, jIn jobIn) {
 
 	startTime := time.Now()
 	err := e.callJobWithRecover(j)
-	if e.monitor != nil {
-		e.monitor.RecordJobTiming(startTime, time.Now(), j.id, j.name, j.tags)
-	}
+	e.recordJobTiming(startTime, time.Now(), j)
 	if err != nil {
 		_ = callJobFuncWithParams(j.afterJobRunsWithError, j.id, j.name, err)
 		e.incrementJobCounter(j, Fail)
@@ -422,6 +420,12 @@ func (e *executor) callJobWithRecover(j internalJob) (err error) {
 	return callJobFuncWithParams(j.function, j.parameters...)
 }
 
+func (e *executor) recordJobTiming(start time.Time, end time.Time, j internalJob) {
+	if e.monitor != nil {
+		e.monitor.RecordJobTiming(start, end, j.id, j.name, j.tags)
+	}
+}
+
 func (e *executor) incrementJobCounter(j internalJob, status JobStatus) {
 	if e.monitor != nil {
 		e.monitor.IncrementJob(j.id, j.name, j.tags, status)

monitor.go

@@ -11,9 +11,10 @@ type JobStatus string
 
 // The different statuses of job that can be used.
 const (
-	Fail    JobStatus = "fail"
-	Success JobStatus = "success"
-	Skip    JobStatus = "skip"
+	Fail                 JobStatus = "fail"
+	Success              JobStatus = "success"
+	Skip                 JobStatus = "skip"
+	SingletonRescheduled JobStatus = "singleton_rescheduled"
 )
 
 // Monitor represents the interface to collect jobs metrics.

scheduler.go

@@ -311,7 +311,7 @@ func (s *scheduler) selectRemoveJob(id uuid.UUID) {
 }
 
 // Jobs coming back from the executor to the scheduler that
-// need to evaluated for rescheduling.
+// need to be evaluated for rescheduling.
 func (s *scheduler) selectExecJobsOutForRescheduling(id uuid.UUID) {
 	select {
 	case <-s.shutdownCtx.Done():