Resolve the requested changes

- Add the logic to retry the test case that is failed.
- Accpet the number of how many times we want to retry
- Run the last retry in debug

.github/workflows/build-test.yml

@@ -44,6 +44,7 @@ jobs:
       - name: Integration Tests
         env:
           GH_ACTION: true
+          GH_ACTION_RETRIES: "3"
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
         run: |
           chmod +x run.sh

v2/cmd/integration-test/integration-test.go

@@ -4,6 +4,7 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"strconv"
 	"strings"
 
 	"github.com/logrusorgru/aurora"
@@ -12,9 +13,10 @@ import (
 )
 
 var (
-	debug        = os.Getenv("DEBUG") == "true"
-	githubAction = os.Getenv("GH_ACTION") == "true"
-	customTests  = os.Getenv("TESTS")
+	debug               = os.Getenv("DEBUG") == "true"
+	githubAction        = os.Getenv("GH_ACTION") == "true"
+	githubActionRetries = os.Getenv("GH_ACTION_RETRIES")
+	customTests         = os.Getenv("TESTS")
 
 	success = aurora.Green("[✓]").String()
 	failed  = aurora.Red("[✘]").String()
@@ -61,16 +63,13 @@ func main() {
 		os.Exit(1)
 	}
 
-	failedTestTemplatePaths := runTests(toMap(toSlice(customTests)))
+	failedTestCases := runTests(toMap(toSlice(customTests)))
 
-	if len(failedTestTemplatePaths) > 0 {
-		if githubAction {
-			debug = true
-			fmt.Println("::group::Failed integration tests in debug mode")
-			_ = runTests(failedTestTemplatePaths)
-			fmt.Println("::endgroup::")
+	if githubAction {
+		if retryOnFailure(failedTestCases) {
+			os.Exit(1)
 		}
-
+	} else if len(failedTestCases) > 0 {
 		os.Exit(1)
 	}
 }
@@ -86,8 +85,8 @@ func debugTests() {
 	}
 }
 
-func runTests(customTemplatePaths map[string]struct{}) map[string]struct{} {
-	failedTestTemplatePaths := map[string]struct{}{}
+func runTests(customTemplatePaths map[string]struct{}) map[string]testutils.TestCase {
+	failedTestCase := map[string]testutils.TestCase{}
 
 	for proto, testCases := range protocolTests {
 		if len(customTemplatePaths) == 0 {
@@ -96,14 +95,14 @@ func runTests(customTemplatePaths map[string]struct{}) map[string]struct{} {
 
 		for templatePath, testCase := range testCases {
 			if len(customTemplatePaths) == 0 || contains(customTemplatePaths, templatePath) {
-				if failedTemplatePath, err := execute(testCase, templatePath); err != nil {
-					failedTestTemplatePaths[failedTemplatePath] = struct{}{}
+				if _, err := execute(testCase, templatePath); err != nil {
+					failedTestCase[templatePath] = testCase
 				}
 			}
 		}
 	}
 
-	return failedTestTemplatePaths
+	return failedTestCase
 }
 
 func execute(testCase testutils.TestCase, templatePath string) (string, error) {
@@ -145,3 +144,29 @@ func contains(input map[string]struct{}, value string) bool {
 	_, ok := input[value]
 	return ok
 }
+
+// retries failed test cases and run the last retry in debug mode
+// if any case failed in retry retrun true
+func retryOnFailure(failedTestCases map[string]testutils.TestCase) bool {
+	retries, _ := strconv.Atoi(githubActionRetries)
+	var failed bool
+	for templatePath, testCase := range failedTestCases {
+		var err error
+		debug = false
+		fmt.Println("::group::Failed integration test", templatePath)
+		for i := 0; i < retries; i++ {
+			if (retries - i) == 1 {
+				debug = true
+			}
+			_, err = execute(testCase, templatePath)
+			if err == nil {
+				break
+			}
+		}
+		fmt.Println("::endgroup::")
+		if err != nil {
+			failed = true
+		}
+	}
+	return failed
+}