Merge branch '658-follow-up-from-new-modules-tidy-and-modules-download-jobs' into 'main'

Optimise Ruby and Go based CI jobs

Closes #658

See merge request https://gitlab.com/gitlab-org/gitlab-shell/-/merge_requests/787



Merged-by: Patrick Bajao <ebajao@gitlab.com>
Approved-by: Patrick Bajao <ebajao@gitlab.com>
Approved-by: Oscar Tovar <otovar@gitlab.com>
Reviewed-by: Patrick Bajao <ebajao@gitlab.com>
Reviewed-by: Ash McKenzie <amckenzie@gitlab.com>
Reviewed-by: Oscar Tovar <otovar@gitlab.com>
Co-authored-by: Ash McKenzie <amckenzie@gitlab.com>

.gitignore

 *.log
 *.swp
+.DS_Store
+.GOPATH
 .bundle
 .bundle/
 .gitlab_shell_secret
@@ -19,4 +21,3 @@ hooks/*.d
 tags
 tmp/*
 vendor
-.DS_Store

.gitlab-ci.yml

@@ -7,12 +7,18 @@ include:
     file:
       - '/ci/danger-review.yml'
 
+stages:
+  - prepare
+  - test
+
 variables:
   DOCKER_VERSION: "20.10.15"
   BUNDLE_FROZEN: "true"
   GO_VERSION: "golang-1.20"
+  GOPATH: $CI_PROJECT_DIR/.GOPATH
   DEBIAN_VERSION: "bullseye"
   RUBY_VERSION: "ruby-3.2"
+  BUNDLE_PATH: vendor/ruby
 
 workflow:
   rules: &workflow_rules
@@ -36,12 +42,56 @@ default:
     # See https://gitlab.com/gitlab-com/www-gitlab-com/-/issues/7019 for tag descriptions
     - gitlab-org-docker
 
-.test:
+
+.cached-job:
+  cache:
+    - key:
+        prefix: $GO_VERSION-cache
+        files:
+          - go.mod
+          - go.sum
+      paths:
+        - .GOPATH/pkg/mod/
+    - key:
+        prefix: $RUBY_VERSION-cache
+        files:
+          - Gemfile.lock
+      paths:
+        - ${BUNDLE_PATH}
+
+.go-matrix-job:
+  parallel:
+    matrix:
+      - GO_VERSION: ["golang-1.18", "golang-1.19", "golang-1.20"]
+
+################################################################################
+# Prepare jobs
+################################################################################
+
+bundle:install:
+  stage: prepare
+  extends: .cached-job
+  script:
+    - bundle install --jobs $(nproc)
+
+modules:download:
+  stage: prepare
+  extends:
+    - .cached-job
+    - .go-matrix-job
+  script:
+    - go mod download
+
+################################################################################
+# Test jobs
+################################################################################
+
+.test-job:
+  needs: ['bundle:install', 'modules:download']
   variables:
     GITALY_CONNECTION_INFO: '{"address":"tcp://gitaly:8075", "storage":"default"}'
   before_script:
     # Set up the environment to run integration tests (still written in Ruby)
-    - bundle install
     - make build
     - cp config.yml.example config.yml
     - go version
@@ -53,10 +103,10 @@ default:
       alias: gitaly
 
 tests:
-  extends: .test
-  parallel:
-    matrix:
-      - GO_VERSION: ["golang-1.18", "golang-1.19", "golang-1.20"]
+  extends:
+    - .cached-job
+    - .go-matrix-job
+    - .test-job
   script:
     - make verify test
   after_script:
@@ -64,7 +114,10 @@ tests:
   coverage: '/\d+.\d+%/'
 
 race:
-  extends: .test
+  extends:
+    - .cached-job
+    - .go-matrix-job
+    - .test-job
   script:
     - make test_golang_race
 
@@ -129,19 +182,3 @@ modules:tidy:
   script:
     - go mod tidy
     - git diff --exit-code go.mod go.sum
-
-modules:download:
-  parallel:
-    matrix:
-      - GO_VERSION: ["golang-1.18", "golang-1.19", "golang-1.20"]
-  script:
-    - go mod download
-  cache:
-    key:
-      prefix: $GO_VERSION
-      files:
-        - go.mod
-        - go.sum
-    paths:
-      - .GOPATH/pkg/mod/
-    policy: push

Makefile

@@ -2,11 +2,13 @@
 
 FIPS_MODE ?= 0
 OS := $(shell uname)
-GO_SOURCES := $(shell find . -name '*.go')
+GO_SOURCES := $(shell git ls-files \*.go)
 VERSION_STRING := $(shell git describe --match v* 2>/dev/null || awk '$$0="v"$$0' VERSION 2>/dev/null || echo unknown)
 BUILD_TIME := $(shell date -u +%Y%m%d.%H%M%S)
 BUILD_TAGS := tracer_static tracer_static_jaeger continuous_profiler_stackdriver
 
+export GOFLAGS := -mod=readonly
+
 ifeq (${FIPS_MODE}, 1)
     # Go 1.19 now requires GOEXPERIMENT=boringcrypto for FIPS compilation.
     # See https://github.com/golang/go/issues/51940 for more details.