Merge branch '762_replace_geo_pull' into 'main'

Geo: Replace Git over HTTP calls with Workhorse HTTP endpoint for SSH pull

See merge request https://gitlab.com/gitlab-org/gitlab-shell/-/merge_requests/1081



Merged-by: Ash McKenzie <amckenzie@gitlab.com>
Approved-by: Ash McKenzie <amckenzie@gitlab.com>
Reviewed-by: Vasilii Iakliushin <viakliushin@gitlab.com>
Reviewed-by: Ash McKenzie <amckenzie@gitlab.com>
Co-authored-by: Vasilii Iakliushin <viakliushin@gitlab.com>

internal/command/githttp/pull.go

@@ -6,11 +6,13 @@ import (
 	"fmt"
 	"io"
 
+	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/commandargs"
 	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/readwriter"
 	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/config"
 	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/gitlabnet/accessverifier"
 	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/gitlabnet/git"
 	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/pktline"
+	"gitlab.com/gitlab-org/labkit/log"
 )
 
 const pullService = "git-upload-pack"
@@ -20,6 +22,7 @@ var uploadPackHttpPrefix = []byte("001e# service=git-upload-pack\n0000")
 type PullCommand struct {
 	Config     *config.Config
 	ReadWriter *readwriter.ReadWriter
+	Args       *commandargs.Shell
 	Response   *accessverifier.Response
 }
 
@@ -37,6 +40,14 @@ func (c *PullCommand) Execute(ctx context.Context) error {
 	data := c.Response.Payload.Data
 	client := &git.Client{URL: data.PrimaryRepo, Headers: data.RequestHeaders}
 
+	// For Git over SSH routing
+	if data.GeoProxyFetchSSHDirectToPrimary {
+		log.ContextLogger(ctx).Info("Using Git over SSH upload pack")
+
+		client.Headers["Git-Protocol"] = c.Args.Env.GitProtocolVersion
+		return c.requestSSHUploadPack(ctx, client)
+	}
+
 	if err := c.requestInfoRefs(ctx, client); err != nil {
 		return err
 	}
@@ -64,6 +75,18 @@ func (c *PullCommand) requestInfoRefs(ctx context.Context, client *git.Client) e
 	return err
 }
 
+func (c *PullCommand) requestSSHUploadPack(ctx context.Context, client *git.Client) error {
+	response, err := client.SSHUploadPack(ctx, io.NopCloser(c.ReadWriter.In))
+	if err != nil {
+		return err
+	}
+	defer response.Body.Close()
+
+	_, err = io.Copy(c.ReadWriter.Out, response.Body)
+
+	return err
+}
+
 func (c *PullCommand) requestUploadPack(ctx context.Context, client *git.Client, geoProxyFetchDirectToPrimaryWithOptions bool) error {
 	pipeReader, pipeWriter := io.Pipe()
 	go c.readFromStdin(pipeWriter, geoProxyFetchDirectToPrimaryWithOptions)

internal/command/githttp/pull_test.go

@@ -11,9 +11,11 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"gitlab.com/gitlab-org/gitlab-shell/v14/client/testserver"
+	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/commandargs"
 	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/readwriter"
 	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/config"
 	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/gitlabnet/accessverifier"
+	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/sshenv"
 )
 
 var cloneResponse = `0090want 11d731b83788cd556abea7b465c6bee52d89923c multi_ack_detailed side-band-64k thin-pack ofs-delta deepen-since deepen-not agent=git/2.41.0
@@ -58,6 +60,35 @@ func TestPullExecuteWithDepth(t *testing.T) {
 	require.Equal(t, infoRefsWithoutPrefix, output.String())
 }
 
+func TestPullExecuteWithSSHUploadPack(t *testing.T) {
+	url := setupSSHPull(t, http.StatusOK)
+	output := &bytes.Buffer{}
+	input := strings.NewReader(cloneResponse + "0009done\n")
+
+	cmd := &PullCommand{
+		Config:     &config.Config{GitlabUrl: url},
+		ReadWriter: &readwriter.ReadWriter{Out: output, In: input},
+		Response: &accessverifier.Response{
+			Payload: accessverifier.CustomPayload{
+				Data: accessverifier.CustomPayloadData{
+					PrimaryRepo:                             url,
+					GeoProxyFetchDirectToPrimaryWithOptions: true,
+					GeoProxyFetchSSHDirectToPrimary:         true,
+					RequestHeaders:                          map[string]string{"Authorization": "token"},
+				},
+			},
+		},
+		Args: &commandargs.Shell{
+			Env: sshenv.Env{
+				GitProtocolVersion: "version=2",
+			},
+		},
+	}
+
+	require.NoError(t, cmd.Execute(context.Background()))
+	require.Equal(t, "upload-pack-response", output.String())
+}
+
 func TestPullExecuteWithFailedInfoRefs(t *testing.T) {
 	testCases := []struct {
 		desc            string
@@ -157,3 +188,25 @@ func setupPull(t *testing.T, uploadPackStatusCode int) string {
 
 	return testserver.StartHttpServer(t, requests)
 }
+
+func setupSSHPull(t *testing.T, uploadPackStatusCode int) string {
+	requests := []testserver.TestRequestHandler{
+		{
+			Path: "/ssh-upload-pack",
+			Handler: func(w http.ResponseWriter, r *http.Request) {
+				body, err := io.ReadAll(r.Body)
+				require.NoError(t, err)
+				defer r.Body.Close()
+
+				require.True(t, strings.HasSuffix(string(body), "0009done\n"))
+				require.Equal(t, "version=2", r.Header.Get("Git-Protocol"))
+				require.Equal(t, "token", r.Header.Get("Authorization"))
+
+				w.Write([]byte("upload-pack-response"))
+				w.WriteHeader(uploadPackStatusCode)
+			},
+		},
+	}
+
+	return testserver.StartHttpServer(t, requests)
+}

internal/command/uploadpack/uploadpack.go

@@ -51,6 +51,7 @@ func (c *Command) Execute(ctx context.Context) (context.Context, error) {
 			cmd := githttp.PullCommand{
 				Config:     c.Config,
 				ReadWriter: c.ReadWriter,
+				Args:       c.Args,
 				Response:   response,
 			}
 

internal/gitlabnet/accessverifier/client.go

@@ -56,6 +56,7 @@ type CustomPayloadData struct {
 	GeoProxyDirectToPrimary                 bool              `json:"geo_proxy_direct_to_primary"`
 	GeoProxyFetchDirectToPrimary            bool              `json:"geo_proxy_fetch_direct_to_primary"`
 	GeoProxyFetchDirectToPrimaryWithOptions bool              `json:"geo_proxy_fetch_direct_to_primary_with_options"`
+	GeoProxyFetchSSHDirectToPrimary         bool              `json:"geo_proxy_fetch_ssh_direct_to_primary"`
 }
 
 // CustomPayload represents a custom payload

internal/gitlabnet/git/client.go

@@ -14,7 +14,10 @@ var httpClient = &http.Client{
 	Transport: client.NewTransport(client.DefaultTransport()),
 }
 
-const repoUnavailableErrMsg = "Remote repository is unavailable"
+const (
+	repoUnavailableErrMsg = "Remote repository is unavailable"
+	sshUploadPackPath     = "/ssh-upload-pack"
+)
 
 // Client represents a client for interacting with Git repositories.
 type Client struct {
@@ -56,6 +59,16 @@ func (c *Client) UploadPack(ctx context.Context, body io.Reader) (*http.Response
 	return c.do(request)
 }
 
+// SSHUploadPack sends a SSH Git fetch request to the server.
+func (c *Client) SSHUploadPack(ctx context.Context, body io.Reader) (*http.Response, error) {
+	request, err := http.NewRequestWithContext(ctx, http.MethodPost, c.URL+sshUploadPackPath, body)
+	if err != nil {
+		return nil, err
+	}
+
+	return c.do(request)
+}
+
 func (c *Client) do(request *http.Request) (*http.Response, error) {
 	for k, v := range c.Headers {
 		request.Header.Add(k, v)

internal/gitlabnet/git/client_test.go

@@ -64,6 +64,20 @@ func TestUploadPack(t *testing.T) {
 	require.Equal(t, "git-upload-pack: content", string(body))
 }
 
+func TestSSHUploadPack(t *testing.T) {
+	client := setup(t)
+
+	refsBody := "0032want 0a53e9ddeaddad63ad106860237bbf53411d11a7\n"
+	response, err := client.SSHUploadPack(context.Background(), bytes.NewReader([]byte(refsBody)))
+	require.NoError(t, err)
+	defer response.Body.Close()
+
+	body, err := io.ReadAll(response.Body)
+	require.NoError(t, err)
+
+	require.Equal(t, "ssh-upload-pack: content", string(body))
+}
+
 func TestFailedHTTPRequest(t *testing.T) {
 	requests := []testserver.TestRequestHandler{
 		{
@@ -166,6 +180,19 @@ func setup(t *testing.T) *Client {
 				w.Write([]byte("git-upload-pack: content"))
 			},
 		},
+		{
+			Path: sshUploadPackPath,
+			Handler: func(w http.ResponseWriter, r *http.Request) {
+				require.Equal(t, customHeaders["Authorization"], r.Header.Get("Authorization"))
+				require.Equal(t, customHeaders["Header-One"], r.Header.Get("Header-One"))
+
+				_, err := io.ReadAll(r.Body)
+				require.NoError(t, err)
+				defer r.Body.Close()
+
+				w.Write([]byte("ssh-upload-pack: content"))
+			},
+		},
 	}
 
 	client := &Client{