Merge branch '656-add-pull-githttp-command' into 'main'

Add PullCommand to githttp package

See merge request https://gitlab.com/gitlab-org/gitlab-shell/-/merge_requests/836



Merged-by: Igor Drozdov <idrozdov@gitlab.com>
Approved-by: Igor Drozdov <idrozdov@gitlab.com>
Reviewed-by: Igor Drozdov <idrozdov@gitlab.com>
Reviewed-by: Ash McKenzie <amckenzie@gitlab.com>
Reviewed-by: Patrick Cyiza <jpcyiza@gitlab.com>
Co-authored-by: Patrick Cyiza <jpcyiza@gitlab.com>

internal/command/githttp/pull.go

+package githttp
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/readwriter"
+	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/config"
+	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/gitlabnet/accessverifier"
+	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/gitlabnet/git"
+	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/pktline"
+	"io"
+)
+
+const pullService = "git-upload-pack"
+
+var uploadPackHttpPrefix = []byte("001e# service=git-upload-pack\n0000")
+
+type PullCommand struct {
+	Config     *config.Config
+	ReadWriter *readwriter.ReadWriter
+	Response   *accessverifier.Response
+}
+
+// See Uploading Data > HTTP(S) section at:
+// https://git-scm.com/book/en/v2/Git-Internals-Transfer-Protocols
+//
+// 1. Perform /info/refs?service=git-upload-pack request
+// 2. Remove the header to make it consumable by SSH protocol
+// 3. Send the result to the user via SSH (writeToStdout)
+// 4. Read the send-pack data provided by user via SSH (stdinReader)
+// 5. Perform /git-upload-pack request and send this data
+// 6. Return the output to the user
+
+func (c *PullCommand) Execute(ctx context.Context) error {
+	data := c.Response.Payload.Data
+	client := &git.Client{Url: data.PrimaryRepo, Headers: data.RequestHeaders}
+
+	if err := c.requestInfoRefs(ctx, client); err != nil {
+		return err
+	}
+
+	return c.requestUploadPack(ctx, client)
+}
+
+func (c *PullCommand) requestInfoRefs(ctx context.Context, client *git.Client) error {
+	response, err := client.InfoRefs(ctx, pullService)
+	if err != nil {
+		return err
+	}
+	defer response.Body.Close()
+
+	// Read the first bytes that contain 001e# service=git-upload-pack\n0000 string
+	// to convert HTTP(S) Git response to the one expected by SSH
+	p := make([]byte, len(uploadPackHttpPrefix))
+	_, err = response.Body.Read(p)
+	if err != nil || !bytes.Equal(p, uploadPackHttpPrefix) {
+		return fmt.Errorf("Unexpected git-upload-pack response")
+	}
+
+	_, err = io.Copy(c.ReadWriter.Out, response.Body)
+
+	return err
+}
+
+func (c *PullCommand) requestUploadPack(ctx context.Context, client *git.Client) error {
+	pipeReader, pipeWriter := io.Pipe()
+	go c.readFromStdin(pipeWriter)
+
+	response, err := client.UploadPack(ctx, pipeReader)
+	if err != nil {
+		return err
+	}
+	defer response.Body.Close()
+
+	_, err = io.Copy(c.ReadWriter.Out, response.Body)
+
+	return err
+}
+
+func (c *PullCommand) readFromStdin(pw *io.PipeWriter) {
+	scanner := pktline.NewScanner(c.ReadWriter.In)
+
+	for scanner.Scan() {
+		line := scanner.Bytes()
+
+		if pktline.IsDone(line) {
+			pw.Write(line)
+
+			break
+		}
+
+		pw.Write(line)
+	}
+
+	pw.Close()
+}

internal/command/githttp/pull_test.go

+package githttp
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"gitlab.com/gitlab-org/gitlab-shell/v14/client/testserver"
+	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/readwriter"
+	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/config"
+	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/gitlabnet/accessverifier"
+)
+
+var cloneResponse = `0090want 11d731b83788cd556abea7b465c6bee52d89923c multi_ack_detailed side-band-64k thin-pack ofs-delta deepen-since deepen-not agent=git/2.41.0
+0032want e56497bb5f03a90a51293fc6d516788730953899
+00000009done
+`
+
+func TestPullExecute(t *testing.T) {
+	url := setupPull(t, http.StatusOK)
+	output := &bytes.Buffer{}
+	input := strings.NewReader(cloneResponse)
+
+	cmd := &PullCommand{
+		Config:     &config.Config{GitlabUrl: url},
+		ReadWriter: &readwriter.ReadWriter{Out: output, In: input},
+		Response: &accessverifier.Response{
+			Payload: accessverifier.CustomPayload{
+				Data: accessverifier.CustomPayloadData{PrimaryRepo: url},
+			},
+		},
+	}
+
+	require.NoError(t, cmd.Execute(context.Background()))
+	require.Equal(t, infoRefsWithoutPrefix, output.String())
+}
+
+func TestPullExecuteWithFailedInfoRefs(t *testing.T) {
+	testCases := []struct {
+		desc            string
+		statusCode      int
+		responseContent string
+		expectedErr     string
+	}{
+		{
+			desc:        "request failed",
+			statusCode:  http.StatusForbidden,
+			expectedErr: "Remote repository is unavailable",
+		}, {
+			desc:            "unexpected response",
+			statusCode:      http.StatusOK,
+			responseContent: "unexpected response",
+			expectedErr:     "Unexpected git-upload-pack response",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			requests := []testserver.TestRequestHandler{
+				{
+					Path: "/info/refs",
+					Handler: func(w http.ResponseWriter, r *http.Request) {
+						require.Equal(t, "git-upload-pack", r.URL.Query().Get("service"))
+
+						w.WriteHeader(tc.statusCode)
+						w.Write([]byte(tc.responseContent))
+					},
+				},
+			}
+
+			url := testserver.StartHttpServer(t, requests)
+
+			cmd := &PullCommand{
+				Config: &config.Config{GitlabUrl: url},
+				Response: &accessverifier.Response{
+					Payload: accessverifier.CustomPayload{
+						Data: accessverifier.CustomPayloadData{PrimaryRepo: url},
+					},
+				},
+			}
+
+			err := cmd.Execute(context.Background())
+			require.Error(t, err)
+			require.Equal(t, tc.expectedErr, err.Error())
+		})
+	}
+}
+
+func TestExecuteWithFailedUploadPack(t *testing.T) {
+	url := setupPull(t, http.StatusForbidden)
+	output := &bytes.Buffer{}
+	input := strings.NewReader(cloneResponse)
+
+	cmd := &PullCommand{
+		Config:     &config.Config{GitlabUrl: url},
+		ReadWriter: &readwriter.ReadWriter{Out: output, In: input},
+		Response: &accessverifier.Response{
+			Payload: accessverifier.CustomPayload{
+				Data: accessverifier.CustomPayloadData{PrimaryRepo: url},
+			},
+		},
+	}
+
+	err := cmd.Execute(context.Background())
+	require.Error(t, err)
+	require.Equal(t, "Remote repository is unavailable", err.Error())
+}
+
+func setupPull(t *testing.T, uploadPackStatusCode int) string {
+	infoRefs := "001e# service=git-upload-pack\n" + flush + infoRefsWithoutPrefix
+
+	requests := []testserver.TestRequestHandler{
+		{
+			Path: "/info/refs",
+			Handler: func(w http.ResponseWriter, r *http.Request) {
+				require.Equal(t, "git-upload-pack", r.URL.Query().Get("service"))
+
+				w.Write([]byte(infoRefs))
+			},
+		},
+		{
+			Path: "/git-upload-pack",
+			Handler: func(w http.ResponseWriter, r *http.Request) {
+				body, err := io.ReadAll(r.Body)
+				require.NoError(t, err)
+				defer r.Body.Close()
+
+				require.True(t, strings.HasSuffix(string(body), "0009done\n"))
+
+				w.WriteHeader(uploadPackStatusCode)
+			},
+		},
+	}
+
+	return testserver.StartHttpServer(t, requests)
+}

internal/command/uploadpack/uploadpack.go

@@ -2,6 +2,7 @@ package uploadpack
 
 import (
 	"context"
+	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/githttp"
 
 	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command"
 	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/commandargs"
@@ -37,6 +38,16 @@ func (c *Command) Execute(ctx context.Context) (context.Context, error) {
 	ctxWithLogData := context.WithValue(ctx, "logData", logData)
 
 	if response.IsCustomAction() {
+		if response.Payload.Data.GeoProxyFetchDirectToPrimary {
+			cmd := githttp.PullCommand{
+				Config:     c.Config,
+				ReadWriter: c.ReadWriter,
+				Response:   response,
+			}
+
+			return ctxWithLogData, cmd.Execute(ctx)
+		}
+
 		customAction := customaction.Command{
 			Config:     c.Config,
 			ReadWriter: c.ReadWriter,

internal/gitlabnet/git/client.go

@@ -39,6 +39,17 @@ func (c *Client) ReceivePack(ctx context.Context, body io.Reader) (*http.Respons
 	return c.do(request)
 }
 
+func (c *Client) UploadPack(ctx context.Context, body io.Reader) (*http.Response, error) {
+	request, err := http.NewRequestWithContext(ctx, http.MethodPost, c.Url+"/git-upload-pack", body)
+	if err != nil {
+		return nil, err
+	}
+	request.Header.Add("Content-Type", "application/x-git-upload-pack-request")
+	request.Header.Add("Accept", "application/x-git-upload-pack-result")
+
+	return c.do(request)
+}
+
 func (c *Client) do(request *http.Request) (*http.Response, error) {
 	for k, v := range c.Headers {
 		request.Header.Add(k, v)

internal/gitlabnet/git/client_test.go

@@ -50,6 +50,20 @@ func TestReceivePack(t *testing.T) {
 	require.Equal(t, "git-receive-pack: content", string(body))
 }
 
+func TestUploadPack(t *testing.T) {
+	client := setup(t)
+
+	refsBody := "0032want 0a53e9ddeaddad63ad106860237bbf53411d11a7\n"
+	response, err := client.UploadPack(context.Background(), bytes.NewReader([]byte(refsBody)))
+	require.NoError(t, err)
+	defer response.Body.Close()
+
+	body, err := io.ReadAll(response.Body)
+	require.NoError(t, err)
+
+	require.Equal(t, "git-upload-pack: content", string(body))
+}
+
 func TestFailedHTTPRequest(t *testing.T) {
 	client := &Client{
 		Url:     testserver.StartHttpServer(t, []testserver.TestRequestHandler{}),
@@ -83,7 +97,6 @@ func setup(t *testing.T) *Client {
 				require.Equal(t, customHeaders["Header-One"], r.Header.Get("Header-One"))
 				require.Equal(t, "application/x-git-receive-pack-request", r.Header.Get("Content-Type"))
 				require.Equal(t, "application/x-git-receive-pack-result", r.Header.Get("Accept"))
-				require.Equal(t, customHeaders["Header-One"], r.Header.Get("Header-One"))
 
 				body, err := io.ReadAll(r.Body)
 				require.NoError(t, err)
@@ -93,6 +106,21 @@ func setup(t *testing.T) *Client {
 				w.Write(body)
 			},
 		},
+		{
+			Path: "/git-upload-pack",
+			Handler: func(w http.ResponseWriter, r *http.Request) {
+				require.Equal(t, customHeaders["Authorization"], r.Header.Get("Authorization"))
+				require.Equal(t, customHeaders["Header-One"], r.Header.Get("Header-One"))
+				require.Equal(t, "application/x-git-upload-pack-request", r.Header.Get("Content-Type"))
+				require.Equal(t, "application/x-git-upload-pack-result", r.Header.Get("Accept"))
+
+				_, err := io.ReadAll(r.Body)
+				require.NoError(t, err)
+				defer r.Body.Close()
+
+				w.Write([]byte("git-upload-pack: content"))
+			},
+		},
 	}
 
 	client := &Client{