Merge branch 'git-lfs-transfer-enable' into 'main'

git-lfs-transfer: Enable in shell and introduce batch download

See merge request https://gitlab.com/gitlab-org/gitlab-shell/-/merge_requests/942



Merged-by: Stan Hu <stanhu@gmail.com>
Approved-by: Stan Hu <stanhu@gmail.com>
Reviewed-by: Joe Snyder <joe.snyder@kitware.com>
Reviewed-by: Ash McKenzie <amckenzie@gitlab.com>
Co-authored-by: Joseph Snyder <joe.snyder@kitware.com>
Co-authored-by: Ash McKenzie <amckenzie@gitlab.com>
Co-authored-by: Kyle Edwards <kyle.edwards@kitware.com>

cmd/gitlab-shell/command/command.go

@@ -5,6 +5,7 @@ import (
 	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/commandargs"
 	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/discover"
 	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/lfsauthenticate"
+	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/lfstransfer"
 	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/personalaccesstoken"
 	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/readwriter"
 	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/receivepack"
@@ -108,6 +109,10 @@ func Build(args *commandargs.Shell, config *config.Config, readWriter *readwrite
 		return &twofactorverify.Command{Config: config, Args: args, ReadWriter: readWriter}
 	case commandargs.LfsAuthenticate:
 		return &lfsauthenticate.Command{Config: config, Args: args, ReadWriter: readWriter}
+	case commandargs.LfsTransfer:
+		if config.LFSConfig.PureSSHProtocol {
+			return &lfstransfer.Command{Config: config, Args: args, ReadWriter: readWriter}
+		}
 	case commandargs.ReceivePack:
 		return &receivepack.Command{Config: config, Args: args, ReadWriter: readWriter}
 	case commandargs.UploadPack:

cmd/gitlab-shell/command/command_test.go

@@ -9,6 +9,7 @@ import (
 	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/commandargs"
 	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/discover"
 	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/lfsauthenticate"
+	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/lfstransfer"
 	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/personalaccesstoken"
 	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/receivepack"
 	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/shared/disallowedcommand"
@@ -103,6 +104,45 @@ func TestNew(t *testing.T) {
 	}
 }
 
+func TestLFSTransferCommands(t *testing.T) {
+	testCases := []struct {
+		desc         string
+		executable   *executable.Executable
+		env          sshenv.Env
+		arguments    []string
+		config       *config.Config
+		expectedType interface{}
+		errorString  string
+	}{
+		{
+			desc:         "it returns an Lfstransfer command",
+			executable:   gitlabShellExec,
+			env:          buildEnv("git-lfs-transfer"),
+			config:       &config.Config{GitlabUrl: "http+unix://gitlab.socket", LFSConfig: config.LFSConfig{PureSSHProtocol: true}},
+			expectedType: &lfstransfer.Command{},
+			errorString:  "",
+		},
+		{
+			desc:         "it does not return an Lfstransfer command when config disallows pureSSH",
+			executable:   gitlabShellExec,
+			env:          buildEnv("git-lfs-transfer"),
+			config:       &config.Config{GitlabUrl: "http+unix://gitlab.socket", LFSConfig: config.LFSConfig{PureSSHProtocol: false}},
+			expectedType: nil,
+			errorString:  "Disallowed command",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			command, err := cmd.New(tc.arguments, tc.env, tc.config, nil)
+
+			if len(tc.errorString) > 0 {
+				require.Equal(t, err.Error(), tc.errorString)
+			}
+			require.IsType(t, tc.expectedType, command)
+		})
+	}
+}
 func TestFailingNew(t *testing.T) {
 	testCases := []struct {
 		desc          string
@@ -253,6 +293,13 @@ func TestParseSuccess(t *testing.T) {
 			arguments:    []string{},
 			expectedArgs: &commandargs.Shell{Arguments: []string{}, SshArgs: []string{"git-lfs-authenticate", "group/repo", "download"}, CommandType: commandargs.LfsAuthenticate, Env: sshenv.Env{IsSSHConnection: true, OriginalCommand: "git-lfs-authenticate 'group/repo' download"}},
 		},
+		{
+			desc:         "It parses git-lfs-transfer command",
+			executable:   &executable.Executable{Name: executable.GitlabShell},
+			env:          sshenv.Env{IsSSHConnection: true, OriginalCommand: "git-lfs-transfer 'group/repo' download"},
+			arguments:    []string{},
+			expectedArgs: &commandargs.Shell{Arguments: []string{}, SshArgs: []string{"git-lfs-transfer", "group/repo", "download"}, CommandType: commandargs.LfsTransfer, Env: sshenv.Env{IsSSHConnection: true, OriginalCommand: "git-lfs-transfer 'group/repo' download"}},
+		},
 	}
 
 	for _, tc := range testCases {

internal/command/commandargs/shell.go

@@ -14,6 +14,7 @@ const (
 	TwoFactorRecover    CommandType = "2fa_recovery_codes"
 	TwoFactorVerify     CommandType = "2fa_verify"
 	LfsAuthenticate     CommandType = "git-lfs-authenticate"
+	LfsTransfer         CommandType = "git-lfs-transfer"
 	ReceivePack         CommandType = "git-receive-pack"
 	UploadPack          CommandType = "git-upload-pack"
 	UploadArchive       CommandType = "git-upload-archive"

internal/command/lfstransfer/gitlab_backend.go

@@ -2,6 +2,10 @@ package lfstransfer
 
 import (
 	"context"
+	"crypto/hmac"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/json"
 	"fmt"
 	"io"
 	"io/fs"
@@ -9,6 +13,7 @@ import (
 	"github.com/charmbracelet/git-lfs-transfer/transfer"
 	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/commandargs"
 	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/config"
+	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/gitlabnet/lfstransfer"
 )
 
 type errCustom struct {
@@ -41,19 +46,113 @@ type GitlabBackend struct {
 	config *config.Config
 	args   *commandargs.Shell
 	auth   *GitlabAuthentication
+	client *lfstransfer.Client
+}
+
+type idData struct {
+	Operation string            `json:"operation"`
+	Oid       string            `json:"oid"`
+	Href      string            `json:"href"`
+	Headers   map[string]string `json:"headers,omitempty"`
 }
 
 func NewGitlabBackend(ctx context.Context, config *config.Config, args *commandargs.Shell, auth *GitlabAuthentication) (*GitlabBackend, error) {
+	client, err := lfstransfer.NewClient(config, args, auth.href, auth.auth)
+	if err != nil {
+		return nil, err
+	}
+
 	return &GitlabBackend{
 		ctx,
 		config,
 		args,
 		auth,
+		client,
 	}, nil
 }
 
+func (b *GitlabBackend) issueBatchArgs(op string, oid string, href string, headers map[string]string) (args transfer.Args, err error) {
+	data := &idData{
+		Operation: op,
+		Oid:       oid,
+		Href:      href,
+		Headers:   headers,
+	}
+
+	args = transfer.Args{
+		"id":    "",
+		"token": "",
+	}
+	dataBinary, err := json.Marshal(data)
+	if err != nil {
+		return args, err
+	}
+
+	h := hmac.New(sha256.New, []byte(b.config.Secret))
+	_, err = h.Write(dataBinary)
+	if err != nil {
+		return args, err
+	}
+
+	args["id"] = base64.StdEncoding.EncodeToString(dataBinary)
+	args["token"] = base64.StdEncoding.EncodeToString(h.Sum(nil))
+
+	return args, nil
+}
+
 func (b *GitlabBackend) Batch(op string, pointers []transfer.BatchItem, args transfer.Args) ([]transfer.BatchItem, error) {
-	return nil, newErrUnsupported("batch")
+	if op != "download" {
+		return nil, newErrUnsupported("upload batch")
+	}
+
+	reqObjects := make([]*lfstransfer.BatchObject, 0)
+
+	for _, pointer := range pointers {
+		reqObject := &lfstransfer.BatchObject{
+			Oid:  pointer.Oid,
+			Size: pointer.Size,
+		}
+		reqObjects = append(reqObjects, reqObject)
+	}
+
+	refName := args["refname"]
+	hashAlgo := args["hash-algo"]
+
+	res, err := b.client.Batch(op, reqObjects, refName, hashAlgo)
+	if err != nil {
+		return nil, err
+	}
+
+	items := make([]transfer.BatchItem, 0)
+
+	for _, retObject := range res.Objects {
+		var present bool
+		var action *lfstransfer.BatchAction
+		var args transfer.Args
+
+		if action, present = retObject.Actions[op]; present {
+			args, err = b.issueBatchArgs(op, retObject.Oid, action.Href, action.Header)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		if op == "upload" {
+			present = !present
+		}
+
+		batchItem := transfer.BatchItem{
+			Pointer: transfer.Pointer{
+				Oid:  retObject.Oid,
+				Size: retObject.Size,
+			},
+			Present: present,
+			Args:    args,
+		}
+		items = append(items, batchItem)
+	}
+
+	return items, nil
 }
 
 func (b *GitlabBackend) StartUpload(oid string, r io.Reader, args transfer.Args) (io.Closer, error) {

internal/command/lfstransfer/lfstransfer_test.go

@@ -2,10 +2,15 @@ package lfstransfer
 
 import (
 	"context"
+	"crypto/hmac"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
+	"strings"
 	"sync"
 	"testing"
 
@@ -18,14 +23,31 @@ import (
 	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/gitlabnet/lfsauthenticate"
 )
 
-func setupWaitGroup(t *testing.T, cmd *Command) *sync.WaitGroup {
+const (
+	largeFileContents      = "This is a large file\n"
+	evenLargerFileContents = "This is an even larger file\n"
+)
+
+var (
+	largeFileLen  = len(largeFileContents)
+	largeFileHash = sha256.Sum256([]byte(largeFileContents))
+	largeFileOid  = hex.EncodeToString(largeFileHash[:])
+
+	evenLargerFileLen  = len(evenLargerFileContents)
+	evenLargerFileHash = sha256.Sum256([]byte(evenLargerFileContents))
+	evenLargerFileOid  = hex.EncodeToString(evenLargerFileHash[:])
+)
+
+func setupWaitGroupForExecute(t *testing.T, cmd *Command) *sync.WaitGroup {
 	wg := &sync.WaitGroup{}
 	wg.Add(1)
+
 	go func() {
 		_, err := cmd.Execute(context.Background())
 		require.NoError(t, err)
 		wg.Done()
 	}()
+
 	return wg
 }
 
@@ -56,10 +78,12 @@ func writeCommandArgsAndBinaryData(t *testing.T, pl *pktline.Pktline, command st
 
 func writeCommandArgsAndTextData(t *testing.T, pl *pktline.Pktline, command string, args []string, data []string) {
 	require.NoError(t, pl.WritePacketText(command))
+
 	for _, arg := range args {
 		require.NoError(t, pl.WritePacketText(arg))
 	}
 	require.NoError(t, pl.WriteDelim())
+
 	for _, datum := range data {
 		require.NoError(t, pl.WritePacketText(datum))
 	}
@@ -181,6 +205,7 @@ func readStatusArgsAndTextData(t *testing.T, pl *pktline.Pktline) (status string
 	// Read status.
 	status, l, err := pl.ReadPacketTextWithLength()
 	require.NoError(t, err)
+
 	switch l {
 	case 0:
 		require.Fail(t, "Expected text, got flush packet")
@@ -193,6 +218,7 @@ func readStatusArgsAndTextData(t *testing.T, pl *pktline.Pktline) (status string
 	for !end {
 		arg, l, err := pl.ReadPacketTextWithLength()
 		require.NoError(t, err)
+
 		switch l {
 		case 0:
 			return status, args, nil
@@ -208,6 +234,7 @@ func readStatusArgsAndTextData(t *testing.T, pl *pktline.Pktline) (status string
 	for !end {
 		datum, l, err := pl.ReadPacketTextWithLength()
 		require.NoError(t, err)
+
 		switch l {
 		case 0:
 			end = true
@@ -217,6 +244,7 @@ func readStatusArgsAndTextData(t *testing.T, pl *pktline.Pktline) (status string
 			data = append(data, datum)
 		}
 	}
+
 	return status, args, data
 }
 
@@ -235,7 +263,7 @@ func quit(t *testing.T, pl *pktline.Pktline) {
 
 func TestLfsTransferCapabilities(t *testing.T) {
 	_, cmd, pl, _ := setup(t, "rw", "group/repo", "upload")
-	wg := setupWaitGroup(t, cmd)
+	wg := setupWaitGroupForExecute(t, cmd)
 	negotiateVersion(t, pl)
 
 	quit(t, pl)
@@ -249,19 +277,66 @@ func TestLfsTransferNoPermissions(t *testing.T) {
 }
 
 func TestLfsTransferBatchDownload(t *testing.T) {
-	_, cmd, pl, _ := setup(t, "rw", "group/repo", "download")
-	wg := setupWaitGroup(t, cmd)
+	url, cmd, pl, _ := setup(t, "rw", "group/repo", "download")
+	wg := setupWaitGroupForExecute(t, cmd)
 	negotiateVersion(t, pl)
 
 	writeCommandArgsAndTextData(t, pl, "batch", nil, []string{
 		"00000000 0",
+		fmt.Sprintf("%s %d", largeFileOid, largeFileLen),
+		fmt.Sprintf("%s %d", evenLargerFileOid, evenLargerFileLen),
 	})
 	status, args, data := readStatusArgsAndTextData(t, pl)
-	require.Equal(t, "status 405", status)
+	require.Equal(t, "status 200", status)
 	require.Empty(t, args)
-	require.Equal(t, []string{
-		"error: batch is not yet supported by git-lfs-transfer. See https://gitlab.com/groups/gitlab-org/-/epics/11872 to track progress.",
-	}, data)
+	require.Equal(t, "00000000 0 noop", data[0])
+
+	largeFileArgs := strings.Split(data[1], " ")
+	require.Equal(t, 5, len(largeFileArgs))
+	require.Equal(t, largeFileOid, largeFileArgs[0])
+	require.Equal(t, fmt.Sprint(largeFileLen), largeFileArgs[1])
+	require.Equal(t, "download", largeFileArgs[2])
+
+	var idArg string
+	var tokenArg string
+	for _, arg := range largeFileArgs[3:] {
+		switch {
+		case strings.HasPrefix(arg, "id="):
+			idArg = arg
+		case strings.HasPrefix(arg, "token="):
+			tokenArg = arg
+		default:
+			require.Fail(t, "Unexpected batch item argument: %v", arg)
+		}
+	}
+
+	idBase64, found := strings.CutPrefix(idArg, "id=")
+	require.True(t, found)
+	idBinary, err := base64.StdEncoding.DecodeString(idBase64)
+	require.NoError(t, err)
+
+	var id map[string]interface{}
+	require.NoError(t, json.Unmarshal(idBinary, &id))
+	require.Equal(t, map[string]interface{}{
+		"operation": "download",
+		"oid":       largeFileOid,
+		"href":      fmt.Sprintf("%s/group/repo/gitlab-lfs/objects/%s", url, largeFileOid),
+		"headers": map[string]interface{}{
+			"Authorization": "Basic 1234567890",
+			"Content-Type":  "application/octet-stream",
+		},
+	}, id)
+
+	h := hmac.New(sha256.New, []byte("very secret"))
+	h.Write(idBinary)
+	tokenBase64, found := strings.CutPrefix(tokenArg, "token=")
+	require.True(t, found)
+
+	tokenBinary, err := base64.StdEncoding.DecodeString(tokenBase64)
+	require.NoError(t, err)
+	require.Equal(t, h.Sum(nil), tokenBinary)
+
+	require.Equal(t, fmt.Sprintf("%s %d noop", evenLargerFileOid, evenLargerFileLen), data[2])
 
 	quit(t, pl)
 	wg.Wait()
@@ -269,7 +344,7 @@ func TestLfsTransferBatchDownload(t *testing.T) {
 
 func TestLfsTransferBatchUpload(t *testing.T) {
 	_, cmd, pl, _ := setup(t, "rw", "group/repo", "upload")
-	wg := setupWaitGroup(t, cmd)
+	wg := setupWaitGroupForExecute(t, cmd)
 	negotiateVersion(t, pl)
 
 	writeCommandArgsAndTextData(t, pl, "batch", nil, []string{
@@ -279,7 +354,7 @@ func TestLfsTransferBatchUpload(t *testing.T) {
 	require.Equal(t, "status 405", status)
 	require.Empty(t, args)
 	require.Equal(t, []string{
-		"error: batch is not yet supported by git-lfs-transfer. See https://gitlab.com/groups/gitlab-org/-/epics/11872 to track progress.",
+		"error: upload batch is not yet supported by git-lfs-transfer. See https://gitlab.com/groups/gitlab-org/-/epics/11872 to track progress.",
 	}, data)
 
 	quit(t, pl)
@@ -288,7 +363,7 @@ func TestLfsTransferBatchUpload(t *testing.T) {
 
 func TestLfsTransferGetObject(t *testing.T) {
 	_, cmd, pl, _ := setup(t, "rw", "group/repo", "download")
-	wg := setupWaitGroup(t, cmd)
+	wg := setupWaitGroupForExecute(t, cmd)
 	negotiateVersion(t, pl)
 
 	writeCommand(t, pl, "get-object 00000000")
@@ -305,7 +380,7 @@ func TestLfsTransferGetObject(t *testing.T) {
 
 func TestLfsTransferPutObject(t *testing.T) {
 	_, cmd, pl, _ := setup(t, "rw", "group/repo", "upload")
-	wg := setupWaitGroup(t, cmd)
+	wg := setupWaitGroupForExecute(t, cmd)
 	negotiateVersion(t, pl)
 
 	writeCommandArgsAndBinaryData(t, pl, "put-object 00000000", []string{"size=0"}, nil)
@@ -322,7 +397,7 @@ func TestLfsTransferPutObject(t *testing.T) {
 
 func TestLfsTransferVerifyObject(t *testing.T) {
 	_, cmd, pl, _ := setup(t, "rw", "group/repo", "upload")
-	wg := setupWaitGroup(t, cmd)
+	wg := setupWaitGroupForExecute(t, cmd)
 	negotiateVersion(t, pl)
 
 	writeCommandArgs(t, pl, "verify-object 00000000", []string{"size=0"})
@@ -339,7 +414,7 @@ func TestLfsTransferVerifyObject(t *testing.T) {
 
 func TestLfsTransferLock(t *testing.T) {
 	_, cmd, pl, _ := setup(t, "rw", "group/repo", "upload")
-	wg := setupWaitGroup(t, cmd)
+	wg := setupWaitGroupForExecute(t, cmd)
 	negotiateVersion(t, pl)
 
 	writeCommandArgs(t, pl, "lock", []string{"path=large/file"})
@@ -356,7 +431,7 @@ func TestLfsTransferLock(t *testing.T) {
 
 func TestLfsTransferUnlock(t *testing.T) {
 	_, cmd, pl, _ := setup(t, "rw", "group/repo", "upload")
-	wg := setupWaitGroup(t, cmd)
+	wg := setupWaitGroupForExecute(t, cmd)
 	negotiateVersion(t, pl)
 
 	writeCommand(t, pl, "unlock lock1")
@@ -373,7 +448,7 @@ func TestLfsTransferUnlock(t *testing.T) {
 
 func TestLfsTransferListLock(t *testing.T) {
 	_, cmd, pl, _ := setup(t, "rw", "group/repo", "download")
-	wg := setupWaitGroup(t, cmd)
+	wg := setupWaitGroupForExecute(t, cmd)
 	negotiateVersion(t, pl)
 
 	writeCommand(t, pl, "list-lock")
@@ -389,8 +464,9 @@ func TestLfsTransferListLock(t *testing.T) {
 }
 
 func setup(t *testing.T, keyId string, repo string, op string) (string, *Command, *pktline.Pktline, *io.PipeReader) {
-	gitalyAddress, _ := testserver.StartGitalyServer(t, "unix")
 	var url string
+
+	gitalyAddress, _ := testserver.StartGitalyServer(t, "unix")
 	requests := []testserver.TestRequestHandler{
 		{
 			Path: "/api/v4/internal/allowed",
@@ -461,7 +537,68 @@ func setup(t *testing.T, keyId string, repo string, op string) (string, *Command
 				}
 			},
 		},
+		{
+			Path: "/group/repo/info/lfs/objects/batch",
+			Handler: func(w http.ResponseWriter, r *http.Request) {
+				require.Equal(t, fmt.Sprintf("Basic %s", base64.StdEncoding.EncodeToString([]byte("john:sometoken"))), r.Header.Get("Authorization"))
+
+				var requestBody map[string]interface{}
+				json.NewDecoder(r.Body).Decode(&requestBody)
+
+				reqObjects := requestBody["objects"].([]interface{})
+				retObjects := make([]map[string]interface{}, 0)
+				for _, o := range reqObjects {
+					reqObject := o.(map[string]interface{})
+					retObject := map[string]interface{}{
+						"oid": reqObject["oid"],
+					}
+					switch reqObject["oid"] {
+					case largeFileOid:
+						retObject["size"] = largeFileLen
+						if op == "download" {
+							retObject["actions"] = map[string]interface{}{
+								"download": map[string]interface{}{
+									"href": fmt.Sprintf("%s/group/repo/gitlab-lfs/objects/%s", url, largeFileOid),
+									"header": map[string]interface{}{
+										"Authorization": "Basic 1234567890",
+										"Content-Type":  "application/octet-stream",
+									},
+								},
+							}
+						}
+					case evenLargerFileOid:
+						require.Equal(t, evenLargerFileLen, int(reqObject["size"].(float64)))
+						retObject["size"] = evenLargerFileLen
+						if op == "upload" {
+							retObject["actions"] = map[string]interface{}{
+								"upload": map[string]interface{}{
+									"href": fmt.Sprintf("%s/group/repo/gitlab-lfs/objects/%s/%d", url, evenLargerFileOid, evenLargerFileLen),
+									"header": map[string]interface{}{
+										"Authorization": "Basic 1234567890",
+										"Content-Type":  "application/octet-stream",
+									},
+								},
+							}
+						}
+					default:
+						retObject["size"] = reqObject["size"]
+						retObject["error"] = map[string]interface{}{
+							"code":    404,
+							"message": "Not found",
+						}
+					}
+					retObjects = append(retObjects, retObject)
+				}
+
+				retBody := map[string]interface{}{
+					"objects": retObjects,
+				}
+				body, _ := json.Marshal(retBody)
+				w.Write(body)
+			},
+		},
 	}
+
 	url = testserver.StartHttpServer(t, requests)
 
 	inputSource, inputSink := io.Pipe()
@@ -474,5 +611,6 @@ func setup(t *testing.T, keyId string, repo string, op string) (string, *Command
 		ReadWriter: &readwriter.ReadWriter{ErrOut: errorSink, Out: outputSink, In: inputSource},
 	}
 	pl := pktline.NewPktline(outputSource, inputSink)
+
 	return url, cmd, pl, errorSource
 }

internal/config/config.go

@@ -60,7 +60,8 @@ type HttpSettingsConfig struct {
 }
 
 type LFSConfig struct {
-	PureSSHProtocol bool `yaml:"pure_ssh_protocol"`
+	// FIXME: Let's not allow this to be set in config.yml
+	PureSSHProtocol bool // `yaml:"pure_ssh_protocol"`
 }
 
 type Config struct {

internal/gitlabnet/lfstransfer/client.go

+package lfstransfer
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"time"
+
+	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/commandargs"
+	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/config"
+	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/gitlabnet"
+)
+
+type Client struct {
+	config *config.Config
+	args   *commandargs.Shell
+	href   string
+	auth   string
+}
+
+type BatchAction struct {
+	Href      string            `json:"href"`
+	Header    map[string]string `json:"header,omitempty"`
+	ExpiresAt time.Time         `json:"expires_at,omitempty"`
+	ExpiresIn int               `json:"expires_in,omitempty"`
+}
+
+type BatchObject struct {
+	Oid           string                  `json:"oid,omitempty"`
+	Size          int64                   `json:"size"`
+	Authenticated bool                    `json:"authenticated,omitempty"`
+	Actions       map[string]*BatchAction `json:"actions,omitempty"`
+}
+
+type batchRef struct {
+	Name string `json:"name,omitempty"`
+}
+
+type batchRequest struct {
+	Operation     string         `json:"operation"`
+	Objects       []*BatchObject `json:"objects"`
+	Ref           *batchRef      `json:"ref,omitempty"`
+	HashAlgorithm string         `json:"hash_algo,omitempty"`
+}
+
+type BatchResponse struct {
+	Objects       []*BatchObject `json:"objects"`
+	HashAlgorithm string         `json:"hash_algo,omitempty"`
+}
+
+func NewClient(config *config.Config, args *commandargs.Shell, href string, auth string) (*Client, error) {
+	return &Client{config: config, args: args, href: href, auth: auth}, nil
+}
+
+func (c *Client) Batch(operation string, reqObjects []*BatchObject, ref string, reqHashAlgo string) (*BatchResponse, error) {
+	var bref *batchRef
+
+	// FIXME: This causes tests to fail
+	// if ref == "" {
+	// 	return nil, errors.New("A ref must be specified.")
+	// }
+
+	bref = &batchRef{Name: ref}
+	body := batchRequest{
+		Operation:     operation,
+		Objects:       reqObjects,
+		Ref:           bref,
+		HashAlgorithm: reqHashAlgo,
+	}
+
+	jsonData, err := json.Marshal(body)
+	if err != nil {
+		return nil, err
+	}
+
+	jsonReader := bytes.NewReader(jsonData)
+
+	req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/objects/batch", c.href), jsonReader)
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Set("Content-Type", "application/vnd.git-lfs+json")
+	req.Header.Set("Authorization", c.auth)
+
+	client := http.Client{}
+	res, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	// Error condition taken from example: https://pkg.go.dev/net/http#example-Get
+	if res.StatusCode > 399 {
+		return nil, errors.New(fmt.Sprintf("Response failed with status code: %d", res.StatusCode))
+	}
+
+	defer func() { _ = res.Body.Close() }()
+
+	response := &BatchResponse{}
+	if err := gitlabnet.ParseJSON(res, response); err != nil {
+		return nil, err
+	}
+
+	return response, nil
+}