Merge branch 'feat/call_rails_record_git_audit' into 'main'

call git_autid_event during git pull/clone if in need

See merge request https://gitlab.com/gitlab-org/gitlab-shell/-/merge_requests/888



Merged-by: Igor Drozdov <idrozdov@gitlab.com>
Approved-by: Igor Drozdov <idrozdov@gitlab.com>
Approved-by: Costel Maxim <cmaxim@gitlab.com>
Reviewed-by: Igor Drozdov <idrozdov@gitlab.com>
Co-authored-by: nanmu42 <i@nanmu.me>
Co-authored-by: rrylee <rrylee1994@gmail.com>

cmd/gitlab-sshd/acceptance_test.go

@@ -186,6 +186,9 @@ func successAPI(t *testing.T, handlers ...customHandler) http.Handler {
 
 			_, err := fmt.Fprint(w, response)
 			require.NoError(t, err)
+		case "/api/v4/internal/shellhorse/git_audit_event":
+			w.WriteHeader(http.StatusOK)
+			return
 		default:
 			t.Logf("Unexpected request to successAPI: %s", r.URL.EscapedPath())
 			t.FailNow()

internal/command/gitauditevent/audit.go

+package gitauditevent
+
+import (
+	"context"
+
+	pb "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb"
+	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/commandargs"
+	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/shared/accessverifier"
+	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/config"
+	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/gitlabnet/gitauditevent"
+	"gitlab.com/gitlab-org/labkit/log"
+)
+
+// Audit is called conditionally during `git-receive-pack` and `git-upload-pack` to generate streaming audit events.
+// Errors are not propagated since this is more a logging process.
+func Audit(ctx context.Context, commandType commandargs.CommandType, c *config.Config, response *accessverifier.Response, packfileStats *pb.PackfileNegotiationStatistics) {
+	ctxlog := log.WithContextFields(ctx, log.Fields{
+		"gl_repository": response.Repo,
+		"command":       commandType,
+		"username":      response.Username,
+	})
+
+	ctxlog.Debug("sending git audit event")
+
+	gitAuditClient, errOnlyLog := gitauditevent.NewClient(c)
+	if errOnlyLog != nil {
+		ctxlog.Errorf("failed to create gitauditevent client: %v", errOnlyLog)
+		return
+	}
+
+	errOnlyLog = gitAuditClient.Audit(ctx, response.Username, commandType, response.Repo, packfileStats)
+	if errOnlyLog != nil {
+		ctxlog.Errorf("failed to audit git event: %v", errOnlyLog)
+		return
+	}
+}

internal/command/gitauditevent/audit_test.go

+package gitauditevent
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"gitlab.com/gitlab-org/gitlab-shell/v14/client/testserver"
+	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/commandargs"
+	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/config"
+	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/gitlabnet/accessverifier"
+	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/gitlabnet/gitauditevent"
+)
+
+var (
+	testUsername = "gitlab-shell"
+	testRepo     = "project-1"
+)
+
+func TestGitAudit(t *testing.T) {
+	called := false
+
+	requests := []testserver.TestRequestHandler{
+		{
+			Path: "/api/v4/internal/shellhorse/git_audit_event",
+			Handler: func(w http.ResponseWriter, r *http.Request) {
+				called = true
+
+				body, err := io.ReadAll(r.Body)
+				require.NoError(t, err)
+				defer r.Body.Close()
+
+				var request *gitauditevent.Request
+				require.NoError(t, json.Unmarshal(body, &request))
+				require.Equal(t, testUsername, request.Username)
+				require.Equal(t, testRepo, request.Repo)
+
+				w.WriteHeader(http.StatusOK)
+			},
+		},
+	}
+
+	s := &commandargs.Shell{
+		CommandType: commandargs.UploadArchive,
+	}
+
+	url := testserver.StartSocketHttpServer(t, requests)
+	Audit(context.Background(), s.CommandType, &config.Config{GitlabUrl: url}, &accessverifier.Response{
+		Username: testUsername,
+		Repo:     testRepo,
+	}, nil)
+
+	require.True(t, called)
+}

internal/command/receivepack/receivepack.go

@@ -5,6 +5,7 @@ import (
 
 	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command"
 	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/commandargs"
+	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/gitauditevent"
 	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/githttp"
 	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/readwriter"
 	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/shared/accessverifier"
@@ -59,7 +60,15 @@ func (c *Command) Execute(ctx context.Context) (context.Context, error) {
 		return ctxWithLogData, customAction.Execute(ctx, response)
 	}
 
-	return ctxWithLogData, c.performGitalyCall(ctx, response)
+	err = c.performGitalyCall(ctx, response)
+	if err != nil {
+		return ctxWithLogData, err
+	}
+
+	if response.NeedAudit {
+		gitauditevent.Audit(ctx, c.Args.CommandType, c.Config, response, nil /* keep nil for `git-receive-pack`*/)
+	}
+	return ctxWithLogData, nil
 }
 
 func (c *Command) verifyAccess(ctx context.Context, repo string) (*accessverifier.Response, error) {

internal/command/uploadpack/gitalycall.go

@@ -12,7 +12,7 @@ import (
 	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/handler"
 )
 
-func (c *Command) performGitalyCall(ctx context.Context, response *accessverifier.Response) error {
+func (c *Command) performGitalyCall(ctx context.Context, response *accessverifier.Response) (*pb.PackfileNegotiationStatistics, error) {
 	gc := handler.NewGitalyCommand(c.Config, string(commandargs.UploadPack), response)
 
 	request := &pb.SSHUploadPackWithSidechannelRequest{
@@ -21,12 +21,24 @@ func (c *Command) performGitalyCall(ctx context.Context, response *accessverifie
 		GitConfigOptions: response.GitConfigOptions,
 	}
 
-	return gc.RunGitalyCommand(ctx, func(ctx context.Context, conn *grpc.ClientConn) (int32, error) {
+	var stats *pb.PackfileNegotiationStatistics
+	err := gc.RunGitalyCommand(ctx, func(ctx context.Context, conn *grpc.ClientConn) (int32, error) {
 		ctx, cancel := gc.PrepareContext(ctx, request.Repository, c.Args.Env)
 		defer cancel()
 
 		registry := c.Config.GitalyClient.SidechannelRegistry
 		rw := c.ReadWriter
-		return client.UploadPackWithSidechannel(ctx, conn, registry, rw.In, rw.Out, rw.ErrOut, request)
+
+		var (
+			result client.UploadPackResult
+			err    error
+		)
+		result, err = client.UploadPackWithSidechannelWithResult(ctx, conn, registry, rw.In, rw.Out, rw.ErrOut, request)
+		if err == nil {
+			stats = result.PackfileNegotiationStatistics
+		}
+		return result.ExitCode, err
 	})
+
+	return stats, err
 }

internal/command/uploadpack/uploadpack.go

@@ -6,6 +6,7 @@ import (
 
 	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command"
 	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/commandargs"
+	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/gitauditevent"
 	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/readwriter"
 	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/shared/accessverifier"
 	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/shared/customaction"
@@ -56,7 +57,15 @@ func (c *Command) Execute(ctx context.Context) (context.Context, error) {
 		return ctxWithLogData, customAction.Execute(ctx, response)
 	}
 
-	return ctxWithLogData, c.performGitalyCall(ctx, response)
+	stats, err := c.performGitalyCall(ctx, response)
+	if err != nil {
+		return ctxWithLogData, err
+	}
+
+	if response.NeedAudit {
+		gitauditevent.Audit(ctx, c.Args.CommandType, c.Config, response, stats)
+	}
+	return ctxWithLogData, nil
 }
 
 func (c *Command) verifyAccess(ctx context.Context, repo string) (*accessverifier.Response, error) {

internal/config/config.go

@@ -9,7 +9,7 @@ import (
 	"sync"
 	"time"
 
-	yaml "gopkg.in/yaml.v3"
+	"gopkg.in/yaml.v3"
 
 	"gitlab.com/gitlab-org/gitlab-shell/v14/client"
 	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/gitaly"

internal/gitlabnet/accessverifier/client.go

@@ -72,6 +72,8 @@ type Response struct {
 	ConsoleMessages  []string      `json:"gl_console_messages"`
 	Who              string
 	StatusCode       int
+	// NeedAudit indicates whether git event should be audited to rails.
+	NeedAudit bool `json:"need_audit"`
 }
 
 func NewClient(config *config.Config) (*Client, error) {

internal/gitlabnet/gitauditevent/client.go

+package gitauditevent
+
+import (
+	"context"
+	"fmt"
+
+	pb "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb"
+	"gitlab.com/gitlab-org/gitlab-shell/v14/client"
+	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/commandargs"
+	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/config"
+	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/gitlabnet"
+)
+
+const uri = "/api/v4/internal/shellhorse/git_audit_event"
+
+type Client struct {
+	config *config.Config
+	client *client.GitlabNetClient
+}
+
+func NewClient(config *config.Config) (*Client, error) {
+	client, err := gitlabnet.GetClient(config)
+	if err != nil {
+		return nil, fmt.Errorf("error creating http client: %w", err)
+	}
+
+	return &Client{config: config, client: client}, nil
+}
+
+type Request struct {
+	Action        commandargs.CommandType           `json:"action"`
+	Protocol      string                            `json:"protocol"`
+	Repo          string                            `json:"gl_repository"`
+	Username      string                            `json:"username"`
+	PackfileStats *pb.PackfileNegotiationStatistics `json:"packfile_stats,omitempty"`
+}
+
+func (c *Client) Audit(ctx context.Context, username string, action commandargs.CommandType, repo string, packfileStats *pb.PackfileNegotiationStatistics) error {
+	request := &Request{
+		Action:        action,
+		Repo:          repo,
+		Protocol:      "ssh",
+		Username:      username,
+		PackfileStats: packfileStats,
+	}
+
+	response, err := c.client.Post(ctx, uri, request)
+	if err != nil {
+		return err
+	}
+	defer response.Body.Close()
+	return nil
+}

internal/gitlabnet/gitauditevent/client_test.go

+package gitauditevent
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	pb "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb"
+	"gitlab.com/gitlab-org/gitlab-shell/v14/client/testserver"
+	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/commandargs"
+	"gitlab.com/gitlab-org/gitlab-shell/v14/internal/config"
+)
+
+var (
+	testUsername                              = "gitlab-shell"
+	testAction        commandargs.CommandType = "git-upload-pack"
+	testRepo                                  = "gitlab-org/gitlab-shell"
+	testPackfileWants int64                   = 100
+	testPackfileHaves int64                   = 100
+)
+
+func TestAudit(t *testing.T) {
+	client := setup(t, http.StatusOK)
+
+	err := client.Audit(context.Background(), testUsername, testAction, testRepo, &pb.PackfileNegotiationStatistics{
+		Wants: testPackfileWants,
+		Haves: testPackfileHaves,
+	})
+	require.NoError(t, err)
+}
+
+func TestAuditFailed(t *testing.T) {
+	client := setup(t, http.StatusBadRequest)
+
+	err := client.Audit(context.Background(), testUsername, testAction, testRepo, &pb.PackfileNegotiationStatistics{
+		Wants: testPackfileWants,
+		Haves: testPackfileHaves,
+	})
+	require.Error(t, err)
+}
+
+func setup(t *testing.T, responseStatus int) *Client {
+	requests := []testserver.TestRequestHandler{
+		{
+			Path: uri,
+			Handler: func(w http.ResponseWriter, r *http.Request) {
+				body, err := io.ReadAll(r.Body)
+				require.NoError(t, err)
+				defer r.Body.Close()
+
+				var request *Request
+				require.NoError(t, json.Unmarshal(body, &request))
+				require.Equal(t, testUsername, request.Username)
+				require.Equal(t, testAction, request.Action)
+				require.Equal(t, testRepo, request.Repo)
+				require.Equal(t, "ssh", request.Protocol)
+				require.Equal(t, testPackfileWants, request.PackfileStats.Wants)
+				require.Equal(t, testPackfileHaves, request.PackfileStats.Haves)
+
+				w.WriteHeader(responseStatus)
+			},
+		},
+	}
+
+	url := testserver.StartSocketHttpServer(t, requests)
+
+	client, err := NewClient(&config.Config{GitlabUrl: url})
+	require.NoError(t, err)
+
+	return client
+}