Merge branch 'id-api-https' into 'master'

Support calling internal API using HTTPS

Closes #179

See merge request gitlab-org/gitlab-shell!297

go/internal/config/config.go

@@ -6,7 +6,6 @@ import (
 	"os"
 	"path"
 	"path/filepath"
-	"strings"
 
 	yaml "gopkg.in/yaml.v2"
 )
@@ -26,6 +25,9 @@ type HttpSettingsConfig struct {
 	User               string `yaml:"user"`
 	Password           string `yaml:"password"`
 	ReadTimeoutSeconds uint64 `yaml:"read_timeout"`
+	CaFile             string `yaml:"ca_file"`
+	CaPath             string `yaml:"ca_path"`
+	SelfSignedCert     bool   `yaml:"self_signed_cert"`
 }
 
 type Config struct {
@@ -59,10 +61,6 @@ func (c *Config) FeatureEnabled(featureName string) bool {
 		return false
 	}
 
-	if !strings.HasPrefix(c.GitlabUrl, "http+unix://") && !strings.HasPrefix(c.GitlabUrl, "http://") {
-		return false
-	}
-
 	for _, enabledFeature := range c.Migration.Features {
 		if enabledFeature == featureName {
 			return true

go/internal/config/config_test.go

@@ -94,6 +94,13 @@ func TestParseConfig(t *testing.T) {
 			secret:       "default-secret-content",
 			httpSettings: HttpSettingsConfig{User: "user_basic_auth", Password: "password_basic_auth", ReadTimeoutSeconds: 500},
 		},
+		{
+			yaml:         "http_settings:\n  ca_file: /etc/ssl/cert.pem\n  ca_path: /etc/pki/tls/certs\n  self_signed_cert: true",
+			path:         path.Join(testRoot, "gitlab-shell.log"),
+			format:       "text",
+			secret:       "default-secret-content",
+			httpSettings: HttpSettingsConfig{CaFile: "/etc/ssl/cert.pem", CaPath: "/etc/pki/tls/certs", SelfSignedCert: true},
+		},
 	}
 
 	for _, tc := range testCases {
@@ -158,13 +165,13 @@ func TestFeatureEnabled(t *testing.T) {
 			expectEnabled: true,
 		},
 		{
-			desc: "When the protocol is not supported",
+			desc: "When the protocol is https and the feature enabled",
 			config: &Config{
 				GitlabUrl: "https://localhost:3000",
 				Migration: MigrationConfig{Enabled: true, Features: []string{"discover"}},
 			},
 			feature:       "discover",
-			expectEnabled: false,
+			expectEnabled: true,
 		},
 	}
 

go/internal/config/httpclient.go

@@ -2,16 +2,21 @@ package config
 
 import (
 	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"io/ioutil"
 	"net"
 	"net/http"
+	"path/filepath"
 	"strings"
 	"time"
 )
 
 const (
 	socketBaseUrl             = "http://unix"
-	UnixSocketProtocol        = "http+unix://"
-	HttpProtocol              = "http://"
+	unixSocketProtocol        = "http+unix://"
+	httpProtocol              = "http://"
+	httpsProtocol             = "https://"
 	defaultReadTimeoutSeconds = 300
 )
 
@@ -27,10 +32,12 @@ func (c *Config) GetHttpClient() *HttpClient {
 
 	var transport *http.Transport
 	var host string
-	if strings.HasPrefix(c.GitlabUrl, UnixSocketProtocol) {
+	if strings.HasPrefix(c.GitlabUrl, unixSocketProtocol) {
 		transport, host = c.buildSocketTransport()
-	} else if strings.HasPrefix(c.GitlabUrl, HttpProtocol) {
+	} else if strings.HasPrefix(c.GitlabUrl, httpProtocol) {
 		transport, host = c.buildHttpTransport()
+	} else if strings.HasPrefix(c.GitlabUrl, httpsProtocol) {
+		transport, host = c.buildHttpsTransport()
 	} else {
 		return nil
 	}
@@ -48,7 +55,7 @@ func (c *Config) GetHttpClient() *HttpClient {
 }
 
 func (c *Config) buildSocketTransport() (*http.Transport, string) {
-	socketPath := strings.TrimPrefix(c.GitlabUrl, UnixSocketProtocol)
+	socketPath := strings.TrimPrefix(c.GitlabUrl, unixSocketProtocol)
 	transport := &http.Transport{
 		DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
 			dialer := net.Dialer{}
@@ -59,6 +66,47 @@ func (c *Config) buildSocketTransport() (*http.Transport, string) {
 	return transport, socketBaseUrl
 }
 
+func (c *Config) buildHttpsTransport() (*http.Transport, string) {
+	certPool, err := x509.SystemCertPool()
+
+	if err != nil {
+		certPool = x509.NewCertPool()
+	}
+
+	caFile := c.HttpSettings.CaFile
+	if caFile != "" {
+		addCertToPool(certPool, caFile)
+	}
+
+	caPath := c.HttpSettings.CaPath
+	if caPath != "" {
+		fis, _ := ioutil.ReadDir(caPath)
+		for _, fi := range fis {
+			if fi.IsDir() {
+				continue
+			}
+
+			addCertToPool(certPool, filepath.Join(caPath, fi.Name()))
+		}
+	}
+
+	transport := &http.Transport{
+		TLSClientConfig: &tls.Config{
+			RootCAs:            certPool,
+			InsecureSkipVerify: c.HttpSettings.SelfSignedCert,
+		},
+	}
+
+	return transport, c.GitlabUrl
+}
+
+func addCertToPool(certPool *x509.CertPool, fileName string) {
+	cert, err := ioutil.ReadFile(fileName)
+	if err == nil {
+		certPool.AppendCertsFromPEM(cert)
+	}
+}
+
 func (c *Config) buildHttpTransport() (*http.Transport, string) {
 	return &http.Transport{}, c.GitlabUrl
 }

go/internal/gitlabnet/client_test.go

@@ -6,15 +6,21 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"path"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"gitlab.com/gitlab-org/gitlab-shell/go/internal/config"
 	"gitlab.com/gitlab-org/gitlab-shell/go/internal/gitlabnet/testserver"
+	"gitlab.com/gitlab-org/gitlab-shell/go/internal/testhelper"
 )
 
 func TestClients(t *testing.T) {
+	testDirCleanup, err := testhelper.PrepareTestRootDir()
+	require.NoError(t, err)
+	defer testDirCleanup()
+
 	requests := []testserver.TestRequestHandler{
 		{
 			Path: "/api/v4/internal/hello",
@@ -64,19 +70,26 @@ func TestClients(t *testing.T) {
 
 	testCases := []struct {
 		desc   string
-		secret string
+		config *config.Config
 		server func([]testserver.TestRequestHandler) (func(), string, error)
 	}{
 		{
 			desc:   "Socket client",
-			secret: "sssh, it's a secret",
+			config: &config.Config{},
 			server: testserver.StartSocketHttpServer,
 		},
 		{
 			desc:   "Http client",
-			secret: "sssh, it's a secret",
+			config: &config.Config{},
 			server: testserver.StartHttpServer,
 		},
+		{
+			desc: "Https client",
+			config: &config.Config{
+				HttpSettings: config.HttpSettingsConfig{CaFile: path.Join(testhelper.TestRoot, "certs/valid/server.crt")},
+			},
+			server: testserver.StartHttpsServer,
+		},
 	}
 
 	for _, tc := range testCases {
@@ -85,7 +98,10 @@ func TestClients(t *testing.T) {
 			defer cleanup()
 			require.NoError(t, err)
 
-			client, err := GetClient(&config.Config{GitlabUrl: url, Secret: tc.secret})
+			tc.config.GitlabUrl = url
+			tc.config.Secret = "sssh, it's a secret"
+
+			client, err := GetClient(tc.config)
 			require.NoError(t, err)
 
 			testBrokenRequest(t, client)

go/internal/gitlabnet/httpsclient_test.go

+package gitlabnet
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"path"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gitlab.com/gitlab-org/gitlab-shell/go/internal/config"
+	"gitlab.com/gitlab-org/gitlab-shell/go/internal/gitlabnet/testserver"
+	"gitlab.com/gitlab-org/gitlab-shell/go/internal/testhelper"
+)
+
+func TestSuccessfulRequests(t *testing.T) {
+	testCases := []struct {
+		desc   string
+		config *config.Config
+	}{
+		{
+			desc: "Valid CaFile",
+			config: &config.Config{
+				HttpSettings: config.HttpSettingsConfig{CaFile: path.Join(testhelper.TestRoot, "certs/valid/server.crt")},
+			},
+		},
+		{
+			desc: "Valid CaPath",
+			config: &config.Config{
+				HttpSettings: config.HttpSettingsConfig{CaPath: path.Join(testhelper.TestRoot, "certs/valid")},
+			},
+		},
+		{
+			desc: "Self signed cert option enabled",
+			config: &config.Config{
+				HttpSettings: config.HttpSettingsConfig{SelfSignedCert: true},
+			},
+		},
+		{
+			desc: "Invalid cert with self signed cert option enabled",
+			config: &config.Config{
+				HttpSettings: config.HttpSettingsConfig{SelfSignedCert: true, CaFile: path.Join(testhelper.TestRoot, "certs/valid/server.crt")},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			client, cleanup := setupWithRequests(t, tc.config)
+			defer cleanup()
+
+			response, err := client.Get("/hello")
+			require.NoError(t, err)
+			require.NotNil(t, response)
+
+			defer response.Body.Close()
+
+			responseBody, err := ioutil.ReadAll(response.Body)
+			assert.NoError(t, err)
+			assert.Equal(t, string(responseBody), "Hello")
+		})
+	}
+}
+
+func TestFailedRequests(t *testing.T) {
+	testCases := []struct {
+		desc   string
+		config *config.Config
+	}{
+		{
+			desc: "Invalid CaFile",
+			config: &config.Config{
+				HttpSettings: config.HttpSettingsConfig{CaFile: path.Join(testhelper.TestRoot, "certs/invalid/server.crt")},
+			},
+		},
+		{
+			desc: "Invalid CaPath",
+			config: &config.Config{
+				HttpSettings: config.HttpSettingsConfig{CaPath: path.Join(testhelper.TestRoot, "certs/invalid")},
+			},
+		},
+		{
+			desc:   "Empty config",
+			config: &config.Config{},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			client, cleanup := setupWithRequests(t, tc.config)
+			defer cleanup()
+
+			_, err := client.Get("/hello")
+			require.Error(t, err)
+
+			assert.Equal(t, err.Error(), "Internal API unreachable")
+		})
+	}
+}
+
+func setupWithRequests(t *testing.T, config *config.Config) (*GitlabClient, func()) {
+	testDirCleanup, err := testhelper.PrepareTestRootDir()
+	require.NoError(t, err)
+	defer testDirCleanup()
+
+	requests := []testserver.TestRequestHandler{
+		{
+			Path: "/api/v4/internal/hello",
+			Handler: func(w http.ResponseWriter, r *http.Request) {
+				require.Equal(t, http.MethodGet, r.Method)
+
+				fmt.Fprint(w, "Hello")
+			},
+		},
+	}
+
+	cleanup, url, err := testserver.StartHttpsServer(requests)
+	require.NoError(t, err)
+
+	config.GitlabUrl = url
+	client, err := GetClient(config)
+	require.NoError(t, err)
+
+	return client, cleanup
+}

go/internal/gitlabnet/testserver/testserver.go

 package testserver
 
 import (
+	"crypto/tls"
 	"io/ioutil"
 	"log"
 	"net"
@@ -9,6 +10,8 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+
+	"gitlab.com/gitlab-org/gitlab-shell/go/internal/testhelper"
 )
 
 var (
@@ -50,6 +53,23 @@ func StartHttpServer(handlers []TestRequestHandler) (func(), string, error) {
 	return server.Close, server.URL, nil
 }
 
+func StartHttpsServer(handlers []TestRequestHandler) (func(), string, error) {
+	crt := path.Join(testhelper.TestRoot, "certs/valid/server.crt")
+	key := path.Join(testhelper.TestRoot, "certs/valid/server.key")
+
+	server := httptest.NewUnstartedServer(buildHandler(handlers))
+	cer, err := tls.LoadX509KeyPair(crt, key)
+
+	if err != nil {
+		return nil, "", err
+	}
+
+	server.TLS = &tls.Config{Certificates: []tls.Certificate{cer}}
+	server.StartTLS()
+
+	return server.Close, server.URL, nil
+}
+
 func cleanupSocket() {
 	os.RemoveAll(tempDir)
 }

go/internal/testhelper/testdata/testroot/certs/invalid/server.crt

+-----BEGIN CERTIFICATE-----
+MinvalidcertAOvHjs6cs1R9MAoGCCqGSM49BAMCMBQxEjAQBgNVBAMMCWxvY2Fs
+ainvalidcertOTA0MjQxNjM4NTBaFw0yOTA0MjExNjM4NTBaMBQxEjAQBgNVBAMM
+CinvalidcertdDB2MBAGByqGSM49AgEGBSuBBAAiA2IABJ5m7oW9OuL7aTAC04sL
+3invalidcertdB2L0GsVCImav4PEpx6UAjkoiNGW9j0zPdNgxTYDjiCaGmr1aY2X
+kinvalidcert7MNq7H8v7Ce/vrKkcDMOX8Gd/ddT3dEVqzAKBggqhkjOPQQDAgNp
+AinvalidcertswcyjiB+A+ZjMSfaOsA2hAP0I3fkTcry386DePViMfnaIjm7rcuu
+Jinvalidcert5V5CHypOxio1tOtGjaDkSH2FCdoatMyIe02+F6TIo44i4J/zjN52
+Jinvalidcert
+-----END CERTIFICATE-----

go/internal/testhelper/testdata/testroot/certs/valid/server.crt

+-----BEGIN CERTIFICATE-----
+MIIBsDCCAVqgAwIBAgIJALlYxbvxYURTMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNV
+BAMMCWxvY2FsaG9zdDAeFw0xOTA1MDEwNzM2MDNaFw0yOTA0MjgwNzM2MDNaMBQx
+EjAQBgNVBAMMCWxvY2FsaG9zdDBcMA0GCSqGSIb3DQEBAQUAA0sAMEgCQQDlgT4b
+0ufjR6mIaTuP4lXXtY74YlHDlh4m/qqD4aTLPU/x6fMvvqsqIqKX9emAz/U5rzXv
+GKwpN2DlBKEADzWzAgMBAAGjgY4wgYswHQYDVR0OBBYEFM13Hmhp8oMJfntPBpO6
+fDJ6e1W0MB8GA1UdIwQYMBaAFM13Hmhp8oMJfntPBpO6fDJ6e1W0MA4GA1UdDwEB
+/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwGgYDVR0RBBMw
+EYIJbG9jYWxob3N0hwR/AAABMA0GCSqGSIb3DQEBCwUAA0EAohV1Gh0A6Q5SwY3I
+I5Dfu3INMU/9Id+EoHorJwkN6oUuRMjwdwjzZaa1WLWlkw7EHYwk1YC3qltKnDp/
+PuFxwQ==
+-----END CERTIFICATE-----

go/internal/testhelper/testdata/testroot/certs/valid/server.key

+-----BEGIN PRIVATE KEY-----
+MIIBVwIBADANBgkqhkiG9w0BAQEFAASCAUEwggE9AgEAAkEA5YE+G9Ln40epiGk7
+j+JV17WO+GJRw5YeJv6qg+Gkyz1P8enzL76rKiKil/XpgM/1Oa817xisKTdg5QSh
+AA81swIDAQABAkEAofMPhsbPB1y8TxwjAadvd+YQW0nV9LRr1oyCesmxZhAp0KQq
+3vLs24nQAYH7mAtfcpv0l1t8NHl6JajZd3sLIQIhAPLQ4HuZSGPTncJFZv4/tqwf
+uXmouo9Fo7+eAXXx1H5LAiEA8fdXw/vuUzb+3I/hwkS5i1zgsYdMye63UffH+aSS
+5TkCIQC1ZskebaCAO7szROgx7+WH59eIBT8DBFLWN7P9qmJGywIhALPDL7gRxgia
+thPE7VN33WUVNnWN8FWhfR5veGkWhG+5AiEAnxDUMc2ZewuNkPZHZAIobRAily+q
+Q5REf1FU92ptvbg=
+-----END PRIVATE KEY-----