As we reevaluate how to best support and maintain Staging Ref in the future, we encourage development teams using this environment to highlight their use cases in the following issue: https://gitlab.com/gitlab-com/gl-infra/software-delivery/framework/software-delivery-framework-issue-tracker/-/issues/36.

Skip to content
Snippets Groups Projects
Commit d8daef30 authored by Ken McKnight's avatar Ken McKnight
Browse files

Merge context logger additions from main

# Conflicts:
#   internal/command/twofactorverify/twofactorverify.go
parents 2e2d3176 fa5bf866
No related branches found
No related tags found
No related merge requests found
config.yml
cover.out
tmp/*
.idea
*.log
*.swp
/*.log*
authorized_keys.lock
.gitlab_shell_secret
.bundle
tags
.bundle/
custom_hooks
hooks/*.d
/go_build
/bin/gitlab-sshd
.gitlab_shell_secret
.idea
/*.log*
/bin/check
/bin/gitlab-shell
/bin/gitlab-shell-authorized-keys-check
/bin/gitlab-shell-authorized-principals-check
/bin/check
/bin/gitlab-sshd
/go_build
authorized_keys.lock
config.yml
cover.out
custom_hooks
hooks/*.d
tags
tmp/*
vendor
Loading
Loading
@@ -3,9 +3,12 @@ include:
- template: Security/SAST.gitlab-ci.yml
- template: Security/Dependency-Scanning.gitlab-ci.yml
- template: Security/Secret-Detection.gitlab-ci.yml
- project: 'gitlab-org/quality/pipeline-common'
file:
- '/ci/danger-review.yml'
variables:
DOCKER_VERSION: "19.03.0"
DOCKER_VERSION: "20.10.3"
workflow:
rules: &workflow_rules
Loading
Loading
@@ -17,7 +20,7 @@ workflow:
- if: '$CI_COMMIT_TAG'
default:
image: golang:1.13
image: golang:1.14
tags:
- gitlab-org
Loading
Loading
@@ -25,51 +28,46 @@ default:
image: docker:${DOCKER_VERSION}
services:
- docker:${DOCKER_VERSION}-dind
variables:
DOCKER_DRIVER: overlay2
DOCKER_HOST: tcp://docker:2375
DOCKER_TLS_CERTDIR: ""
tags:
# See https://gitlab.com/gitlab-com/www-gitlab-com/-/issues/7019 for tag descriptions
- gitlab-org-docker
.test:
variables:
GITALY_CONNECTION_INFO: '{"address":"tcp://gitaly:8075", "storage":"default"}'
before_script:
# Set up the environment to run integration tests (still written in Ruby)
- apt-get update -qq && apt-get install -y ruby ruby-dev
- ruby -v
- export PATH=~/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/go/bin
- gem install --force --bindir /usr/local/bin bundler -v 2.1.4
- gem install --force --bindir /usr/local/bin bundler -v 2.3.6
- bundle install
# Now set up to run the Golang tests
- make build
- cp config.yml.example config.yml
- go version
- which go
services:
- name: registry.gitlab.com/gitlab-org/build/cng/gitaly:master
# Disable the hooks so we don't have to stub the GitLab API
command: ["bash", "-c", "mkdir -p /home/git/repositories && rm -rf /srv/gitlab-shell/hooks/* && exec /usr/bin/env GITALY_TESTING_NO_GIT_HOOKS=1 /scripts/process-wrapper"]
alias: gitaly
script:
- make verify test
go:1.13:
extends: .test
image: golang:1.13
go:1.14:
extends: .test
image: golang:1.14
after_script:
- make coverage
coverage: '/\d+.\d+%/'
go:1.15:
tests:
extends: .test
image: golang:1.15
image: golang:${GO_VERSION}
parallel:
matrix:
- GO_VERSION: ["1.16", "1.17", "1.18"]
after_script:
- make coverage
coverage: '/\d+.\d+%/'
race:
extends: .test
image: golang:1.15
image: golang:1.18
script:
- make test_golang_race
Loading
Loading
@@ -78,7 +76,7 @@ code_quality:
rules: *workflow_rules
code_navigation:
image: sourcegraph/lsif-go:v1
image: sourcegraph/lsif-go:v1.3.1
allow_failure: true
script:
- lsif-go
Loading
Loading
* @ashmckenzie @igor.drozdov @nick.thomas @patrickbajao
[Documentation]
*.md @aqualls
2.7.2
2.7.5
ruby 2.7.5
golang 1.17.7
v13.24.2
- Bump gitaly client !584
v13.24.1
- Default to info level for an empty log-level !579
- Update Gitaly dependency to v14.9.0-rc1 !578
- Reuse Gitaly connections and sidechannel !575
v13.24.0
- Upgrade golang to 1.17.7 !576
- Add more metrics for gitlab-sshd !574
- Move code guidelines to doc/beginners_guide.md !572
- Add docs for full feature list !571
- Add aqualls as codeowner for docs files !573
v13.23.2
- Bump labkit version to 1.12.0 !569
- Add title and correct copyright notice to license !568
- Bump go-proxyproto package !563
- Update Go to version 1.17.6 !562
v13.23.1
- Replace golang.org/x/crypto with gitlab-org/golang-crypto !560
v13.23.0
- Add support for SSHUploadPackWithSidechannel RPC !557
- Rate limiting documentation !556
v13.22.2
- Update to Ruby 2.7.5 !553
- Deprecate self_signed_cert config setting !552
- Send full git request/response in SSHD tests !550
- Suppress internal errors in client output !549
- Bump .tool_versions to use Go v1.16.12 !548
v13.22.1
- Remove SSL_CERT_DIR logging !546
v13.22.0
- Relax key and username matching for sshd !540
- Add logging to handler/exec.go and config/config.go !539
- Improve logging for non-git commands !538
- Update to Go v1.16.9 !537
- Reject non-proxied connections when proxy protocol is enabled !536
- Log command invocation !535
- Fix logging channel type !534
- Resolve an error-swallowing issue !533
- Add more logging to gitlab-sshd !531
- Respect log-level configuration again !530
- Improve err message given when Gitaly unavailable !526
- makefile: properly escape '$' in VERSION_STRING !525
- Add context fields to logging !524
- Extract server config related code out of sshd.go !523
- Add TestInvalidClientConfig and TestNewServerWithoutHosts for sshd.go !518
- Update Ruby version to 2.7.4 and add Go version 1.16.8 for tooling !517
v13.21.1
- Only validate SSL cert file exists if a value is supplied !527
v13.21.0
- Switch to labkit for logging system setup !504
- Remove some unreliable tests !503
- Make gofmt check fail if there are any matching files !500
- Update go-proxyproto to v0.6.0 !499
- Switch to labkit/log for logging functionality !498
- Unit tests for internal/sshd/connection.go !497
- Prometheus metrics for HTTP requests !496
- Refactor testhelper.PrepareTestRootDir using t.Cleanup !493
- Change default logging format to JSON !476
- Shutdown sshd gracefully !484
- Provide liveness and readiness probes !494
- Add tracing instrumentation to http client !495
- Log same correlation_id on auth keys check of ssh connections !501
- fix: validate client cert paths exist on disk before proceeding !508
- Modify regex to prevent partial matches
v13.20.0
- Remove bin/authorized_keys !491
- Add a make install command !490
- Create PROCESS.md page with Security release process !488
- Fix the Geo SSH push proxy hanging !487
- Standardize logging timestamp format !485
v13.19.1
- Modify regex to prevent partial matches
v13.19.0
- Don't finish the opentracing span early !466
- gitlab-sshd: Respect the ssl_cert_dir config !467
- Stop changing directory to the filesystem root !470
- Fix opentracing setup for gitlab-sshd !473
v13.18.1
- Modify regex to prevent partial matches
v13.18.0
- Fix thread-safety issues in gitlab-shell !463
- gitlab-sshd: Support the PROXY protocol !461
- sshd: Recover from per-session and per-connection panics !464
v13.17.0
- Fix gitlab-shell panic when log file not writable !453
- Add monitoring endpoint to built-in SSH server !449
v13.16.1
- Read limited input when asking to generate new two-factor recovery codes
Loading
Loading
# frozen_string_literal: true
require 'gitlab-dangerfiles'
Gitlab::Dangerfiles.for_project(self) do |gitlab_dangerfiles|
gitlab_dangerfiles.import_plugins
gitlab_dangerfiles.import_dangerfiles(except: %w[changelog commit_messages simple_roulette])
end
Loading
Loading
@@ -3,3 +3,7 @@ source 'https://rubygems.org'
group :development, :test do
gem 'rspec', '~> 3.8.0'
end
group :development, :danger do
gem 'gitlab-dangerfiles', '~> 3.0.0'
end
GEM
remote: https://rubygems.org/
specs:
addressable (2.8.0)
public_suffix (>= 2.0.2, < 5.0)
claide (1.1.0)
claide-plugins (0.9.2)
cork
nap
open4 (~> 1.3)
colored2 (3.1.2)
cork (0.3.0)
colored2 (~> 3.1)
danger (8.5.0)
claide (~> 1.0)
claide-plugins (>= 0.9.2)
colored2 (~> 3.1)
cork (~> 0.1)
faraday (>= 0.9.0, < 2.0)
faraday-http-cache (~> 2.0)
git (~> 1.7)
kramdown (~> 2.3)
kramdown-parser-gfm (~> 1.0)
no_proxy_fix
octokit (~> 4.7)
terminal-table (>= 1, < 4)
danger-gitlab (8.0.0)
danger
gitlab (~> 4.2, >= 4.2.0)
diff-lcs (1.3)
faraday (1.10.0)
faraday-em_http (~> 1.0)
faraday-em_synchrony (~> 1.0)
faraday-excon (~> 1.1)
faraday-httpclient (~> 1.0)
faraday-multipart (~> 1.0)
faraday-net_http (~> 1.0)
faraday-net_http_persistent (~> 1.0)
faraday-patron (~> 1.0)
faraday-rack (~> 1.0)
faraday-retry (~> 1.0)
ruby2_keywords (>= 0.0.4)
faraday-em_http (1.0.0)
faraday-em_synchrony (1.0.0)
faraday-excon (1.1.0)
faraday-http-cache (2.2.0)
faraday (>= 0.8)
faraday-httpclient (1.0.1)
faraday-multipart (1.0.3)
multipart-post (>= 1.2, < 3)
faraday-net_http (1.0.1)
faraday-net_http_persistent (1.2.0)
faraday-patron (1.0.0)
faraday-rack (1.0.0)
faraday-retry (1.0.3)
git (1.10.2)
rchardet (~> 1.8)
gitlab (4.18.0)
httparty (~> 0.18)
terminal-table (>= 1.5.1)
gitlab-dangerfiles (3.0.0)
danger (>= 8.4.5)
danger-gitlab (>= 8.0.0)
rake
httparty (0.20.0)
mime-types (~> 3.0)
multi_xml (>= 0.5.2)
kramdown (2.3.2)
rexml
kramdown-parser-gfm (1.1.0)
kramdown (~> 2.0)
mime-types (3.4.1)
mime-types-data (~> 3.2015)
mime-types-data (3.2022.0105)
multi_xml (0.6.0)
multipart-post (2.1.1)
nap (1.1.0)
no_proxy_fix (0.1.2)
octokit (4.22.0)
faraday (>= 0.9)
sawyer (~> 0.8.0, >= 0.5.3)
open4 (1.3.4)
public_suffix (4.0.6)
rake (13.0.6)
rchardet (1.8.0)
rexml (3.2.5)
rspec (3.8.0)
rspec-core (~> 3.8.0)
rspec-expectations (~> 3.8.0)
Loading
Loading
@@ -15,12 +97,20 @@ GEM
diff-lcs (>= 1.2.0, < 2.0)
rspec-support (~> 3.8.0)
rspec-support (3.8.0)
ruby2_keywords (0.0.5)
sawyer (0.8.2)
addressable (>= 2.3.5)
faraday (> 0.8, < 2.0)
terminal-table (3.0.2)
unicode-display_width (>= 1.1.1, < 3)
unicode-display_width (2.1.0)
PLATFORMS
ruby
DEPENDENCIES
gitlab-dangerfiles (~> 3.0.0)
rspec (~> 3.8.0)
BUNDLED WITH
2.1.4
2.3.6
Copyright (c) 2011-2018 GitLab B.V.
MIT License
With regard to the GitLab Software:
Copyright (c) 2011-present GitLab B.V.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
Loading
Loading
@@ -9,17 +9,13 @@ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
For all third party components incorporated into the GitLab Software, those
components are licensed under the original license provided by the owner of the
applicable component.
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
.PHONY: validate verify verify_ruby verify_golang test test_ruby test_golang coverage coverage_golang setup _install build compile check clean
.PHONY: validate verify verify_ruby verify_golang test test_ruby test_golang coverage coverage_golang setup _script_install build compile check clean install
GO_SOURCES := $(shell find . -name '*.go')
VERSION_STRING := $(shell git describe --match v* 2>/dev/null || awk '$0="v"$0' VERSION 2>/dev/null || echo unknown)
VERSION_STRING := $(shell git describe --match v* 2>/dev/null || awk '$$0="v"$$0' VERSION 2>/dev/null || echo unknown)
BUILD_TIME := $(shell date -u +%Y%m%d.%H%M%S)
GOBUILD_FLAGS := -ldflags "-X main.Version=$(VERSION_STRING) -X main.BuildTime=$(BUILD_TIME)"
BUILD_TAGS := tracer_static tracer_static_jaeger continuous_profiler_stackdriver
GOBUILD_FLAGS := -ldflags "-X main.Version=$(VERSION_STRING) -X main.BuildTime=$(BUILD_TIME)" -tags "$(BUILD_TAGS)" -mod=mod
PREFIX ?= /usr/local
build: bin/gitlab-shell
validate: verify test
verify: verify_golang
verify_golang:
gofmt -s -l $(GO_SOURCES)
gofmt -s -l $(GO_SOURCES) | awk '{ print } END { if (NR > 0) { print "Please run make fmt"; exit 1 } }'
fmt:
gofmt -w -s $(GO_SOURCES)
test: test_ruby test_golang
Loading
Loading
@@ -29,12 +37,11 @@ coverage: coverage_golang
coverage_golang:
[ -f cover.out ] && go tool cover -func cover.out
setup: _install bin/gitlab-shell
setup: _script_install bin/gitlab-shell
_install:
_script_install:
bin/install
build: bin/gitlab-shell
compile: bin/gitlab-shell
bin/gitlab-shell: $(GO_SOURCES)
GOBIN="$(CURDIR)/bin" go install $(GOBUILD_FLAGS) ./cmd/...
Loading
Loading
@@ -43,4 +50,13 @@ check:
bin/check
clean:
rm -f bin/check bin/gitlab-shell bin/gitlab-shell-authorized-keys-check bin/gitlab-shell-authorized-principals-check
rm -f bin/check bin/gitlab-shell bin/gitlab-shell-authorized-keys-check bin/gitlab-shell-authorized-principals-check bin/gitlab-sshd
install: compile
mkdir -p $(DESTDIR)$(PREFIX)/bin/
install -m755 bin/check $(DESTDIR)$(PREFIX)/bin/check
install -m755 bin/gitlab-shell $(DESTDIR)$(PREFIX)/bin/gitlab-shell
install -m755 bin/gitlab-shell $(DESTDIR)$(PREFIX)/bin/gitlab-shell-authorized-keys-check
install -m755 bin/gitlab-shell $(DESTDIR)$(PREFIX)/bin/gitlab-shell-authorized-principals-check
install -m755 bin/gitlab-shell $(DESTDIR)$(PREFIX)/bin/gitlab-sshd
## Releasing a new version
GitLab Shell is versioned by git tags, and the version used by the Rails
application is stored in
[`GITLAB_SHELL_VERSION`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/GITLAB_SHELL_VERSION).
For each version, there is a raw version and a tag version:
- The **raw version** is the version number. For instance, `15.2.8`.
- The **tag version** is the raw version prefixed with `v`. For instance, `v15.2.8`.
To release a new version of GitLab Shell and have that version available to the
Rails application:
1. Create a merge request to update the [`CHANGELOG`](CHANGELOG) with the
**tag version** and the [`VERSION`](VERSION) file with the **raw version**.
2. Ask a maintainer to review and merge the merge request. If you're already a
maintainer, second maintainer review is not required.
3. Add a new git tag with the **tag version**.
4. Update `GITLAB_SHELL_VERSION` in the Rails application to the **raw
version**. (Note: this can be done as a separate MR to that, or in and MR
that will make use of the latest GitLab Shell changes.)
## Security releases
GitLab Shell is included in the packages we create for GitLab, and each version of GitLab specifies the version of GitLab Shell it uses in the `GITLAB_SHELL_VERSION` file, so security fixes in GitLab Shell are tightly coupled to the [GitLab security release](https://about.gitlab.com/handbook/engineering/workflow/#security-issues) workflow.
For a security fix in GitLab Shell, two sets of merge requests are required:
* The fix itself, in the `gitlab-org/security/gitlab-shell` repository and its backports to the previous versions of GitLab Shell
* Merge requests to change the versions of GitLab Shell included in the GitLab security release, in the `gitlab-org/security/gitlab` repository
The first step could be to create a merge request with a fix targeting `main` in `gitlab-org/security/gitlab-shell`. When the merge request is approved by maintainers, backports targeting previous 3 versions of GitLab Shell must be created. The stable branches for those versions may not exist, feel free to ask a maintainer to create ones. The stable branches must be created out of the GitLab Shell tags/version used by the 3 previous GitLab releases. In order to find out the GitLab Shell version that is used on a particular GitLab stable release, the following steps may be helpful:
```shell
git fetch security 13-9-stable-ee
git show refs/remotes/security/13-9-stable-ee:GITLAB_SHELL_VERSION
```
These steps display the version that is used by `13.9` version of GitLab.
Close to the GitLab security release, a maintainer should merge the fix and backports and cut all the necessary GitLab Shell versions. This allows bumping the `GITLAB_SHELL_VERSION` for `gitlab-org/security/gitlab`. The GitLab merge request will be handled by the general GitLab security release process.
Once the security release is done, a GitLab Shell maintainer is responsible for syncing tags and `main` to the `gitlab-org/gitlab-shell` repository.
---
stage: Create
group: Source Code
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
---
# GitLab Shell
## GitLab Shell handles git SSH sessions for GitLab
Loading
Loading
@@ -7,17 +13,19 @@ GitLab Shell is not a Unix shell nor a replacement for Bash or Zsh.
When you access the GitLab server over SSH then GitLab Shell will:
1. Limits you to predefined git commands (git push, git pull).
1. Limit you to predefined git commands (git push, git pull).
1. Call the GitLab Rails API to check if you are authorized, and what Gitaly server your repository is on
1. Copy data back and forth between the SSH client and the Gitaly server
If you access a GitLab server over HTTP(S) you end up in [gitlab-workhorse](https://gitlab.com/gitlab-org/gitlab-workhorse).
If you access a GitLab server over HTTP(S) you end up in [gitlab-workhorse](https://gitlab.com/gitlab-org/gitlab/tree/master/workhorse).
An overview of the four cases described above:
1. git pull over SSH -> gitlab-shell -> API call to gitlab-rails (Authorization) -> accept or decline -> establish Gitaly session
1. git push over SSH -> gitlab-shell (git command is not executed yet) -> establish Gitaly session -> (in Gitaly) gitlab-shell pre-receive hook -> API call to gitlab-rails (authorization) -> accept or decline push
[Full feature list](doc/features.md)
## Code status
[![pipeline status](https://gitlab.com/gitlab-org/gitlab-shell/badges/main/pipeline.svg)](https://gitlab.com/gitlab-org/gitlab-shell/-/pipelines?ref=main)
Loading
Loading
@@ -31,62 +39,27 @@ Ruby to build and test, but not to run.
Download and install the current version of Go from https://golang.org/dl/
## Setup
make setup
## Check
Checks if GitLab API access and redis via internal API can be reached:
make check
## Testing
Run tests:
bundle install
make test
Run gofmt:
make verify
Run both test and verify (the default Makefile target):
bundle install
make validate
## Git LFS remark
We follow the [Golang Release Policy](https://golang.org/doc/devel/release.html#policy)
of supporting the current stable version and the previous two major versions.
Starting with GitLab 8.12, GitLab supports Git LFS authentication through SSH.
## Rate Limiting
## Releasing a new version
GitLab Shell performs rate-limiting by user account and project for git operations. GitLab Shell accepts git operation requests and then makes a call to the Rails rate-limiter (backed by Redis). If the `user + project` exceeds the rate limit then GitLab Shell will then drop further connection requests for that `user + project`.
GitLab Shell is versioned by git tags, and the version used by the Rails
application is stored in
[`GITLAB_SHELL_VERSION`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/GITLAB_SHELL_VERSION).
The rate-limiter is applied at the git command (plumbing) level. Each command has a rate limit of 600/minute. For example, `git push` has 600/minute and `git pull` has another 600/minute.
For each version, there is a raw version and a tag version:
Because they are using the same plumbing command `git-upload-pack`, `git pull` and `git clone` are in effect the same command for the purposes of rate-limiting.
- The **raw version** is the version number. For instance, `15.2.8`.
- The **tag version** is the raw version prefixed with `v`. For instance, `v15.2.8`.
There is also a rate-limiter in place in Gitaly, but the calls will never be made to Gitaly if the rate limit is exceeded in Gitlab Shell (Rails).
To release a new version of GitLab Shell and have that version available to the
Rails application:
## Releasing
1. Create a merge request to update the [`CHANGELOG`](CHANGELOG) with the
**tag version** and the [`VERSION`](VERSION) file with the **raw version**.
2. Ask a maintainer to review and merge the merge request. If you're already a
maintainer, second maintainer review is not required.
3. Add a new git tag with the **tag version**.
4. Update `GITLAB_SHELL_VERSION` in the Rails application to the **raw
version**. (Note: this can be done as a separate MR to that, or in and MR
that will make use of the latest GitLab Shell changes.)
See [PROCESS.md](./PROCESS.md)
## Contributing
See [CONTRIBUTING.md](./CONTRIBUTING.md).
- See [CONTRIBUTING.md](./CONTRIBUTING.md).
- See the [beginner's guide](doc/beginners_guide.md).
## License
Loading
Loading
13.16.1
13.24.2
#!/bin/sh
# Legacy script used for AuthorizedKeysCommand when configured without username.
# Executes gitlab-shell-authorized-keys-check with "git" as expected and actual
# username and with the passed key.
#
# TODO: Remove this in https://gitlab.com/gitlab-org/gitlab-shell/issues/209.
$(dirname $0)/gitlab-shell-authorized-keys-check git git $1
Loading
Loading
@@ -5,28 +5,26 @@ import (
"encoding/base64"
"encoding/json"
"fmt"
"io/ioutil"
"io"
"net/http"
"path"
"strings"
"testing"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/require"
"gitlab.com/gitlab-org/gitlab-shell/client/testserver"
"gitlab.com/gitlab-org/gitlab-shell/internal/testhelper"
)
func TestClients(t *testing.T) {
testDirCleanup, err := testhelper.PrepareTestRootDir()
require.NoError(t, err)
defer testDirCleanup()
testhelper.PrepareTestRootDir(t)
testCases := []struct {
desc string
relativeURLRoot string
caFile string
server func(*testing.T, []testserver.TestRequestHandler) (string, func())
server func(*testing.T, []testserver.TestRequestHandler) string
}{
{
desc: "Socket client",
Loading
Loading
@@ -49,7 +47,7 @@ func TestClients(t *testing.T) {
{
desc: "Https client",
caFile: path.Join(testhelper.TestRoot, "certs/valid/server.crt"),
server: func(t *testing.T, handlers []testserver.TestRequestHandler) (string, func()) {
server: func(t *testing.T, handlers []testserver.TestRequestHandler) string {
return testserver.StartHttpsServer(t, handlers, "")
},
},
Loading
Loading
@@ -57,12 +55,12 @@ func TestClients(t *testing.T) {
for _, tc := range testCases {
t.Run(tc.desc, func(t *testing.T) {
url, cleanup := tc.server(t, buildRequests(t, tc.relativeURLRoot))
defer cleanup()
url := tc.server(t, buildRequests(t, tc.relativeURLRoot))
secret := "sssh, it's a secret"
httpClient := NewHTTPClient(url, tc.relativeURLRoot, tc.caFile, "", false, 1)
httpClient, err := NewHTTPClientWithOpts(url, tc.relativeURLRoot, tc.caFile, "", false, 1, nil)
require.NoError(t, err)
client, err := NewGitlabNetClient("", "", secret, httpClient)
require.NoError(t, err)
Loading
Loading
@@ -79,32 +77,20 @@ func TestClients(t *testing.T) {
func testSuccessfulGet(t *testing.T, client *GitlabNetClient) {
t.Run("Successful get", func(t *testing.T) {
hook := testhelper.SetupLogger()
response, err := client.Get(context.Background(), "/hello")
require.NoError(t, err)
require.NotNil(t, response)
defer response.Body.Close()
responseBody, err := ioutil.ReadAll(response.Body)
responseBody, err := io.ReadAll(response.Body)
require.NoError(t, err)
require.Equal(t, string(responseBody), "Hello")
require.True(t, testhelper.WaitForLogEvent(hook))
entries := hook.AllEntries()
require.Equal(t, 1, len(entries))
require.Equal(t, logrus.InfoLevel, entries[0].Level)
require.Contains(t, entries[0].Message, "method=GET")
require.Contains(t, entries[0].Message, "status=200")
require.Contains(t, entries[0].Message, "content_length_bytes=")
require.Contains(t, entries[0].Message, "Finished HTTP request")
require.Contains(t, entries[0].Message, "correlation_id=")
})
}
func testSuccessfulPost(t *testing.T, client *GitlabNetClient) {
t.Run("Successful Post", func(t *testing.T) {
hook := testhelper.SetupLogger()
data := map[string]string{"key": "value"}
response, err := client.Post(context.Background(), "/post_endpoint", data)
Loading
Loading
@@ -113,53 +99,23 @@ func testSuccessfulPost(t *testing.T, client *GitlabNetClient) {
defer response.Body.Close()
responseBody, err := ioutil.ReadAll(response.Body)
responseBody, err := io.ReadAll(response.Body)
require.NoError(t, err)
require.Equal(t, "Echo: {\"key\":\"value\"}", string(responseBody))
require.True(t, testhelper.WaitForLogEvent(hook))
entries := hook.AllEntries()
require.Equal(t, 1, len(entries))
require.Equal(t, logrus.InfoLevel, entries[0].Level)
require.Contains(t, entries[0].Message, "method=POST")
require.Contains(t, entries[0].Message, "status=200")
require.Contains(t, entries[0].Message, "content_length_bytes=")
require.Contains(t, entries[0].Message, "Finished HTTP request")
require.Contains(t, entries[0].Message, "correlation_id=")
})
}
func testMissing(t *testing.T, client *GitlabNetClient) {
t.Run("Missing error for GET", func(t *testing.T) {
hook := testhelper.SetupLogger()
response, err := client.Get(context.Background(), "/missing")
require.EqualError(t, err, "Internal API error (404)")
require.Nil(t, response)
require.True(t, testhelper.WaitForLogEvent(hook))
entries := hook.AllEntries()
require.Equal(t, 1, len(entries))
require.Equal(t, logrus.InfoLevel, entries[0].Level)
require.Contains(t, entries[0].Message, "method=GET")
require.Contains(t, entries[0].Message, "status=404")
require.Contains(t, entries[0].Message, "Internal API error")
require.Contains(t, entries[0].Message, "correlation_id=")
})
t.Run("Missing error for POST", func(t *testing.T) {
hook := testhelper.SetupLogger()
response, err := client.Post(context.Background(), "/missing", map[string]string{})
require.EqualError(t, err, "Internal API error (404)")
require.Nil(t, response)
require.True(t, testhelper.WaitForLogEvent(hook))
entries := hook.AllEntries()
require.Equal(t, 1, len(entries))
require.Equal(t, logrus.InfoLevel, entries[0].Level)
require.Contains(t, entries[0].Message, "method=POST")
require.Contains(t, entries[0].Message, "status=404")
require.Contains(t, entries[0].Message, "Internal API error")
require.Contains(t, entries[0].Message, "correlation_id=")
})
}
Loading
Loading
@@ -179,37 +135,15 @@ func testErrorMessage(t *testing.T, client *GitlabNetClient) {
func testBrokenRequest(t *testing.T, client *GitlabNetClient) {
t.Run("Broken request for GET", func(t *testing.T) {
hook := testhelper.SetupLogger()
response, err := client.Get(context.Background(), "/broken")
require.EqualError(t, err, "Internal API unreachable")
require.Nil(t, response)
require.True(t, testhelper.WaitForLogEvent(hook))
entries := hook.AllEntries()
require.Equal(t, 1, len(entries))
require.Equal(t, logrus.InfoLevel, entries[0].Level)
require.Contains(t, entries[0].Message, "method=GET")
require.NotContains(t, entries[0].Message, "status=")
require.Contains(t, entries[0].Message, "Internal API unreachable")
require.Contains(t, entries[0].Message, "correlation_id=")
})
t.Run("Broken request for POST", func(t *testing.T) {
hook := testhelper.SetupLogger()
response, err := client.Post(context.Background(), "/broken", map[string]string{})
require.EqualError(t, err, "Internal API unreachable")
require.Nil(t, response)
require.True(t, testhelper.WaitForLogEvent(hook))
entries := hook.AllEntries()
require.Equal(t, 1, len(entries))
require.Equal(t, logrus.InfoLevel, entries[0].Level)
require.Contains(t, entries[0].Message, "method=POST")
require.NotContains(t, entries[0].Message, "status=")
require.Contains(t, entries[0].Message, "Internal API unreachable")
require.Contains(t, entries[0].Message, "correlation_id=")
})
}
Loading
Loading
@@ -221,7 +155,7 @@ func testAuthenticationHeader(t *testing.T, client *GitlabNetClient) {
defer response.Body.Close()
responseBody, err := ioutil.ReadAll(response.Body)
responseBody, err := io.ReadAll(response.Body)
require.NoError(t, err)
header, err := base64.StdEncoding.DecodeString(string(responseBody))
Loading
Loading
@@ -236,7 +170,7 @@ func testAuthenticationHeader(t *testing.T, client *GitlabNetClient) {
defer response.Body.Close()
responseBody, err := ioutil.ReadAll(response.Body)
responseBody, err := io.ReadAll(response.Body)
require.NoError(t, err)
header, err := base64.StdEncoding.DecodeString(string(responseBody))
Loading
Loading
@@ -260,7 +194,7 @@ func buildRequests(t *testing.T, relativeURLRoot string) []testserver.TestReques
Handler: func(w http.ResponseWriter, r *http.Request) {
require.Equal(t, http.MethodPost, r.Method)
b, err := ioutil.ReadAll(r.Body)
b, err := io.ReadAll(r.Body)
defer r.Body.Close()
require.NoError(t, err)
Loading
Loading
Loading
Loading
@@ -11,9 +11,7 @@ import (
"strings"
"time"
"gitlab.com/gitlab-org/labkit/correlation"
log "github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/labkit/log"
)
const (
Loading
Loading
@@ -71,12 +69,12 @@ func normalizePath(path string) string {
return path
}
func newRequest(ctx context.Context, method, host, path string, data interface{}) (*http.Request, string, error) {
func newRequest(ctx context.Context, method, host, path string, data interface{}) (*http.Request, error) {
var jsonReader io.Reader
if data != nil {
jsonData, err := json.Marshal(data)
if err != nil {
return nil, "", err
return nil, err
}
jsonReader = bytes.NewReader(jsonData)
Loading
Loading
@@ -84,12 +82,10 @@ func newRequest(ctx context.Context, method, host, path string, data interface{}
request, err := http.NewRequestWithContext(ctx, method, host+path, jsonReader)
if err != nil {
return nil, "", err
return nil, err
}
correlationID := correlation.ExtractFromContext(ctx)
return request, correlationID, nil
return request, nil
}
func parseError(resp *http.Response) error {
Loading
Loading
@@ -116,7 +112,7 @@ func (c *GitlabNetClient) Post(ctx context.Context, path string, data interface{
}
func (c *GitlabNetClient) DoRequest(ctx context.Context, method, path string, data interface{}) (*http.Response, error) {
request, correlationID, err := newRequest(ctx, method, c.httpClient.Host, path, data)
request, err := newRequest(ctx, method, c.httpClient.Host, path, data)
if err != nil {
return nil, err
}
Loading
Loading
@@ -136,12 +132,11 @@ func (c *GitlabNetClient) DoRequest(ctx context.Context, method, path string, da
start := time.Now()
response, err := c.httpClient.Do(request)
fields := log.Fields{
"correlation_id": correlationID,
"method": method,
"url": request.URL.String(),
"duration_ms": time.Since(start) / time.Millisecond,
"method": method,
"url": request.URL.String(),
"duration_ms": time.Since(start) / time.Millisecond,
}
logger := log.WithFields(fields)
logger := log.WithContextFields(ctx, fields)
if err != nil {
logger.WithError(err).Error("Internal API unreachable")
Loading
Loading
Loading
Loading
@@ -5,15 +5,17 @@ import (
"crypto/tls"
"crypto/x509"
"errors"
"io/ioutil"
"fmt"
"net"
"net/http"
"os"
"path/filepath"
"strings"
"time"
log "github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/labkit/correlation"
"gitlab.com/gitlab-org/labkit/log"
"gitlab.com/gitlab-org/labkit/tracing"
)
const (
Loading
Loading
@@ -24,6 +26,10 @@ const (
defaultReadTimeoutSeconds = 300
)
var (
ErrCafileNotFound = errors.New("cafile not found")
)
type HttpClient struct {
*http.Client
Host string
Loading
Loading
@@ -48,6 +54,22 @@ func WithClientCert(certPath, keyPath string) HTTPClientOpt {
}
}
func validateCaFile(filename string) error {
if filename == "" {
return nil
}
if _, err := os.Stat(filename); err != nil {
if os.IsNotExist(err) {
return fmt.Errorf("cannot find cafile '%s': %w", filename, ErrCafileNotFound)
}
return err
}
return nil
}
// Deprecated: use NewHTTPClientWithOpts - https://gitlab.com/gitlab-org/gitlab-shell/-/issues/484
func NewHTTPClient(gitlabURL, gitlabRelativeURLRoot, caFile, caPath string, selfSignedCert bool, readTimeoutSeconds uint64) *HttpClient {
c, err := NewHTTPClientWithOpts(gitlabURL, gitlabRelativeURLRoot, caFile, caPath, selfSignedCert, readTimeoutSeconds, nil)
Loading
Loading
@@ -59,15 +81,6 @@ func NewHTTPClient(gitlabURL, gitlabRelativeURLRoot, caFile, caPath string, self
// NewHTTPClientWithOpts builds an HTTP client using the provided options
func NewHTTPClientWithOpts(gitlabURL, gitlabRelativeURLRoot, caFile, caPath string, selfSignedCert bool, readTimeoutSeconds uint64, opts []HTTPClientOpt) (*HttpClient, error) {
hcc := &httpClientCfg{
caFile: caFile,
caPath: caPath,
}
for _, opt := range opts {
opt(hcc)
}
var transport *http.Transport
var host string
var err error
Loading
Loading
@@ -76,6 +89,20 @@ func NewHTTPClientWithOpts(gitlabURL, gitlabRelativeURLRoot, caFile, caPath stri
} else if strings.HasPrefix(gitlabURL, httpProtocol) {
transport, host = buildHttpTransport(gitlabURL)
} else if strings.HasPrefix(gitlabURL, httpsProtocol) {
err = validateCaFile(caFile)
if err != nil {
return nil, err
}
hcc := &httpClientCfg{
caFile: caFile,
caPath: caPath,
}
for _, opt := range opts {
opt(hcc)
}
transport, host, err = buildHttpsTransport(*hcc, selfSignedCert, gitlabURL)
if err != nil {
return nil, err
Loading
Loading
@@ -85,7 +112,7 @@ func NewHTTPClientWithOpts(gitlabURL, gitlabRelativeURLRoot, caFile, caPath stri
}
c := &http.Client{
Transport: correlation.NewInstrumentedRoundTripper(transport),
Transport: correlation.NewInstrumentedRoundTripper(tracing.NewRoundTripper(transport)),
Timeout: readTimeout(readTimeoutSeconds),
}
Loading
Loading
@@ -125,7 +152,7 @@ func buildHttpsTransport(hcc httpClientCfg, selfSignedCert bool, gitlabURL strin
}
if hcc.caPath != "" {
fis, _ := ioutil.ReadDir(hcc.caPath)
fis, _ := os.ReadDir(hcc.caPath)
for _, fi := range fis {
if fi.IsDir() {
continue
Loading
Loading
@@ -135,7 +162,10 @@ func buildHttpsTransport(hcc httpClientCfg, selfSignedCert bool, gitlabURL strin
}
}
tlsConfig := &tls.Config{
RootCAs: certPool,
RootCAs: certPool,
// The self_signed_cert config setting is deprecated
// The field and its usage is going to be removed in
// https://gitlab.com/gitlab-org/gitlab-shell/-/issues/541
InsecureSkipVerify: selfSignedCert,
MinVersion: tls.VersionTLS12,
}
Loading
Loading
@@ -146,7 +176,6 @@ func buildHttpsTransport(hcc httpClientCfg, selfSignedCert bool, gitlabURL strin
return nil, "", err
}
tlsConfig.Certificates = []tls.Certificate{cert}
tlsConfig.BuildNameToCertificate()
}
transport := &http.Transport{
Loading
Loading
@@ -157,7 +186,7 @@ func buildHttpsTransport(hcc httpClientCfg, selfSignedCert bool, gitlabURL strin
}
func addCertToPool(certPool *x509.CertPool, fileName string) {
cert, err := ioutil.ReadFile(fileName)
cert, err := os.ReadFile(fileName)
if err == nil {
certPool.AppendCertsFromPEM(cert)
}
Loading
Loading
Loading
Loading
@@ -4,7 +4,7 @@ import (
"context"
"encoding/base64"
"fmt"
"io/ioutil"
"io"
"net/http"
"strings"
"testing"
Loading
Loading
@@ -17,7 +17,8 @@ import (
func TestReadTimeout(t *testing.T) {
expectedSeconds := uint64(300)
client := NewHTTPClient("http://localhost:3000", "", "", "", false, expectedSeconds)
client, err := NewHTTPClientWithOpts("http://localhost:3000", "", "", "", false, expectedSeconds, nil)
require.NoError(t, err)
require.NotNil(t, client)
require.Equal(t, time.Duration(expectedSeconds)*time.Second, client.Client.Timeout)
Loading
Loading
@@ -48,8 +49,7 @@ func TestBasicAuthSettings(t *testing.T) {
},
}
client, cleanup := setup(t, username, password, requests)
defer cleanup()
client := setup(t, username, password, requests)
response, err := client.Get(context.Background(), "/get_endpoint")
require.NoError(t, err)
Loading
Loading
@@ -64,7 +64,7 @@ func testBasicAuthHeaders(t *testing.T, response *http.Response) {
defer response.Body.Close()
require.NotNil(t, response)
responseBody, err := ioutil.ReadAll(response.Body)
responseBody, err := io.ReadAll(response.Body)
require.NoError(t, err)
headerParts := strings.Split(string(responseBody), " ")
Loading
Loading
@@ -86,8 +86,7 @@ func TestEmptyBasicAuthSettings(t *testing.T) {
},
}
client, cleanup := setup(t, "", "", requests)
defer cleanup()
client := setup(t, "", "", requests)
_, err := client.Get(context.Background(), "/empty_basic_auth")
require.NoError(t, err)
Loading
Loading
@@ -110,8 +109,7 @@ func TestRequestWithUserAgent(t *testing.T) {
},
}
client, cleanup := setup(t, "", "", requests)
defer cleanup()
client := setup(t, "", "", requests)
_, err := client.Get(context.Background(), "/default_user_agent")
require.NoError(t, err)
Loading
Loading
@@ -122,13 +120,14 @@ func TestRequestWithUserAgent(t *testing.T) {
}
func setup(t *testing.T, username, password string, requests []testserver.TestRequestHandler) (*GitlabNetClient, func()) {
url, cleanup := testserver.StartHttpServer(t, requests)
func setup(t *testing.T, username, password string, requests []testserver.TestRequestHandler) *GitlabNetClient {
url := testserver.StartHttpServer(t, requests)
httpClient := NewHTTPClient(url, "", "", "", false, 1)
httpClient, err := NewHTTPClientWithOpts(url, "", "", "", false, 1, nil)
require.NoError(t, err)
client, err := NewGitlabNetClient(username, password, "", httpClient)
require.NoError(t, err)
return client, cleanup
return client
}
Loading
Loading
@@ -3,7 +3,7 @@ package client
import (
"context"
"fmt"
"io/ioutil"
"io"
"net/http"
"path"
"testing"
Loading
Loading
@@ -28,10 +28,7 @@ func TestSuccessfulRequests(t *testing.T) {
{
desc: "Valid CaPath",
caPath: path.Join(testhelper.TestRoot, "certs/valid"),
},
{
desc: "Self signed cert option enabled",
selfSigned: true,
caFile: path.Join(testhelper.TestRoot, "certs/valid/server.crt"),
},
{
desc: "Invalid cert with self signed cert option enabled",
Loading
Loading
@@ -51,8 +48,8 @@ func TestSuccessfulRequests(t *testing.T) {
for _, tc := range testCases {
t.Run(tc.desc, func(t *testing.T) {
client, cleanup := setupWithRequests(t, tc.caFile, tc.caPath, tc.clientCAPath, tc.clientCertPath, tc.clientKeyPath, tc.selfSigned)
defer cleanup()
client, err := setupWithRequests(t, tc.caFile, tc.caPath, tc.clientCAPath, tc.clientCertPath, tc.clientKeyPath, tc.selfSigned)
require.NoError(t, err)
response, err := client.Get(context.Background(), "/hello")
require.NoError(t, err)
Loading
Loading
@@ -60,7 +57,7 @@ func TestSuccessfulRequests(t *testing.T) {
defer response.Body.Close()
responseBody, err := ioutil.ReadAll(response.Body)
responseBody, err := io.ReadAll(response.Body)
require.NoError(t, err)
require.Equal(t, string(responseBody), "Hello")
})
Loading
Loading
@@ -69,40 +66,51 @@ func TestSuccessfulRequests(t *testing.T) {
func TestFailedRequests(t *testing.T) {
testCases := []struct {
desc string
caFile string
caPath string
desc string
caFile string
caPath string
expectedCaFileNotFound bool
expectedError string
}{
{
desc: "Invalid CaFile",
caFile: path.Join(testhelper.TestRoot, "certs/invalid/server.crt"),
desc: "Invalid CaFile",
caFile: path.Join(testhelper.TestRoot, "certs/invalid/server.crt"),
expectedError: "Internal API unreachable",
},
{
desc: "Invalid CaPath",
caPath: path.Join(testhelper.TestRoot, "certs/invalid"),
desc: "Missing CaFile",
caFile: path.Join(testhelper.TestRoot, "certs/invalid/missing.crt"),
expectedCaFileNotFound: true,
},
{
desc: "Empty config",
desc: "Invalid CaPath",
caPath: path.Join(testhelper.TestRoot, "certs/invalid"),
expectedError: "Internal API unreachable",
},
{
desc: "Empty config",
expectedError: "Internal API unreachable",
},
}
for _, tc := range testCases {
t.Run(tc.desc, func(t *testing.T) {
client, cleanup := setupWithRequests(t, tc.caFile, tc.caPath, "", "", "", false)
defer cleanup()
_, err := client.Get(context.Background(), "/hello")
require.Error(t, err)
require.Equal(t, err.Error(), "Internal API unreachable")
client, err := setupWithRequests(t, tc.caFile, tc.caPath, "", "", "", false)
if tc.expectedCaFileNotFound {
require.Error(t, err)
require.ErrorIs(t, err, ErrCafileNotFound)
} else {
_, err = client.Get(context.Background(), "/hello")
require.Error(t, err)
require.Equal(t, err.Error(), tc.expectedError)
}
})
}
}
func setupWithRequests(t *testing.T, caFile, caPath, clientCAPath, clientCertPath, clientKeyPath string, selfSigned bool) (*GitlabNetClient, func()) {
testDirCleanup, err := testhelper.PrepareTestRootDir()
require.NoError(t, err)
defer testDirCleanup()
func setupWithRequests(t *testing.T, caFile, caPath, clientCAPath, clientCertPath, clientKeyPath string, selfSigned bool) (*GitlabNetClient, error) {
testhelper.PrepareTestRootDir(t)
requests := []testserver.TestRequestHandler{
{
Loading
Loading
@@ -115,7 +123,7 @@ func setupWithRequests(t *testing.T, caFile, caPath, clientCAPath, clientCertPat
},
}
url, cleanup := testserver.StartHttpsServer(t, requests, clientCAPath)
url := testserver.StartHttpsServer(t, requests, clientCAPath)
var opts []HTTPClientOpt
if clientCertPath != "" && clientKeyPath != "" {
Loading
Loading
@@ -123,10 +131,11 @@ func setupWithRequests(t *testing.T, caFile, caPath, clientCAPath, clientCertPat
}
httpClient, err := NewHTTPClientWithOpts(url, "", caFile, caPath, selfSigned, 1, opts)
require.NoError(t, err)
if err != nil {
return nil, err
}
client, err := NewGitlabNetClient("", "", "", httpClient)
require.NoError(t, err)
return client, cleanup
return client, err
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment