As we reevaluate how to best support and maintain Staging Ref in the future, we encourage development teams using this environment to highlight their use cases in the following issue: https://gitlab.com/gitlab-com/gl-infra/software-delivery/framework/software-delivery-framework-issue-tracker/-/issues/36.

Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • idrozdov/gitlab-shell
  • mmj/gitlab-shell
2 results
Show changes
Commits on Source (1546)
config.yml
cover.out
tmp/*
.idea
*.log *.log
*.swp *.swp
/*.log* .DS_Store
authorized_keys.lock .GOPATH
.gitlab_shell_secret
.bundle .bundle
tags
.bundle/ .bundle/
.gitlab_shell_secret
.idea
/*.log*
/bin/*
/gl-code-quality-report.json
/go_build
/support/bin/golangci-*
/support/bin/gotestsum-*
authorized_keys.lock
config.yml
cover.out
cover.xml
custom_hooks custom_hooks
hooks/*.d hooks/*.d
/go_build tags
/bin/gitlab-shell tmp/*
/bin/gitlab-shell-authorized-keys-check vendor
/bin/gitlab-shell-authorized-principals-check
/bin/check
Loading
@@ -4,8 +4,28 @@ include:
Loading
@@ -4,8 +4,28 @@ include:
- template: Security/Dependency-Scanning.gitlab-ci.yml - template: Security/Dependency-Scanning.gitlab-ci.yml
- template: Security/Secret-Detection.gitlab-ci.yml - template: Security/Secret-Detection.gitlab-ci.yml
stages:
- prepare
- lint
- test
- post-test
variables: variables:
DOCKER_VERSION: "19.03.0" FF_USE_FASTZIP: 'true'
TRANSFER_METER_FREQUENCY: "1s"
DOCKER_VERSION: "20.10.15"
BUNDLE_FROZEN: "true"
GO_VERSION: "1.23"
GOPATH: $CI_PROJECT_DIR/.GOPATH
DEBIAN_VERSION: "bookworm"
RUBY_VERSION: "3.2.5"
BUNDLE_PATH: vendor/ruby
POLICY: pull
CI_DEBUG_SERVICES: 'true'
RUST_VERSION: "1.73"
UBI_VERSION: "8.6"
IMAGE_TAG: "rubygems-3.5-git-2.45-exiftool-12.60"
GITLAB_ADVANCED_SAST_ENABLED: 'true'
workflow: workflow:
rules: &workflow_rules rules: &workflow_rules
Loading
@@ -16,8 +36,15 @@ workflow:
Loading
@@ -16,8 +36,15 @@ workflow:
# For tags, create a pipeline. # For tags, create a pipeline.
- if: '$CI_COMMIT_TAG' - if: '$CI_COMMIT_TAG'
.rules:go-changes:
rules:
- changes:
- 'go.mod'
- 'go.sum'
- '**/*.go'
default: default:
image: golang:1.13 image: registry.gitlab.com/gitlab-org/gitlab-build-images/debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}-golang-${GO_VERSION}-rust-${RUST_VERSION}:${IMAGE_TAG}
tags: tags:
- gitlab-org - gitlab-org
Loading
@@ -25,53 +52,145 @@ default:
Loading
@@ -25,53 +52,145 @@ default:
image: docker:${DOCKER_VERSION} image: docker:${DOCKER_VERSION}
services: services:
- docker:${DOCKER_VERSION}-dind - docker:${DOCKER_VERSION}-dind
variables:
DOCKER_DRIVER: overlay2
DOCKER_HOST: tcp://docker:2375
DOCKER_TLS_CERTDIR: ""
tags: tags:
# See https://gitlab.com/gitlab-com/www-gitlab-com/-/issues/7019 for tag descriptions # See https://gitlab.com/gitlab-com/www-gitlab-com/-/issues/7019 for tag descriptions
- gitlab-org-docker - gitlab-org-docker
.test: .cached-go: &cached_go
- key:
prefix: "golang-${GO_VERSION}-cache"
files:
- go.mod
- go.sum
policy: $POLICY
paths:
- .GOPATH/pkg/mod/
.cached-ruby: &cached_ruby
- key:
prefix: "ruby-${RUBY_VERSION}-cache"
files:
- Gemfile.lock
policy: $POLICY
paths:
- ${BUNDLE_PATH}
.cached-go-job:
variables:
CACHE_COMPRESSION_LEVEL: "fastest"
cache:
- *cached_go
.cached-ruby-job:
cache:
- *cached_ruby
.cached-job:
cache:
- *cached_go
- *cached_ruby
.go-matrix-job:
parallel:
matrix:
- GO_VERSION: ["1.22", "1.23"]
################################################################################
# Prepare jobs
################################################################################
bundle:install:
stage: prepare
extends: .cached-ruby-job
variables:
POLICY: pull-push
script:
- bundle install --jobs $(nproc)
modules:download:
stage: prepare
extends:
- .cached-go-job
- .go-matrix-job
variables:
POLICY: pull-push
script:
- go mod download
################################################################################
# Test jobs
################################################################################
.test-job:
needs: ['bundle:install', 'modules:download']
rules: !reference [".rules:go-changes", rules]
variables:
GITALY_CONNECTION_INFO: '{"address":"tcp://gitaly:8075", "storage":"default"}'
before_script: before_script:
# Set up the environment to run integration tests (still written in Ruby) # Set up the environment to run integration tests (still written in Ruby)
- apt-get update -qq && apt-get install -y ruby ruby-dev
- ruby -v
- export PATH=~/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/go/bin
- gem install --force --bindir /usr/local/bin bundler -v 2.1.4
- bundle install
# Now set up to run the Golang tests
- make build - make build
- cp config.yml.example config.yml - cp config.yml.example config.yml
- go version - go version
- which go - which go
script: services:
- make verify test - name: registry.gitlab.com/gitlab-org/build/cng/gitaly:master
# Disable the hooks so we don't have to stub the GitLab API
go:1.13: command: ["bash", "-c", "mkdir -p /home/git/repositories && rm -rf /srv/gitlab-shell/hooks/* && touch /srv/gitlab-shell/.gitlab_shell_secret && exec /usr/bin/env GITALY_TESTING_NO_GIT_HOOKS=1 /scripts/process-wrapper"]
extends: .test alias: gitaly
image: golang:1.13
go:1.14: tests:
extends: .test extends:
image: golang:1.14 - .cached-job
- .go-matrix-job
- .test-job
script:
- make verify test_fancy
after_script: after_script:
- make coverage - make coverage
coverage: '/\d+.\d+%/' coverage: '/\d+.\d+%/'
artifacts:
when: always
paths:
- cover.xml
reports:
junit: cover.xml
tests_without_cgo:
extends:
- .cached-job
- .go-matrix-job
- .test-job
variables:
CGO_ENABLED: 0
script:
- make verify test_fancy
tests:fips:
image: registry.gitlab.com/gitlab-org/gitlab-build-images/ubi-${UBI_VERSION}-ruby-${RUBY_VERSION}-golang-${GO_VERSION}-rust-${RUST_VERSION}:${IMAGE_TAG}
extends:
- .cached-job
- .test-job
variables:
FIPS_MODE: 1
script:
- make test_fancy
race: race:
extends: .test extends:
image: golang:1.14 - .cached-go-job
- .go-matrix-job
- .test-job
script: script:
- make test_golang_race - make test_golang_race
code_quality: code_quality:
stage: lint
extends: .use-docker-in-docker extends: .use-docker-in-docker
rules: *workflow_rules rules: *workflow_rules
code_navigation: code_navigation:
image: sourcegraph/lsif-go:v1 stage: post-test
image: sourcegraph/lsif-go:v1.9
allow_failure: true allow_failure: true
script: script:
- lsif-go - lsif-go
Loading
@@ -80,16 +199,85 @@ code_navigation:
Loading
@@ -80,16 +199,85 @@ code_navigation:
lsif: dump.lsif lsif: dump.lsif
# SAST # SAST
gosec-sast: semgrep-sast:
stage: lint
rules: *workflow_rules rules: *workflow_rules
# Dependency Scanning gitlab-advanced-sast:
gemnasium-dependency_scanning: stage: lint
rules: *workflow_rules rules: *workflow_rules
bundler-audit-dependency_scanning: # Dependency Scanning
gemnasium-dependency_scanning:
stage: lint
rules: *workflow_rules rules: *workflow_rules
# Secret Detection # Secret Detection
secret_detection: secret_detection:
stage: lint
rules: *workflow_rules rules: *workflow_rules
build-package-and-qa:
stage: post-test
trigger:
project: 'gitlab-org/build/omnibus-gitlab-mirror'
branch: 'master'
strategy: depend
inherit:
variables: false
variables:
GITLAB_SHELL_VERSION: $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA
TOP_UPSTREAM_SOURCE_PROJECT: $CI_PROJECT_PATH
TOP_UPSTREAM_SOURCE_REF: $CI_COMMIT_REF_NAME
TOP_UPSTREAM_SOURCE_JOB: $CI_JOB_URL
ee: "true"
rules:
# For MRs that change dependencies, we want to automatically ensure builds
# aren't broken. In such cases, we don't want the QA tests to be run
# automatically, but still available for developers to manually run.
- if: '$CI_MERGE_REQUEST_IID'
changes:
- go.sum
variables:
BUILD_ON_ALL_OS: "true"
MANUAL_QA_TEST: "true"
allow_failure: false
# For other MRs, we still provide this job as a manual job for developers
# to obtain a package for testing and run QA tests.
- if: '$CI_MERGE_REQUEST_IID'
when: manual
allow_failure: true
needs: []
modules:tidy:
stage: lint
needs: ['modules:download']
script:
- go mod tidy
- git diff --exit-code go.mod go.sum
lint:
stage: lint
script:
# Write the code coverage report to gl-code-quality-report.json
# and print linting issues to stdout in the format: path/to/file:line description
# remove `--issues-exit-code 0` or set to non-zero to fail the job if linting issues are detected
- apt update && apt install -y jq
- make lint GOLANGCI_LINT_ARGS="--out-format code-climate:gl-code-quality-report-temp.json,line-number"
- cat gl-code-quality-report-temp.json | jq '[ .[] | select(.severity == "warning").severity |= "minor" ]' > gl-code-quality-report.json
- rm -f gl-code-quality-report-temp.json
artifacts:
reports:
codequality: gl-code-quality-report.json
paths:
- gl-code-quality-report.json
nilaway:
stage: lint
rules: !reference [".rules:go-changes", rules]
before_script:
- go install go.uber.org/nilaway/cmd/nilaway@latest
script:
- ${GOPATH}/bin/nilaway ./... > /tmp/out.txt 2>&1 || true
- cat /tmp/out.txt
allow_failure: true
* @ashmckenzie @igor.drozdov @nick.thomas @patrickbajao # https://gitlab.com/groups/gitlab-org/maintainers/gitlab-shell/-/group_members?with_inherited_permissions=exclude
* @gitlab-org/maintainers/gitlab-shell
[Documentation] @gl-docsteam
*.md
/doc/
# This file contains all available configuration options
# with their default values.
# options for analysis running
run:
# default concurrency is a available CPU number
# concurrency: 4
# timeout for analysis, e.g. 30s, 5m, default is 1m
timeout: 30m
# exit code when at least one issue was found, default is 1
issues-exit-code: 1
# include test files or not, default is true
tests: true
# list of build tags, all linters use it. Default is empty list.
# build-tags:
# - mytag
# which dirs to skip: issues from them won't be reported;
# can use regexp here: generated.*, regexp is applied on full path;
# default value is empty list, but default dirs are skipped independently
# from this option's value (see skip-dirs-use-default).
# skip-dirs:
# - src/external_libs
# - autogenerated_by_my_lib
# default is true. Enables skipping of directories:
# vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
skip-dirs-use-default: true
# which files to skip: they will be analyzed, but issues from them
# won't be reported. Default value is empty list, but there is
# no need to include all autogenerated files, we confidently recognize
# autogenerated files. If it's not please let us know.
# skip-files:
# - ".*\\.my\\.go$"
# - lib/bad.go
# by default isn't set. If set we pass it to "go list -mod={option}". From "go help modules":
# If invoked with -mod=readonly, the go command is disallowed from the implicit
# automatic updating of go.mod described above. Instead, it fails when any changes
# to go.mod are needed. This setting is most useful to check that go.mod does
# not need updates, such as in a continuous integration and testing system.
# If invoked with -mod=vendor, the go command assumes that the vendor
# directory holds the correct copies of dependencies and ignores
# the dependency descriptions in go.mod.
# modules-download-mode: readonly|release|vendor
# output configuration options
output:
# colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number"
formats:
- format: line-number
# print lines of code with issue, default is true
print-issued-lines: true
# print linter name in the end of issue text, default is true
print-linter-name: true
sort-results: true
# all available settings of specific linters
linters-settings:
errcheck:
# report about not checking of errors in type assetions: `a := b.(MyStruct)`;
# default is false: such cases aren't reported by default.
check-type-assertions: false
# report about assignment of errors to blank identifier: `num, _ := strconv.Atoi(numStr)`;
# default is false: such cases aren't reported by default.
check-blank: false
# [deprecated] comma-separated list of pairs of the form pkg:regex
# the regex is used to ignore names within pkg. (default "fmt:.*").
# see https://github.com/kisielk/errcheck#the-deprecated-method for details
# ignore: fmt:.*,io/ioutil:^Read.*
ignore: ''
# path to a file containing a list of functions to exclude from checking
# see https://github.com/kisielk/errcheck#excluding-functions for details
# exclude: /path/to/file.txt
# Disable error checking, as errorcheck detects more errors and is more configurable.
gosec:
exclude:
- "G104"
funlen:
lines: 60
statements: 40
govet:
# report about shadowed variables
enable:
- shadow
# settings per analyzer
settings:
printf: # analyzer name, run `go tool vet help` to see all analyzers
funcs: # run `go tool vet help printf` to see available settings for `printf` analyzer
- (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof
- (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf
- (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf
- (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf
# enable or disable analyzers by name
# enable:
# - atomicalign
# enable-all: false
# disable:
# - shadow
# disable-all: false
gofmt:
# simplify code: gofmt with `-s` option, true by default
simplify: true
goimports:
# put imports beginning with prefix after 3rd-party packages;
# it's a comma-separated list of prefixes
# local-prefixes: github.com/org/project
gocyclo:
# minimal code complexity to report, 30 by default (but we recommend 10-20)
min-complexity: 30
gocognit:
# minimal code complexity to report, 30 by default (but we recommend 10-20)
min-complexity: 20
maligned:
# print struct with more effective memory layout or not, false by default
suggest-new: true
dupl:
# tokens count to trigger issue, 150 by default
threshold: 100
goconst:
# minimal length of string constant, 3 by default
min-len: 3
# minimal occurrences count to trigger, 3 by default
min-occurrences: 3
depguard:
rules:
test:
files:
- $test
allow:
- $gostd
- github.com/stretchr/testify
- gitlab.com/gitlab-org/gitlab-shell
- gitlab.com/gitlab-org/labkit
- gitlab.com/gitlab-org/gitaly
- github.com/prometheus/client_golang/prometheus
- github.com/pires/go-proxyproto
- github.com/otiai10/copy
- github.com/hashicorp/go-retryablehttp
- github.com/golang-jwt/jwt
- github.com/mikesmitty/edkey
- github.com/sirupsen/logrus
- github.com/grpc-ecosystem/go-grpc-prometheus
- github.com/mattn/go-shellwords
# list-type: blacklist
# include-go-root: false
# packages:
# - github.com/sirupsen/logrus
# packages-with-error-messages:
# # specify an error message to output when a blacklisted package is used
# github.com/sirupsen/logrus: "logging is allowed only by logutils.Log"
misspell:
# Correct spellings using locale preferences for US or UK.
# Default is to use a neutral variety of English.
# Setting locale to US will correct the British spelling of 'colour' to 'color'.
locale: US
ignore-words:
- GitLab
lll:
# max line length, lines longer will be reported. Default is 120.
# '\t' is counted as 1 character by default, and can be changed with the tab-width option
line-length: 120
# tab width in spaces. Default to 1.
tab-width: 1
unused:
# treat code as a program (not a library) and report unused exported identifiers; default is false.
# XXX: if you enable this setting, unused will report a lot of false-positives in text editors:
# if it's called for subdir of a project it can't find funcs usages. All text editor integrations
# with golangci-lint call it on a directory with the changed file.
check-exported: false
unparam:
# Inspect exported functions, default is false. Set to true if no external program/library imports your code.
# XXX: if you enable this setting, unparam will report a lot of false-positives in text editors:
# if it's called for subdir of a project it can't find external interfaces. All text editor integrations
# with golangci-lint call it on a directory with the changed file.
check-exported: false
nakedret:
# make an issue if func has more lines of code than this setting and it has naked returns; default is 30
max-func-lines: 30
prealloc:
# XXX: we don't recommend using this linter before doing performance profiling.
# For most programs usage of prealloc will be a premature optimization.
# Report preallocation suggestions only on simple loops that have no returns/breaks/continues/gotos in them.
# True by default.
simple: true
range-loops: true # Report preallocation suggestions on range loops, true by default
for-loops: false # Report preallocation suggestions on for loops, false by default
gocritic:
# Which checks should be enabled; can't be combined with 'disabled-checks';
# See https://go-critic.github.io/overview#checks-overview
# To check which checks are enabled run `GL_DEBUG=gocritic golangci-lint run`
# By default list of stable checks is used.
# enabled-checks:
# - rangeValCopy
# Which checks should be disabled; can't be combined with 'enabled-checks'; default is empty
# disabled-checks:
# - regexpMust
# Enable multiple checks by tags, run `GL_DEBUG=gocritic golangci-lint run` to see all tags and checks.
# Empty list by default. See https://github.com/go-critic/go-critic#usage -> section "Tags".
# enabled-tags:
# - performance
settings: # settings passed to gocritic
captLocal: # must be valid enabled check name
paramsOnly: true
# rangeValCopy:
# sizeThreshold: 32
godox:
# report any comments starting with keywords, this is useful for TODO or FIXME comments that
# might be left in the code accidentally and should be resolved before merging
keywords: # default keywords are TODO, BUG, and FIXME, these can be overwritten by this setting
- TODO
- BUG
- FIXME
- NOTE
- OPTIMIZE # marks code that should be optimized before merging
- HACK # marks hack-arounds that should be removed before merging
dogsled:
# checks assignments with too many blank identifiers; default is 2
max-blank-identifiers: 2
whitespace:
multi-if: false # Enforces newlines (or comments) after every multi-line if statement
multi-func: false # Enforces newlines (or comments) after every multi-line function signature
wsl:
# If true append is only allowed to be cuddled if appending value is
# matching variables, fields or types on line above. Default is true.
strict-append: true
# Allow calls and assignments to be cuddled as long as the lines have any
# matching variables, fields or types. Default is true.
allow-assign-and-call: true
# Allow multiline assignments to be cuddled. Default is true.
allow-multiline-assign: true
# Allow declarations (var) to be cuddled.
allow-cuddle-declarations: false
# Allow trailing comments in ending of blocks
allow-trailing-comment: false
# Force newlines in end of case at this limit (0 = never).
force-case-trailing-whitespace: 0
linters:
# please, do not use `enable-all`: it's deprecated and will be removed soon.
# inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint
disable-all: true
enable:
- bodyclose
- copyloopvar
- depguard
- dogsled
- dupl
- errcheck
- funlen
- gocognit
- goconst
- gocritic
- godox
- gofmt
- goimports
- gosec
- gosimple
- govet
- ineffassign
- misspell
- nakedret
- revive
- staticcheck
- stylecheck
- testifylint
- typecheck
- unconvert
- unparam
- unused
- whitespace
# don't enable:
# - deadcode
# - gochecknoglobals
# - gochecknoinits
# - gocyclo
# - lll
# - maligned
# - prealloc
# - varcheck
issues:
# List of regexps of issue texts to exclude, empty list by default.
# But independently from this option we use default exclude patterns,
# it can be disabled by `exclude-use-default: false`. To list all
# excluded by default patterns execute `golangci-lint run --help`
# exclude:
# - abcdef
# Excluding configuration per-path, per-linter, per-text and per-source
exclude-rules:
# Exclude some linters from running on tests files.
- path: _test\.go
linters:
- gocyclo
- errcheck
- dupl
- gosec
- funlen
# Exclude known linters from partially hard-vendored code,
# which is impossible to exclude via "nolint" comments.
# - path: internal/hmac/
# text: "weak cryptographic primitive"
# linters:
# - gosec
# Exclude some staticcheck messages
# - linters:
# - staticcheck
# text: "SA9003:"
# Exclude lll issues for long lines with go:generate
- linters:
- lll
source: "^//go:generate "
# Independently from option `exclude` we use default exclude patterns,
# it can be disabled by this option. To list all
# excluded by default patterns execute `golangci-lint run --help`.
# Default value for this option is true.
exclude-use-default: false
# Maximum issues count per one linter. Set to 0 to disable. Default is 50.
max-issues-per-linter: 0
# Maximum count of issues with the same text. Set to 0 to disable. Default is 3.
max-same-issues: 0
# Show only new issues: if there are unstaged changes or untracked files,
# only those changes are analyzed, else only changes in HEAD~ are analyzed.
# It's a super-useful option for integration of golangci-lint into existing
# large codebase. It's not practical to fix all existing issues at the moment
# of integration: much better don't allow issues in new code.
# Default is false.
new: false
# Show only new issues created after git revision `REV`
# This should be passed as flag during individual CI jobs.
# new-from-rev: REV
# Show only new issues created in git patch with set file path.
# new-from-patch: path/to/patch/file
2.7.2 3.3.4
ruby 3.3.5
golang 1.23.2
This diff is collapsed.
# frozen_string_literal: true
require 'gitlab-dangerfiles'
Gitlab::Dangerfiles.for_project(self) do |gitlab_dangerfiles|
gitlab_dangerfiles.import_plugins
gitlab_dangerfiles.import_dangerfiles(except: %w[changelog commit_messages])
end
source 'https://rubygems.org' source 'https://rubygems.org'
group :development, :test do group :development, :test do
gem 'rspec', '~> 3.8.0' gem 'rspec', '~> 3.13.0'
gem 'webrick', '~> 1.8', '>= 1.8.2'
end
group :development, :danger do
gem 'gitlab-dangerfiles', '~> 4.8.0'
end end
GEM GEM
remote: https://rubygems.org/ remote: https://rubygems.org/
specs: specs:
diff-lcs (1.3) addressable (2.8.7)
rspec (3.8.0) public_suffix (>= 2.0.2, < 7.0)
rspec-core (~> 3.8.0) bigdecimal (3.1.8)
rspec-expectations (~> 3.8.0) claide (1.1.0)
rspec-mocks (~> 3.8.0) claide-plugins (0.9.2)
rspec-core (3.8.0) cork
rspec-support (~> 3.8.0) nap
rspec-expectations (3.8.1) open4 (~> 1.3)
colored2 (3.1.2)
cork (0.3.0)
colored2 (~> 3.1)
csv (3.3.0)
danger (9.4.3)
claide (~> 1.0)
claide-plugins (>= 0.9.2)
colored2 (~> 3.1)
cork (~> 0.1)
faraday (>= 0.9.0, < 3.0)
faraday-http-cache (~> 2.0)
git (~> 1.13)
kramdown (~> 2.3)
kramdown-parser-gfm (~> 1.0)
no_proxy_fix
octokit (>= 4.0)
terminal-table (>= 1, < 4)
danger-gitlab (8.0.0)
danger
gitlab (~> 4.2, >= 4.2.0)
diff-lcs (1.5.1)
faraday (2.9.2)
faraday-net_http (>= 2.0, < 3.2)
faraday-http-cache (2.5.1)
faraday (>= 0.8)
faraday-net_http (3.1.0)
net-http
git (1.19.1)
addressable (~> 2.8)
rchardet (~> 1.8)
gitlab (4.20.1)
httparty (~> 0.20)
terminal-table (>= 1.5.1)
gitlab-dangerfiles (4.8.0)
danger (>= 9.3.0)
danger-gitlab (>= 8.0.0)
rake (~> 13.0)
httparty (0.22.0)
csv
mini_mime (>= 1.0.0)
multi_xml (>= 0.5.2)
kramdown (2.4.0)
rexml
kramdown-parser-gfm (1.1.0)
kramdown (~> 2.0)
mini_mime (1.1.5)
multi_xml (0.7.1)
bigdecimal (~> 3.1)
nap (1.1.0)
net-http (0.4.1)
uri
no_proxy_fix (0.1.2)
octokit (6.1.1)
faraday (>= 1, < 3)
sawyer (~> 0.9)
open4 (1.3.4)
public_suffix (5.1.1)
rake (13.2.1)
rchardet (1.8.0)
rexml (3.3.1)
strscan
rspec (3.13.0)
rspec-core (~> 3.13.0)
rspec-expectations (~> 3.13.0)
rspec-mocks (~> 3.13.0)
rspec-core (3.13.0)
rspec-support (~> 3.13.0)
rspec-expectations (3.13.0)
diff-lcs (>= 1.2.0, < 2.0) diff-lcs (>= 1.2.0, < 2.0)
rspec-support (~> 3.8.0) rspec-support (~> 3.13.0)
rspec-mocks (3.8.0) rspec-mocks (3.13.0)
diff-lcs (>= 1.2.0, < 2.0) diff-lcs (>= 1.2.0, < 2.0)
rspec-support (~> 3.8.0) rspec-support (~> 3.13.0)
rspec-support (3.8.0) rspec-support (3.13.0)
sawyer (0.9.2)
addressable (>= 2.3.5)
faraday (>= 0.17.3, < 3)
strscan (3.1.0)
terminal-table (3.0.2)
unicode-display_width (>= 1.1.1, < 3)
unicode-display_width (2.5.0)
uri (0.13.0)
webrick (1.8.2)
PLATFORMS PLATFORMS
ruby ruby
DEPENDENCIES DEPENDENCIES
rspec (~> 3.8.0) gitlab-dangerfiles (~> 4.8.0)
rspec (~> 3.13.0)
webrick (~> 1.8, >= 1.8.2)
BUNDLED WITH BUNDLED WITH
2.1.4 2.5.11
Copyright (c) 2011-2018 GitLab B.V. MIT License
With regard to the GitLab Software: Copyright (c) 2011-present GitLab B.V.
Permission is hereby granted, free of charge, to any person obtaining a copy Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal of this software and associated documentation files (the "Software"), to deal
Loading
@@ -9,17 +9,13 @@ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
Loading
@@ -9,17 +9,13 @@ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions: furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in The above copyright notice and this permission notice shall be included in all
all copies or substantial portions of the Software. copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
THE SOFTWARE. SOFTWARE.
For all third party components incorporated into the GitLab Software, those
components are licensed under the original license provided by the owner of the
applicable component.
.PHONY: validate verify verify_ruby verify_golang test test_ruby test_golang coverage coverage_golang setup _install build compile check clean .PHONY: validate verify verify_ruby verify_golang test test_ruby test_golang test_fancy test_golang_fancy coverage coverage_golang setup _script_install make_necessary_dirs build compile check clean install lint
GO_SOURCES := $(shell find . -name '*.go') FIPS_MODE ?= 0
VERSION_STRING := $(shell git describe --match v* 2>/dev/null || awk '$0="v"$0' VERSION 2>/dev/null || echo unknown) OS := $(shell uname | tr A-Z a-z)
GO_SOURCES := $(shell git ls-files \*.go)
VERSION_STRING := $(shell git describe --match v* 2>/dev/null || awk '$$0="v"$$0' VERSION 2>/dev/null || echo unknown)
BUILD_TIME := $(shell date -u +%Y%m%d.%H%M%S) BUILD_TIME := $(shell date -u +%Y%m%d.%H%M%S)
GOBUILD_FLAGS := -ldflags "-X main.Version=$(VERSION_STRING) -X main.BuildTime=$(BUILD_TIME)" GO_TAGS := tracer_static tracer_static_jaeger continuous_profiler_stackdriver
ARCH ?= $(shell uname -m | sed -e 's/x86_64/amd64/' | sed -e 's/aarch64/arm64/')
GOTESTSUM_VERSION := 1.12.0
GOTESTSUM_FILE := support/bin/gotestsum-${GOTESTSUM_VERSION}
GOLANGCI_LINT_VERSION := 1.60.3
GOLANGCI_LINT_FILE := support/bin/golangci-lint-${GOLANGCI_LINT_VERSION}
export GOFLAGS := -mod=readonly
ifeq (${FIPS_MODE}, 1)
GO_TAGS += fips
# If the golang-fips compiler is built with CGO_ENABLED=0, this needs to be
# explicitly switched on.
export CGO_ENABLED=1
# Go 1.19 now requires GOEXPERIMENT=boringcrypto for FIPS compilation.
# See https://github.com/golang/go/issues/51940 for more details.
BORINGCRYPTO_SUPPORT := $(shell GOEXPERIMENT=boringcrypto go version > /dev/null 2>&1; echo $$?)
ifeq ($(BORINGCRYPTO_SUPPORT), 0)
export GOEXPERIMENT=boringcrypto
endif
endif
ifneq (${CGO_ENABLED}, 0)
GO_TAGS += gssapi
endif
ifeq (${OS}, darwin) # Mac OS
BREW_PREFIX := $(shell brew --prefix 2>/dev/null || echo "/opt/homebrew")
# To be able to compile gssapi library
export CGO_CFLAGS="-I$(BREW_PREFIX)/opt/heimdal/include"
endif
GOBUILD_FLAGS := -ldflags "-X main.Version=$(VERSION_STRING) -X main.BuildTime=$(BUILD_TIME)" -tags "$(GO_TAGS)" -mod=mod
PREFIX ?= /usr/local
build: compile
validate: verify test validate: verify test
verify: verify_golang verify: verify_golang
verify_golang: verify_golang:
gofmt -s -l $(GO_SOURCES) gofmt -s -l $(GO_SOURCES) | awk '{ print } END { if (NR > 0) { print "Please run make fmt"; exit 1 } }'
fmt:
gofmt -w -s $(GO_SOURCES)
test: test_ruby test_golang test: test_ruby test_golang
test_fancy: test_ruby test_golang_fancy
# The Ruby tests are now all integration specs that test the Go implementation. # The Ruby tests are now all integration specs that test the Go implementation.
test_ruby: test_ruby:
bundle exec rspec --color --format d spec bundle exec rspec --color --format d spec
test_golang: test_golang:
go test -cover -coverprofile=cover.out ./... go test -cover -coverprofile=cover.out -count 1 -tags "$(GO_TAGS)" ./...
test_golang_fancy: ${GOTESTSUM_FILE}
@${GOTESTSUM_FILE} --version
@${GOTESTSUM_FILE} --junitfile ./cover.xml --format pkgname -- -coverprofile=./cover.out -covermode=atomic -count 1 -tags "$(GO_TAGS)" ./...
${GOTESTSUM_FILE}:
mkdir -p $(shell dirname ${GOTESTSUM_FILE})
curl -L https://github.com/gotestyourself/gotestsum/releases/download/v${GOTESTSUM_VERSION}/gotestsum_${GOTESTSUM_VERSION}_${OS}_${ARCH}.tar.gz | tar -zOxf - gotestsum > ${GOTESTSUM_FILE} && chmod +x ${GOTESTSUM_FILE}
test_golang_race: test_golang_race:
go test -race ./... go test -race -count 1 ./...
coverage: coverage_golang coverage: coverage_golang
coverage_golang: coverage_golang:
[ -f cover.out ] && go tool cover -func cover.out [ -f cover.out ] && go tool cover -func cover.out
setup: _install bin/gitlab-shell lint:
@support/lint.sh ./...
golangci: ${GOLANGCI_LINT_FILE}
@${GOLANGCI_LINT_FILE} run --issues-exit-code 0 --print-issued-lines=false ${GOLANGCI_LINT_ARGS}
_install: ${GOLANGCI_LINT_FILE}:
bin/install @mkdir -p $(shell dirname ${GOLANGCI_LINT_FILE})
@curl -L https://github.com/golangci/golangci-lint/releases/download/v${GOLANGCI_LINT_VERSION}/golangci-lint-${GOLANGCI_LINT_VERSION}-${OS}-${ARCH}.tar.gz | tar --strip-components 1 -zOxf - golangci-lint-${GOLANGCI_LINT_VERSION}-${OS}-${ARCH}/golangci-lint > ${GOLANGCI_LINT_FILE} && chmod +x ${GOLANGCI_LINT_FILE}
build: bin/gitlab-shell setup: make_necessary_dirs bin/gitlab-shell
compile: bin/gitlab-shell
bin/gitlab-shell: $(GO_SOURCES) make_necessary_dirs:
GOBIN="$(CURDIR)/bin" go install $(GOBUILD_FLAGS) ./cmd/... support/make_necessary_dirs
compile: bin/gitlab-shell bin/gitlab-sshd
bin:
mkdir -p bin
bin/gitlab-shell: bin $(GO_SOURCES)
go build $(GOBUILD_FLAGS) -o $(CURDIR)/bin ./cmd/...
bin/gitlab-sshd: bin $(GO_SOURCES)
go build $(GOBUILD_FLAGS) -o $(CURDIR)/bin/gitlab-sshd ./cmd/gitlab-sshd
check: check:
bin/check bin/gitlab-shell-check
clean: clean:
rm -f bin/check bin/gitlab-shell bin/gitlab-shell-authorized-keys-check bin/gitlab-shell-authorized-principals-check rm -f bin/*
install: compile
mkdir -p $(DESTDIR)$(PREFIX)/bin/
install -m755 bin/gitlab-shell-check $(DESTDIR)$(PREFIX)/bin/
install -m755 bin/gitlab-shell $(DESTDIR)$(PREFIX)/bin/
install -m755 bin/gitlab-shell-authorized-keys-check $(DESTDIR)$(PREFIX)/bin/
install -m755 bin/gitlab-shell-authorized-principals-check $(DESTDIR)$(PREFIX)/bin/
install -m755 bin/gitlab-sshd $(DESTDIR)$(PREFIX)/bin/
# GitLab Shell process
This page [has moved into the `gitlab` repository](https://docs.gitlab.com/ee/development/gitlab_shell/process.html).
# GitLab Shell ---
stage: Create
## GitLab Shell handles git SSH sessions for GitLab group: Source Code
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
GitLab Shell handles git SSH sessions for GitLab and modifies the list of authorized keys. ---
GitLab Shell is not a Unix shell nor a replacement for Bash or Zsh.
When you access the GitLab server over SSH then GitLab Shell will:
1. Limits you to predefined git commands (git push, git pull).
1. Call the GitLab Rails API to check if you are authorized, and what Gitaly server your repository is on
1. Copy data back and forth between the SSH client and the Gitaly server
If you access a GitLab server over HTTP(S) you end up in [gitlab-workhorse](https://gitlab.com/gitlab-org/gitlab-workhorse).
An overview of the four cases described above:
1. git pull over SSH -> gitlab-shell -> API call to gitlab-rails (Authorization) -> accept or decline -> establish Gitaly session
1. git push over SSH -> gitlab-shell (git command is not executed yet) -> establish Gitaly session -> (in Gitaly) gitlab-shell pre-receive hook -> API call to gitlab-rails (authorization) -> accept or decline push
## Default branch
GitLab Shell is transitioning its default branch from `master` to `main`. For now,
both branches are valid. All changes go to the `main` branch and are synced manually
to `master` by the maintainers. We plan to remove the `master` branch as soon as
possible. The current status is being tracked in [issue 489](https://gitlab.com/gitlab-org/gitlab-shell/-/issues/489).
## Code status
[![pipeline status](https://gitlab.com/gitlab-org/gitlab-shell/badges/main/pipeline.svg)](https://gitlab.com/gitlab-org/gitlab-shell/-/pipelines?ref=main) [![pipeline status](https://gitlab.com/gitlab-org/gitlab-shell/badges/main/pipeline.svg)](https://gitlab.com/gitlab-org/gitlab-shell/-/pipelines?ref=main)
[![coverage report](https://gitlab.com/gitlab-org/gitlab-shell/badges/main/coverage.svg)](https://gitlab.com/gitlab-org/gitlab-shell/-/pipelines?ref=main) [![coverage report](https://gitlab.com/gitlab-org/gitlab-shell/badges/main/coverage.svg)](https://gitlab.com/gitlab-org/gitlab-shell/-/pipelines?ref=main)
[![Code Climate](https://codeclimate.com/github/gitlabhq/gitlab-shell.svg)](https://codeclimate.com/github/gitlabhq/gitlab-shell) [![Code Climate](https://codeclimate.com/github/gitlabhq/gitlab-shell.svg)](https://codeclimate.com/github/gitlabhq/gitlab-shell)
## Requirements # GitLab Shell
GitLab Shell is written in Go, and needs a Go compiler to build. It still requires
Ruby to build and test, but not to run.
Download and install the current version of Go from https://golang.org/dl/
## Setup
make setup
## Check
Checks if GitLab API access and redis via internal API can be reached:
make check
## Testing
Run tests:
bundle install
make test
Run gofmt:
make verify
Run both test and verify (the default Makefile target): GitLab Shell handles Git SSH sessions for GitLab and modifies the list of
authorized keys. GitLab Shell is not a Unix shell nor a replacement for Bash or Zsh.
bundle install GitLab supports Git LFS authentication through SSH.
make validate
## Git LFS remark ## Development Documentation
Starting with GitLab 8.12, GitLab supports Git LFS authentication through SSH. Development documentation for GitLab Shell [has moved into the `gitlab` repository](https://docs.gitlab.com/ee/development/gitlab_shell/).
## Releasing a new version ## Project structure
GitLab Shell is versioned by git tags, and the version used by the Rails | Directory | Description |
application is stored in |-----------|-------------|
[`GITLAB_SHELL_VERSION`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/GITLAB_SHELL_VERSION). | `cmd/` | 'Commands' that will ultimately be compiled into binaries. |
| `internal/` | Internal Go source code that is not intended to be used outside of the project/module. |
| `client/` | HTTP and GitLab client logic that is used internally and by other modules, e.g. Gitaly. |
| `bin/` | Compiled binaries are created here. |
| `support/` | Scripts and tools that assist in development and/or testing. |
| `spec/` | Ruby based integration tests. |
For each version, there is a raw version and a tag version: ## Building
- The **raw version** is the version number. For instance, `15.2.8`. Run `make` or `make build`.
- The **tag version** is the raw version prefixed with `v`. For instance, `v15.2.8`.
To release a new version of GitLab Shell and have that version available to the ## Testing
Rails application:
1. Create a merge request to update the [`CHANGELOG`](CHANGELOG) with the Run `make test`.
**tag version** and the [`VERSION`](VERSION) file with the **raw version**.
2. Ask a maintainer to review and merge the merge request. If you're already a
maintainer, second maintainer review is not required.
3. Add a new git tag with the **tag version**.
4. Update `GITLAB_SHELL_VERSION` in the Rails application to the **raw
version**. (Note: this can be done as a separate MR to that, or in and MR
that will make use of the latest GitLab Shell changes.)
## Contributing ## Release Process
See [CONTRIBUTING.md](./CONTRIBUTING.md). 1. Create a `gitlab-org/gitlab-shell` MR to update [`VERSION`](https://gitlab.com/gitlab-org/gitlab-shell/-/blob/main/VERSION) and [`CHANGELOG`](https://gitlab.com/gitlab-org/gitlab-shell/-/blob/main/CHANGELOG) files, e.g. [Release v14.39.0](https://gitlab.com/gitlab-org/gitlab-shell/-/merge_requests/1123).
2. Once `gitlab-org/gitlab-shell` MR is merged, create the corresponding git tag, e.g. https://gitlab.com/gitlab-org/gitlab-shell/-/tags/v14.39.0.
3. Create a `gitlab-org/gitlab` MR to update [`GITLAB_SHELL_VERSION`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/GITLAB_SHELL_VERSION) to the proposed tag, e.g. [Bump GitLab Shell to 14.39.0](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/162661).
4. Announce in `#gitlab-shell` a new version has been created.
## License ## Licensing
See [LICENSE](./LICENSE). See the `LICENSE` file for licensing information as it pertains to files in
this repository.
13.13.0 14.39.0
#!/bin/sh
# Legacy script used for AuthorizedKeysCommand when configured without username.
# Executes gitlab-shell-authorized-keys-check with "git" as expected and actual
# username and with the passed key.
#
# TODO: Remove this in https://gitlab.com/gitlab-org/gitlab-shell/issues/209.
$(dirname $0)/gitlab-shell-authorized-keys-check git git $1
Loading
@@ -2,69 +2,92 @@ package client
Loading
@@ -2,69 +2,92 @@ package client
import ( import (
"context" "context"
"encoding/base64"
"encoding/json" "encoding/json"
"fmt" "fmt"
"io/ioutil" "io"
"net/http" "net/http"
"net/http/httptest"
"path" "path"
"strings" "strings"
"testing" "testing"
"time"
"github.com/sirupsen/logrus" "github.com/golang-jwt/jwt/v5"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"gitlab.com/gitlab-org/gitlab-shell/client/testserver"
"gitlab.com/gitlab-org/gitlab-shell/internal/testhelper" "gitlab.com/gitlab-org/gitlab-shell/v14/client/testserver"
"gitlab.com/gitlab-org/gitlab-shell/v14/internal/testhelper"
)
var (
secret = "sssh, it's a secret"
defaultHttpOpts = []HTTPClientOpt{WithHTTPRetryOpts(time.Millisecond, time.Millisecond, 2)}
) )
func TestClients(t *testing.T) { func TestClients(t *testing.T) {
testDirCleanup, err := testhelper.PrepareTestRootDir() testRoot := testhelper.PrepareTestRootDir(t)
require.NoError(t, err)
defer testDirCleanup()
testCases := []struct { testCases := []struct {
desc string desc string
relativeURLRoot string relativeURLRoot string
caFile string caFile string
server func(*testing.T, []testserver.TestRequestHandler) (string, func()) server func(*testing.T, []testserver.TestRequestHandler) string
secret string
}{ }{
{ {
desc: "Socket client", desc: "Socket client",
server: testserver.StartSocketHttpServer, server: testserver.StartSocketHTTPServer,
secret: secret,
}, },
{ {
desc: "Socket client with a relative URL at /", desc: "Socket client with a relative URL at /",
relativeURLRoot: "/", relativeURLRoot: "/",
server: testserver.StartSocketHttpServer, server: testserver.StartSocketHTTPServer,
secret: secret,
}, },
{ {
desc: "Socket client with relative URL at /gitlab", desc: "Socket client with relative URL at /gitlab",
relativeURLRoot: "/gitlab", relativeURLRoot: "/gitlab",
server: testserver.StartSocketHttpServer, server: testserver.StartSocketHTTPServer,
secret: secret,
}, },
{ {
desc: "Http client", desc: "Http client",
server: testserver.StartHttpServer, server: testserver.StartHTTPServer,
secret: secret,
}, },
{ {
desc: "Https client", desc: "Https client",
caFile: path.Join(testhelper.TestRoot, "certs/valid/server.crt"), caFile: path.Join(testRoot, "certs/valid/server.crt"),
server: func(t *testing.T, handlers []testserver.TestRequestHandler) (string, func()) { server: func(t *testing.T, handlers []testserver.TestRequestHandler) string {
return testserver.StartHttpsServer(t, handlers, "") return testserver.StartHTTPSServer(t, handlers, "")
},
secret: secret,
},
{
desc: "Secret with newlines",
caFile: path.Join(testRoot, "certs/valid/server.crt"),
server: func(t *testing.T, handlers []testserver.TestRequestHandler) string {
return testserver.StartHTTPSServer(t, handlers, "")
}, },
secret: "\n" + secret + "\n",
},
{
desc: "Retry client",
server: testserver.StartRetryHTTPServer,
secret: secret,
}, },
} }
for _, tc := range testCases { for _, tc := range testCases {
t.Run(tc.desc, func(t *testing.T) { t.Run(tc.desc, func(t *testing.T) {
url, cleanup := tc.server(t, buildRequests(t, tc.relativeURLRoot)) url := tc.server(t, buildRequests(t, tc.relativeURLRoot))
defer cleanup()
secret := "sssh, it's a secret" httpClient, err := NewHTTPClientWithOpts(url, tc.relativeURLRoot, tc.caFile, "", 1, defaultHttpOpts)
require.NoError(t, err)
httpClient := NewHTTPClient(url, tc.relativeURLRoot, tc.caFile, "", false, 1)
client, err := NewGitlabNetClient("", "", secret, httpClient) client, err := NewGitlabNetClient("", "", tc.secret, httpClient)
require.NoError(t, err) require.NoError(t, err)
testBrokenRequest(t, client) testBrokenRequest(t, client)
Loading
@@ -72,39 +95,29 @@ func TestClients(t *testing.T) {
Loading
@@ -72,39 +95,29 @@ func TestClients(t *testing.T) {
testSuccessfulPost(t, client) testSuccessfulPost(t, client)
testMissing(t, client) testMissing(t, client)
testErrorMessage(t, client) testErrorMessage(t, client)
testAuthenticationHeader(t, client) testJWTAuthenticationHeader(t, client)
testXForwardedForHeader(t, client)
testHostWithTrailingSlash(t, client)
}) })
} }
} }
func testSuccessfulGet(t *testing.T, client *GitlabNetClient) { func testSuccessfulGet(t *testing.T, client *GitlabNetClient) {
t.Run("Successful get", func(t *testing.T) { t.Run("Successful get", func(t *testing.T) {
hook := testhelper.SetupLogger()
response, err := client.Get(context.Background(), "/hello") response, err := client.Get(context.Background(), "/hello")
require.NoError(t, err) require.NoError(t, err)
require.NotNil(t, response) require.NotNil(t, response)
defer response.Body.Close() defer response.Body.Close()
responseBody, err := ioutil.ReadAll(response.Body) responseBody, err := io.ReadAll(response.Body)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, string(responseBody), "Hello") require.Equal(t, string(responseBody), "Hello")
require.True(t, testhelper.WaitForLogEvent(hook))
entries := hook.AllEntries()
require.Equal(t, 1, len(entries))
require.Equal(t, logrus.InfoLevel, entries[0].Level)
require.Contains(t, entries[0].Message, "method=GET")
require.Contains(t, entries[0].Message, "status=200")
require.Contains(t, entries[0].Message, "content_length_bytes=")
require.Contains(t, entries[0].Message, "Finished HTTP request")
require.Contains(t, entries[0].Message, "correlation_id=")
}) })
} }
func testSuccessfulPost(t *testing.T, client *GitlabNetClient) { func testSuccessfulPost(t *testing.T, client *GitlabNetClient) {
t.Run("Successful Post", func(t *testing.T) { t.Run("Successful Post", func(t *testing.T) {
hook := testhelper.SetupLogger()
data := map[string]string{"key": "value"} data := map[string]string{"key": "value"}
response, err := client.Post(context.Background(), "/post_endpoint", data) response, err := client.Post(context.Background(), "/post_endpoint", data)
Loading
@@ -113,53 +126,23 @@ func testSuccessfulPost(t *testing.T, client *GitlabNetClient) {
Loading
@@ -113,53 +126,23 @@ func testSuccessfulPost(t *testing.T, client *GitlabNetClient) {
defer response.Body.Close() defer response.Body.Close()
responseBody, err := ioutil.ReadAll(response.Body) responseBody, err := io.ReadAll(response.Body)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, "Echo: {\"key\":\"value\"}", string(responseBody)) require.Equal(t, "Echo: {\"key\":\"value\"}", string(responseBody))
require.True(t, testhelper.WaitForLogEvent(hook))
entries := hook.AllEntries()
require.Equal(t, 1, len(entries))
require.Equal(t, logrus.InfoLevel, entries[0].Level)
require.Contains(t, entries[0].Message, "method=POST")
require.Contains(t, entries[0].Message, "status=200")
require.Contains(t, entries[0].Message, "content_length_bytes=")
require.Contains(t, entries[0].Message, "Finished HTTP request")
require.Contains(t, entries[0].Message, "correlation_id=")
}) })
} }
func testMissing(t *testing.T, client *GitlabNetClient) { func testMissing(t *testing.T, client *GitlabNetClient) {
t.Run("Missing error for GET", func(t *testing.T) { t.Run("Missing error for GET", func(t *testing.T) {
hook := testhelper.SetupLogger()
response, err := client.Get(context.Background(), "/missing") response, err := client.Get(context.Background(), "/missing")
require.EqualError(t, err, "Internal API error (404)") require.EqualError(t, err, "Internal API error (404)")
require.Nil(t, response) require.Nil(t, response)
require.True(t, testhelper.WaitForLogEvent(hook))
entries := hook.AllEntries()
require.Equal(t, 1, len(entries))
require.Equal(t, logrus.InfoLevel, entries[0].Level)
require.Contains(t, entries[0].Message, "method=GET")
require.Contains(t, entries[0].Message, "status=404")
require.Contains(t, entries[0].Message, "Internal API error")
require.Contains(t, entries[0].Message, "correlation_id=")
}) })
t.Run("Missing error for POST", func(t *testing.T) { t.Run("Missing error for POST", func(t *testing.T) {
hook := testhelper.SetupLogger()
response, err := client.Post(context.Background(), "/missing", map[string]string{}) response, err := client.Post(context.Background(), "/missing", map[string]string{})
require.EqualError(t, err, "Internal API error (404)") require.EqualError(t, err, "Internal API error (404)")
require.Nil(t, response) require.Nil(t, response)
require.True(t, testhelper.WaitForLogEvent(hook))
entries := hook.AllEntries()
require.Equal(t, 1, len(entries))
require.Equal(t, logrus.InfoLevel, entries[0].Level)
require.Contains(t, entries[0].Message, "method=POST")
require.Contains(t, entries[0].Message, "status=404")
require.Contains(t, entries[0].Message, "Internal API error")
require.Contains(t, entries[0].Message, "correlation_id=")
}) })
} }
Loading
@@ -179,78 +162,86 @@ func testErrorMessage(t *testing.T, client *GitlabNetClient) {
Loading
@@ -179,78 +162,86 @@ func testErrorMessage(t *testing.T, client *GitlabNetClient) {
func testBrokenRequest(t *testing.T, client *GitlabNetClient) { func testBrokenRequest(t *testing.T, client *GitlabNetClient) {
t.Run("Broken request for GET", func(t *testing.T) { t.Run("Broken request for GET", func(t *testing.T) {
hook := testhelper.SetupLogger()
response, err := client.Get(context.Background(), "/broken") response, err := client.Get(context.Background(), "/broken")
require.EqualError(t, err, "Internal API unreachable") require.EqualError(t, err, "Internal API unreachable")
require.Nil(t, response) require.Nil(t, response)
require.True(t, testhelper.WaitForLogEvent(hook))
entries := hook.AllEntries()
require.Equal(t, 1, len(entries))
require.Equal(t, logrus.InfoLevel, entries[0].Level)
require.Contains(t, entries[0].Message, "method=GET")
require.NotContains(t, entries[0].Message, "status=")
require.Contains(t, entries[0].Message, "Internal API unreachable")
require.Contains(t, entries[0].Message, "correlation_id=")
}) })
t.Run("Broken request for POST", func(t *testing.T) { t.Run("Broken request for POST", func(t *testing.T) {
hook := testhelper.SetupLogger()
response, err := client.Post(context.Background(), "/broken", map[string]string{}) response, err := client.Post(context.Background(), "/broken", map[string]string{})
require.EqualError(t, err, "Internal API unreachable") require.EqualError(t, err, "Internal API unreachable")
require.Nil(t, response) require.Nil(t, response)
require.True(t, testhelper.WaitForLogEvent(hook))
entries := hook.AllEntries()
require.Equal(t, 1, len(entries))
require.Equal(t, logrus.InfoLevel, entries[0].Level)
require.Contains(t, entries[0].Message, "method=POST")
require.NotContains(t, entries[0].Message, "status=")
require.Contains(t, entries[0].Message, "Internal API unreachable")
require.Contains(t, entries[0].Message, "correlation_id=")
}) })
} }
func testAuthenticationHeader(t *testing.T, client *GitlabNetClient) { func testJWTAuthenticationHeader(t *testing.T, client *GitlabNetClient) {
t.Run("Authentication headers for GET", func(t *testing.T) { verifyJWTToken := func(t *testing.T, response *http.Response) {
response, err := client.Get(context.Background(), "/auth") responseBody, err := io.ReadAll(response.Body)
require.NoError(t, err) require.NoError(t, err)
require.NotNil(t, response)
defer response.Body.Close()
responseBody, err := ioutil.ReadAll(response.Body) claims := &jwt.RegisteredClaims{}
token, err := jwt.ParseWithClaims(string(responseBody), claims, func(token *jwt.Token) (interface{}, error) {
return []byte(secret), nil
})
require.NoError(t, err) require.NoError(t, err)
require.True(t, token.Valid)
require.Equal(t, "gitlab-shell", claims.Issuer)
require.WithinDuration(t, time.Now().Truncate(time.Second), claims.IssuedAt.Time, time.Second)
require.WithinDuration(t, time.Now().Truncate(time.Second).Add(time.Minute), claims.ExpiresAt.Time, time.Second)
}
header, err := base64.StdEncoding.DecodeString(string(responseBody)) t.Run("JWT authentication headers for GET", func(t *testing.T) {
response, err := client.Get(context.Background(), "/jwt_auth")
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, "sssh, it's a secret", string(header)) require.NotNil(t, response)
defer response.Body.Close()
verifyJWTToken(t, response)
}) })
t.Run("Authentication headers for POST", func(t *testing.T) { t.Run("JWT authentication headers for POST", func(t *testing.T) {
response, err := client.Post(context.Background(), "/auth", map[string]string{}) response, err := client.Post(context.Background(), "/jwt_auth", map[string]string{})
require.NoError(t, err) require.NoError(t, err)
require.NotNil(t, response) require.NotNil(t, response)
defer response.Body.Close() defer response.Body.Close()
responseBody, err := ioutil.ReadAll(response.Body) verifyJWTToken(t, response)
})
}
func testXForwardedForHeader(t *testing.T, client *GitlabNetClient) {
t.Run("X-Forwarded-For Header inserted if original address in context", func(t *testing.T) {
ctx := context.WithValue(context.Background(), OriginalRemoteIPContextKey{}, "196.7.0.238")
response, err := client.Get(ctx, "/x_forwarded_for")
require.NoError(t, err) require.NoError(t, err)
require.NotNil(t, response)
defer response.Body.Close()
header, err := base64.StdEncoding.DecodeString(string(responseBody)) responseBody, err := io.ReadAll(response.Body)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, "sssh, it's a secret", string(header)) require.Equal(t, "196.7.0.238", string(responseBody))
}) })
} }
func testHostWithTrailingSlash(t *testing.T, client *GitlabNetClient) {
oldHost := client.httpClient.Host
client.httpClient.Host = oldHost + "/"
testSuccessfulGet(t, client)
testSuccessfulPost(t, client)
client.httpClient.Host = oldHost
}
func buildRequests(t *testing.T, relativeURLRoot string) []testserver.TestRequestHandler { func buildRequests(t *testing.T, relativeURLRoot string) []testserver.TestRequestHandler {
requests := []testserver.TestRequestHandler{ requests := []testserver.TestRequestHandler{
{ {
Path: "/api/v4/internal/hello", Path: "/api/v4/internal/hello",
Handler: func(w http.ResponseWriter, r *http.Request) { Handler: func(w http.ResponseWriter, r *http.Request) {
require.Equal(t, http.MethodGet, r.Method) assert.Equal(t, http.MethodGet, r.Method)
fmt.Fprint(w, "Hello") fmt.Fprint(w, "Hello")
}, },
Loading
@@ -258,20 +249,26 @@ func buildRequests(t *testing.T, relativeURLRoot string) []testserver.TestReques
Loading
@@ -258,20 +249,26 @@ func buildRequests(t *testing.T, relativeURLRoot string) []testserver.TestReques
{ {
Path: "/api/v4/internal/post_endpoint", Path: "/api/v4/internal/post_endpoint",
Handler: func(w http.ResponseWriter, r *http.Request) { Handler: func(w http.ResponseWriter, r *http.Request) {
require.Equal(t, http.MethodPost, r.Method) assert.Equal(t, http.MethodPost, r.Method)
b, err := ioutil.ReadAll(r.Body) b, err := io.ReadAll(r.Body)
defer r.Body.Close() defer r.Body.Close()
require.NoError(t, err) assert.NoError(t, err)
fmt.Fprint(w, "Echo: "+string(b)) fmt.Fprint(w, "Echo: "+string(b))
}, },
}, },
{ {
Path: "/api/v4/internal/auth", Path: "/api/v4/internal/jwt_auth",
Handler: func(w http.ResponseWriter, r *http.Request) {
fmt.Fprint(w, r.Header.Get(apiSecretHeaderName))
},
},
{
Path: "/api/v4/internal/x_forwarded_for",
Handler: func(w http.ResponseWriter, r *http.Request) { Handler: func(w http.ResponseWriter, r *http.Request) {
fmt.Fprint(w, r.Header.Get(secretHeaderName)) fmt.Fprint(w, r.Header.Get("X-Forwarded-For"))
}, },
}, },
{ {
Loading
@@ -302,3 +299,22 @@ func buildRequests(t *testing.T, relativeURLRoot string) []testserver.TestReques
Loading
@@ -302,3 +299,22 @@ func buildRequests(t *testing.T, relativeURLRoot string) []testserver.TestReques
return requests return requests
} }
func TestRetryOnFailure(t *testing.T) {
reqAttempts := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
reqAttempts++
w.WriteHeader(500)
}))
defer srv.Close()
httpClient, err := NewHTTPClientWithOpts(srv.URL, "/", "", "", 1, defaultHttpOpts)
require.NoError(t, err)
require.NotNil(t, httpClient.RetryableHTTP)
client, err := NewGitlabNetClient("", "", "", httpClient)
require.NoError(t, err)
_, err = client.Get(context.Background(), "/")
require.EqualError(t, err, "Internal API unreachable")
require.Equal(t, 3, reqAttempts)
}
// Package client provides a client for interacting with GitLab API
package client package client
import ( import (
"bytes" "bytes"
"context" "context"
"encoding/base64"
"encoding/json" "encoding/json"
"fmt" "fmt"
"io" "io"
Loading
@@ -11,38 +11,53 @@ import (
Loading
@@ -11,38 +11,53 @@ import (
"strings" "strings"
"time" "time"
"gitlab.com/gitlab-org/labkit/correlation" "github.com/golang-jwt/jwt/v5"
"github.com/hashicorp/go-retryablehttp"
log "github.com/sirupsen/logrus"
) )
const ( const (
internalApiPath = "/api/v4/internal" internalAPIPath = "/api/v4/internal"
secretHeaderName = "Gitlab-Shared-Secret" apiSecretHeaderName = "Gitlab-Shell-Api-Request" // #nosec G101
defaultUserAgent = "GitLab-Shell" defaultUserAgent = "GitLab-Shell"
jwtTTL = time.Minute
jwtIssuer = "gitlab-shell"
) )
// ErrorResponse represents an error response from the API
type ErrorResponse struct { type ErrorResponse struct {
Message string `json:"message"` Message string `json:"message"`
} }
// GitlabNetClient is a client for interacting with GitLab API
type GitlabNetClient struct { type GitlabNetClient struct {
httpClient *HttpClient httpClient *HTTPClient
user string user string
password string password string
secret string secret string
userAgent string userAgent string
} }
// APIError represents an API error
type APIError struct {
Msg string
}
// OriginalRemoteIPContextKey is used as the key in a Context to set an X-Forwarded-For header in a request
type OriginalRemoteIPContextKey struct{}
func (e *APIError) Error() string {
return e.Msg
}
// NewGitlabNetClient creates a new GitlabNetClient instance
func NewGitlabNetClient( func NewGitlabNetClient(
user, user,
password, password,
secret string, secret string,
httpClient *HttpClient, httpClient *HTTPClient,
) (*GitlabNetClient, error) { ) (*GitlabNetClient, error) {
if httpClient == nil { if httpClient == nil {
return nil, fmt.Errorf("Unsupported protocol") return nil, fmt.Errorf("unsupported protocol")
} }
return &GitlabNetClient{ return &GitlabNetClient{
Loading
@@ -65,58 +80,75 @@ func normalizePath(path string) string {
Loading
@@ -65,58 +80,75 @@ func normalizePath(path string) string {
path = "/" + path path = "/" + path
} }
if !strings.HasPrefix(path, internalApiPath) { if !strings.HasPrefix(path, internalAPIPath) {
path = internalApiPath + path path = internalAPIPath + path
} }
return path return path
} }
func newRequest(ctx context.Context, method, host, path string, data interface{}) (*http.Request, string, error) { func appendPath(host string, path string) string {
return strings.TrimSuffix(host, "/") + "/" + strings.TrimPrefix(path, "/")
}
func newRequest(ctx context.Context, method, host, path string, data interface{}) (*retryablehttp.Request, error) {
var jsonReader io.Reader var jsonReader io.Reader
if data != nil { if data != nil {
jsonData, err := json.Marshal(data) jsonData, err := json.Marshal(data)
if err != nil { if err != nil {
return nil, "", err return nil, err
} }
jsonReader = bytes.NewReader(jsonData) jsonReader = bytes.NewReader(jsonData)
} }
request, err := http.NewRequestWithContext(ctx, method, host+path, jsonReader) request, err := retryablehttp.NewRequestWithContext(ctx, method, appendPath(host, path), jsonReader)
if err != nil { if err != nil {
return nil, "", err return nil, err
} }
correlationID := correlation.ExtractFromContext(ctx) return request, nil
return request, correlationID, nil
} }
func parseError(resp *http.Response) error { func parseError(resp *http.Response, respErr error) error {
if resp == nil || respErr != nil {
return &APIError{"Internal API unreachable"}
}
if resp.StatusCode >= 200 && resp.StatusCode <= 399 { if resp.StatusCode >= 200 && resp.StatusCode <= 399 {
return nil return nil
} }
defer resp.Body.Close() defer func() { _ = resp.Body.Close() }()
parsedResponse := &ErrorResponse{} parsedResponse := &ErrorResponse{}
if err := json.NewDecoder(resp.Body).Decode(parsedResponse); err != nil { if err := json.NewDecoder(resp.Body).Decode(parsedResponse); err != nil {
return fmt.Errorf("Internal API error (%v)", resp.StatusCode) return &APIError{fmt.Sprintf("Internal API error (%v)", resp.StatusCode)}
} else {
return fmt.Errorf(parsedResponse.Message)
} }
return &APIError{parsedResponse.Message}
} }
// Get makes a GET request
func (c *GitlabNetClient) Get(ctx context.Context, path string) (*http.Response, error) { func (c *GitlabNetClient) Get(ctx context.Context, path string) (*http.Response, error) {
return c.DoRequest(ctx, http.MethodGet, normalizePath(path), nil) return c.DoRequest(ctx, http.MethodGet, normalizePath(path), nil)
} }
// Post makes a POST request
func (c *GitlabNetClient) Post(ctx context.Context, path string, data interface{}) (*http.Response, error) { func (c *GitlabNetClient) Post(ctx context.Context, path string, data interface{}) (*http.Response, error) {
return c.DoRequest(ctx, http.MethodPost, normalizePath(path), data) return c.DoRequest(ctx, http.MethodPost, normalizePath(path), data)
} }
// Do executes a request
func (c *GitlabNetClient) Do(request *http.Request) (*http.Response, error) {
response, respErr := c.httpClient.RetryableHTTP.HTTPClient.Do(request)
if err := parseError(response, respErr); err != nil {
return nil, err
}
return response, nil
}
// DoRequest executes a request with the given method, path, and data
func (c *GitlabNetClient) DoRequest(ctx context.Context, method, path string, data interface{}) (*http.Response, error) { func (c *GitlabNetClient) DoRequest(ctx context.Context, method, path string, data interface{}) (*http.Response, error) {
request, correlationID, err := newRequest(ctx, method, c.httpClient.Host, path, data) request, err := newRequest(ctx, method, c.httpClient.Host, path, data)
if err != nil { if err != nil {
return nil, err return nil, err
} }
Loading
@@ -126,41 +158,25 @@ func (c *GitlabNetClient) DoRequest(ctx context.Context, method, path string, da
Loading
@@ -126,41 +158,25 @@ func (c *GitlabNetClient) DoRequest(ctx context.Context, method, path string, da
request.SetBasicAuth(user, password) request.SetBasicAuth(user, password)
} }
encodedSecret := base64.StdEncoding.EncodeToString([]byte(c.secret)) claims := jwt.RegisteredClaims{
request.Header.Set(secretHeaderName, encodedSecret) Issuer: jwtIssuer,
IssuedAt: jwt.NewNumericDate(time.Now()),
request.Header.Add("Content-Type", "application/json") ExpiresAt: jwt.NewNumericDate(time.Now().Add(jwtTTL)),
request.Header.Add("User-Agent", c.userAgent)
request.Close = true
start := time.Now()
response, err := c.httpClient.Do(request)
fields := log.Fields{
"correlation_id": correlationID,
"method": method,
"url": request.URL.String(),
"duration_ms": time.Since(start) / time.Millisecond,
} }
logger := log.WithFields(fields) secretBytes := []byte(strings.TrimSpace(c.secret))
tokenString, err := jwt.NewWithClaims(jwt.SigningMethodHS256, claims).SignedString(secretBytes)
if err != nil { if err != nil {
logger.WithError(err).Error("Internal API unreachable")
return nil, fmt.Errorf("Internal API unreachable")
}
if response != nil {
logger = logger.WithField("status", response.StatusCode)
}
if err := parseError(response); err != nil {
logger.WithError(err).Error("Internal API error")
return nil, err return nil, err
} }
request.Header.Set(apiSecretHeaderName, tokenString)
if response.ContentLength >= 0 { request.Header.Add("Content-Type", "application/json")
logger = logger.WithField("content_length_bytes", response.ContentLength) request.Header.Add("User-Agent", c.userAgent)
}
logger.Info("Finished HTTP request") response, respErr := c.httpClient.RetryableHTTP.Do(request)
if err := parseError(response, respErr); err != nil {
return nil, err
}
return response, nil return response, nil
} }
// Package client provides functionality for interacting with HTTP clients
package client package client
import ( import (
Loading
@@ -5,33 +6,42 @@ import (
Loading
@@ -5,33 +6,42 @@ import (
"crypto/tls" "crypto/tls"
"crypto/x509" "crypto/x509"
"errors" "errors"
"io/ioutil" "fmt"
"net" "net"
"net/http" "net/http"
"os"
"path/filepath" "path/filepath"
"strings" "strings"
"time" "time"
log "github.com/sirupsen/logrus" "github.com/hashicorp/go-retryablehttp"
"gitlab.com/gitlab-org/labkit/correlation"
) )
const ( const (
socketBaseUrl = "http://unix" socketBaseURL = "http://unix"
unixSocketProtocol = "http+unix://" unixSocketProtocol = "http+unix://"
httpProtocol = "http://" httpProtocol = "http://"
httpsProtocol = "https://" httpsProtocol = "https://"
defaultReadTimeoutSeconds = 300 defaultReadTimeoutSeconds = 300
defaultRetryWaitMinimum = time.Second
defaultRetryWaitMaximum = 15 * time.Second
defaultRetryMax = 2
) )
type HttpClient struct { // ErrCafileNotFound indicates that the specified CA file was not found
*http.Client var ErrCafileNotFound = errors.New("cafile not found")
Host string
// HTTPClient provides an HTTP client with retry capabilities
type HTTPClient struct {
RetryableHTTP *retryablehttp.Client
Host string
} }
type httpClientCfg struct { type httpClientCfg struct {
keyPath, certPath string keyPath, certPath string
caFile, caPath string caFile, caPath string
retryWaitMin, retryWaitMax time.Duration
retryMax int
} }
func (hcc httpClientCfg) HaveCertAndKey() bool { return hcc.keyPath != "" && hcc.certPath != "" } func (hcc httpClientCfg) HaveCertAndKey() bool { return hcc.keyPath != "" && hcc.certPath != "" }
Loading
@@ -48,20 +58,39 @@ func WithClientCert(certPath, keyPath string) HTTPClientOpt {
Loading
@@ -48,20 +58,39 @@ func WithClientCert(certPath, keyPath string) HTTPClientOpt {
} }
} }
// Deprecated: use NewHTTPClientWithOpts - https://gitlab.com/gitlab-org/gitlab-shell/-/issues/484 // WithHTTPRetryOpts configures HTTP retry options for the HttpClient
func NewHTTPClient(gitlabURL, gitlabRelativeURLRoot, caFile, caPath string, selfSignedCert bool, readTimeoutSeconds uint64) *HttpClient { func WithHTTPRetryOpts(waitMin, waitMax time.Duration, maxAttempts int) HTTPClientOpt {
c, err := NewHTTPClientWithOpts(gitlabURL, gitlabRelativeURLRoot, caFile, caPath, selfSignedCert, readTimeoutSeconds, nil) return func(hcc *httpClientCfg) {
if err != nil { hcc.retryWaitMin = waitMin
log.WithError(err).Error("new http client with opts") hcc.retryWaitMax = waitMax
hcc.retryMax = maxAttempts
} }
return c }
func validateCaFile(filename string) error {
if filename == "" {
return nil
}
if _, err := os.Stat(filename); err != nil {
if os.IsNotExist(err) {
return fmt.Errorf("cannot find cafile '%s': %w", filename, ErrCafileNotFound)
}
return err
}
return nil
} }
// NewHTTPClientWithOpts builds an HTTP client using the provided options // NewHTTPClientWithOpts builds an HTTP client using the provided options
func NewHTTPClientWithOpts(gitlabURL, gitlabRelativeURLRoot, caFile, caPath string, selfSignedCert bool, readTimeoutSeconds uint64, opts []HTTPClientOpt) (*HttpClient, error) { func NewHTTPClientWithOpts(gitlabURL, gitlabRelativeURLRoot, caFile, caPath string, readTimeoutSeconds uint64, opts []HTTPClientOpt) (*HTTPClient, error) {
hcc := &httpClientCfg{ hcc := &httpClientCfg{
caFile: caFile, caFile: caFile,
caPath: caPath, caPath: caPath,
retryWaitMin: defaultRetryWaitMinimum,
retryWaitMax: defaultRetryWaitMaximum,
retryMax: defaultRetryMax,
} }
for _, opt := range opts { for _, opt := range opts {
Loading
@@ -71,25 +100,33 @@ func NewHTTPClientWithOpts(gitlabURL, gitlabRelativeURLRoot, caFile, caPath stri
Loading
@@ -71,25 +100,33 @@ func NewHTTPClientWithOpts(gitlabURL, gitlabRelativeURLRoot, caFile, caPath stri
var transport *http.Transport var transport *http.Transport
var host string var host string
var err error var err error
if strings.HasPrefix(gitlabURL, unixSocketProtocol) { switch {
case strings.HasPrefix(gitlabURL, unixSocketProtocol):
transport, host = buildSocketTransport(gitlabURL, gitlabRelativeURLRoot) transport, host = buildSocketTransport(gitlabURL, gitlabRelativeURLRoot)
} else if strings.HasPrefix(gitlabURL, httpProtocol) { case strings.HasPrefix(gitlabURL, httpProtocol):
transport, host = buildHttpTransport(gitlabURL) transport, host = buildHTTPTransport(gitlabURL)
} else if strings.HasPrefix(gitlabURL, httpsProtocol) { case strings.HasPrefix(gitlabURL, httpsProtocol):
transport, host, err = buildHttpsTransport(*hcc, selfSignedCert, gitlabURL) err = validateCaFile(caFile)
if err != nil {
return nil, err
}
transport, host, err = buildHTTPSTransport(*hcc, gitlabURL)
if err != nil { if err != nil {
return nil, err return nil, err
} }
} else { default:
return nil, errors.New("unknown GitLab URL prefix") return nil, errors.New("unknown GitLab URL prefix")
} }
c := &http.Client{ c := retryablehttp.NewClient()
Transport: correlation.NewInstrumentedRoundTripper(transport), c.RetryMax = hcc.retryMax
Timeout: readTimeout(readTimeoutSeconds), c.RetryWaitMax = hcc.retryWaitMax
} c.RetryWaitMin = hcc.retryWaitMin
c.Logger = nil
c.HTTPClient.Transport = NewTransport(transport)
c.HTTPClient.Timeout = readTimeout(readTimeoutSeconds)
client := &HttpClient{Client: c, Host: host} client := &HTTPClient{RetryableHTTP: c, Host: host}
return client, nil return client, nil
} }
Loading
@@ -104,7 +141,7 @@ func buildSocketTransport(gitlabURL, gitlabRelativeURLRoot string) (*http.Transp
Loading
@@ -104,7 +141,7 @@ func buildSocketTransport(gitlabURL, gitlabRelativeURLRoot string) (*http.Transp
}, },
} }
host := socketBaseUrl host := socketBaseURL
gitlabRelativeURLRoot = strings.Trim(gitlabRelativeURLRoot, "/") gitlabRelativeURLRoot = strings.Trim(gitlabRelativeURLRoot, "/")
if gitlabRelativeURLRoot != "" { if gitlabRelativeURLRoot != "" {
host = host + "/" + gitlabRelativeURLRoot host = host + "/" + gitlabRelativeURLRoot
Loading
@@ -113,9 +150,8 @@ func buildSocketTransport(gitlabURL, gitlabRelativeURLRoot string) (*http.Transp
Loading
@@ -113,9 +150,8 @@ func buildSocketTransport(gitlabURL, gitlabRelativeURLRoot string) (*http.Transp
return transport, host return transport, host
} }
func buildHttpsTransport(hcc httpClientCfg, selfSignedCert bool, gitlabURL string) (*http.Transport, string, error) { func buildHTTPSTransport(hcc httpClientCfg, gitlabURL string) (*http.Transport, string, error) {
certPool, err := x509.SystemCertPool() certPool, err := x509.SystemCertPool()
if err != nil { if err != nil {
certPool = x509.NewCertPool() certPool = x509.NewCertPool()
} }
Loading
@@ -125,7 +161,7 @@ func buildHttpsTransport(hcc httpClientCfg, selfSignedCert bool, gitlabURL strin
Loading
@@ -125,7 +161,7 @@ func buildHttpsTransport(hcc httpClientCfg, selfSignedCert bool, gitlabURL strin
} }
if hcc.caPath != "" { if hcc.caPath != "" {
fis, _ := ioutil.ReadDir(hcc.caPath) fis, _ := os.ReadDir(hcc.caPath)
for _, fi := range fis { for _, fi := range fis {
if fi.IsDir() { if fi.IsDir() {
continue continue
Loading
@@ -135,17 +171,16 @@ func buildHttpsTransport(hcc httpClientCfg, selfSignedCert bool, gitlabURL strin
Loading
@@ -135,17 +171,16 @@ func buildHttpsTransport(hcc httpClientCfg, selfSignedCert bool, gitlabURL strin
} }
} }
tlsConfig := &tls.Config{ tlsConfig := &tls.Config{
RootCAs: certPool, RootCAs: certPool,
InsecureSkipVerify: selfSignedCert, MinVersion: tls.VersionTLS12,
} }
if hcc.HaveCertAndKey() { if hcc.HaveCertAndKey() {
cert, err := tls.LoadX509KeyPair(hcc.certPath, hcc.keyPath) cert, loadErr := tls.LoadX509KeyPair(hcc.certPath, hcc.keyPath)
if err != nil { if loadErr != nil {
return nil, "", err return nil, "", loadErr
} }
tlsConfig.Certificates = []tls.Certificate{cert} tlsConfig.Certificates = []tls.Certificate{cert}
tlsConfig.BuildNameToCertificate()
} }
transport := &http.Transport{ transport := &http.Transport{
Loading
@@ -156,13 +191,13 @@ func buildHttpsTransport(hcc httpClientCfg, selfSignedCert bool, gitlabURL strin
Loading
@@ -156,13 +191,13 @@ func buildHttpsTransport(hcc httpClientCfg, selfSignedCert bool, gitlabURL strin
} }
func addCertToPool(certPool *x509.CertPool, fileName string) { func addCertToPool(certPool *x509.CertPool, fileName string) {
cert, err := ioutil.ReadFile(fileName) cert, err := os.ReadFile(filepath.Clean(fileName))
if err == nil { if err == nil {
certPool.AppendCertsFromPEM(cert) certPool.AppendCertsFromPEM(cert)
} }
} }
func buildHttpTransport(gitlabURL string) (*http.Transport, string) { func buildHTTPTransport(gitlabURL string) (*http.Transport, string) {
return &http.Transport{}, gitlabURL return &http.Transport{}, gitlabURL
} }
Loading
Loading
Loading
@@ -4,23 +4,25 @@ import (
Loading
@@ -4,23 +4,25 @@ import (
"context" "context"
"encoding/base64" "encoding/base64"
"fmt" "fmt"
"io/ioutil" "io"
"net/http" "net/http"
"strings" "strings"
"testing" "testing"
"time" "time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"gitlab.com/gitlab-org/gitlab-shell/client/testserver" "gitlab.com/gitlab-org/gitlab-shell/v14/client/testserver"
) )
func TestReadTimeout(t *testing.T) { func TestReadTimeout(t *testing.T) {
expectedSeconds := uint64(300) expectedSeconds := uint64(300)
client := NewHTTPClient("http://localhost:3000", "", "", "", false, expectedSeconds) client, err := NewHTTPClientWithOpts("http://localhost:3000", "", "", "", expectedSeconds, nil)
require.NoError(t, err)
require.NotNil(t, client) require.NotNil(t, client)
require.Equal(t, time.Duration(expectedSeconds)*time.Second, client.Client.Timeout) require.Equal(t, time.Duration(expectedSeconds)*time.Second, client.RetryableHTTP.HTTPClient.Timeout)
} }
const ( const (
Loading
@@ -33,7 +35,7 @@ func TestBasicAuthSettings(t *testing.T) {
Loading
@@ -33,7 +35,7 @@ func TestBasicAuthSettings(t *testing.T) {
{ {
Path: "/api/v4/internal/get_endpoint", Path: "/api/v4/internal/get_endpoint",
Handler: func(w http.ResponseWriter, r *http.Request) { Handler: func(w http.ResponseWriter, r *http.Request) {
require.Equal(t, http.MethodGet, r.Method) assert.Equal(t, http.MethodGet, r.Method)
fmt.Fprint(w, r.Header.Get("Authorization")) fmt.Fprint(w, r.Header.Get("Authorization"))
}, },
Loading
@@ -41,15 +43,14 @@ func TestBasicAuthSettings(t *testing.T) {
Loading
@@ -41,15 +43,14 @@ func TestBasicAuthSettings(t *testing.T) {
{ {
Path: "/api/v4/internal/post_endpoint", Path: "/api/v4/internal/post_endpoint",
Handler: func(w http.ResponseWriter, r *http.Request) { Handler: func(w http.ResponseWriter, r *http.Request) {
require.Equal(t, http.MethodPost, r.Method) assert.Equal(t, http.MethodPost, r.Method)
fmt.Fprint(w, r.Header.Get("Authorization")) fmt.Fprint(w, r.Header.Get("Authorization"))
}, },
}, },
} }
client, cleanup := setup(t, username, password, requests) client := setup(t, username, password, requests)
defer cleanup()
response, err := client.Get(context.Background(), "/get_endpoint") response, err := client.Get(context.Background(), "/get_endpoint")
require.NoError(t, err) require.NoError(t, err)
Loading
@@ -64,10 +65,11 @@ func testBasicAuthHeaders(t *testing.T, response *http.Response) {
Loading
@@ -64,10 +65,11 @@ func testBasicAuthHeaders(t *testing.T, response *http.Response) {
defer response.Body.Close() defer response.Body.Close()
require.NotNil(t, response) require.NotNil(t, response)
responseBody, err := ioutil.ReadAll(response.Body) responseBody, err := io.ReadAll(response.Body)
require.NoError(t, err) require.NoError(t, err)
headerParts := strings.Split(string(responseBody), " ") headerParts := strings.Split(string(responseBody), " ")
require.NotNil(t, headerParts)
require.Equal(t, "Basic", headerParts[0]) require.Equal(t, "Basic", headerParts[0])
credentials, err := base64.StdEncoding.DecodeString(headerParts[1]) credentials, err := base64.StdEncoding.DecodeString(headerParts[1])
Loading
@@ -80,17 +82,17 @@ func TestEmptyBasicAuthSettings(t *testing.T) {
Loading
@@ -80,17 +82,17 @@ func TestEmptyBasicAuthSettings(t *testing.T) {
requests := []testserver.TestRequestHandler{ requests := []testserver.TestRequestHandler{
{ {
Path: "/api/v4/internal/empty_basic_auth", Path: "/api/v4/internal/empty_basic_auth",
Handler: func(w http.ResponseWriter, r *http.Request) { Handler: func(_ http.ResponseWriter, r *http.Request) {
require.Equal(t, "", r.Header.Get("Authorization")) assert.Equal(t, "", r.Header.Get("Authorization"))
}, },
}, },
} }
client, cleanup := setup(t, "", "", requests) client := setup(t, "", "", requests)
defer cleanup()
_, err := client.Get(context.Background(), "/empty_basic_auth") resp, err := client.Get(context.Background(), "/empty_basic_auth")
require.NoError(t, err) require.NoError(t, err)
resp.Body.Close()
} }
func TestRequestWithUserAgent(t *testing.T) { func TestRequestWithUserAgent(t *testing.T) {
Loading
@@ -98,37 +100,39 @@ func TestRequestWithUserAgent(t *testing.T) {
Loading
@@ -98,37 +100,39 @@ func TestRequestWithUserAgent(t *testing.T) {
requests := []testserver.TestRequestHandler{ requests := []testserver.TestRequestHandler{
{ {
Path: "/api/v4/internal/default_user_agent", Path: "/api/v4/internal/default_user_agent",
Handler: func(w http.ResponseWriter, r *http.Request) { Handler: func(_ http.ResponseWriter, r *http.Request) {
require.Equal(t, defaultUserAgent, r.UserAgent()) assert.Equal(t, defaultUserAgent, r.UserAgent())
}, },
}, },
{ {
Path: "/api/v4/internal/override_user_agent", Path: "/api/v4/internal/override_user_agent",
Handler: func(w http.ResponseWriter, r *http.Request) { Handler: func(_ http.ResponseWriter, r *http.Request) {
require.Equal(t, gitalyUserAgent, r.UserAgent()) assert.Equal(t, gitalyUserAgent, r.UserAgent())
}, },
}, },
} }
client, cleanup := setup(t, "", "", requests) client := setup(t, "", "", requests)
defer cleanup()
_, err := client.Get(context.Background(), "/default_user_agent") defaultUserAgentResp, err := client.Get(context.Background(), "/default_user_agent")
require.NoError(t, err) require.NoError(t, err)
client.SetUserAgent(gitalyUserAgent) client.SetUserAgent(gitalyUserAgent)
_, err = client.Get(context.Background(), "/override_user_agent") overriddenUserAgentResp, err := client.Get(context.Background(), "/override_user_agent")
require.NoError(t, err) require.NoError(t, err)
defaultUserAgentResp.Body.Close()
overriddenUserAgentResp.Body.Close()
} }
func setup(t *testing.T, username, password string, requests []testserver.TestRequestHandler) (*GitlabNetClient, func()) { func setup(t *testing.T, username, password string, requests []testserver.TestRequestHandler) *GitlabNetClient {
url, cleanup := testserver.StartHttpServer(t, requests) url := testserver.StartHTTPServer(t, requests)
httpClient := NewHTTPClient(url, "", "", "", false, 1) httpClient, err := NewHTTPClientWithOpts(url, "", "", "", 1, nil)
require.NoError(t, err)
client, err := NewGitlabNetClient(username, password, "", httpClient) client, err := NewGitlabNetClient(username, password, "", httpClient)
require.NoError(t, err) require.NoError(t, err)
return client, cleanup return client
} }