diff options
| author | Achilleas Pipinellis <axil@gitlab.com> | 2019-08-20 20:22:43 +0200 |
|---|---|---|
| committer | Achilleas Pipinellis <axil@gitlab.com> | 2019-08-20 20:22:43 +0200 |
| commit | e61308ce1d82e12e5087371469baea4a452875d1 (patch) | |
| tree | 62551a3ae4eab75e5af7e3b35358c07a51b2132f /lib | |
| parent | 4f323bb62fbe71a4352de25cab141f361a3fe1a6 (diff) | |
| parent | 2989ed078c1d45b0959dcecb1bc3c8f4740a3c0d (diff) | |
| download | gitlab-ce-docs-patch-71.tar.gz | |
Merge branch 'master' into docs-patch-71docs-patch-71
Diffstat (limited to 'lib')
489 files changed, 8364 insertions, 5143 deletions
diff --git a/lib/after_commit_queue.rb b/lib/after_commit_queue.rb index 6fb7985f955..6a180fdf338 100644 --- a/lib/after_commit_queue.rb +++ b/lib/after_commit_queue.rb @@ -15,7 +15,7 @@ module AfterCommitQueue end def run_after_commit_or_now(&block) - if AfterCommitQueue.inside_transaction? + if Gitlab::Database.inside_transaction? if ActiveRecord::Base.connection.current_transaction.records.include?(self) run_after_commit(&block) else @@ -32,18 +32,6 @@ module AfterCommitQueue true end - def self.open_transactions_baseline - if ::Rails.env.test? - return DatabaseCleaner.connections.count { |conn| conn.strategy.is_a?(DatabaseCleaner::ActiveRecord::Transaction) } - end - - 0 - end - - def self.inside_transaction? - ActiveRecord::Base.connection.open_transactions > open_transactions_baseline - end - protected def _run_after_commit_queue diff --git a/lib/api/api.rb b/lib/api/api.rb index 20f8c637274..219ed45eff6 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -10,14 +10,15 @@ module API NAMESPACE_OR_PROJECT_REQUIREMENTS = { id: NO_SLASH_URL_PART_REGEX }.freeze COMMIT_ENDPOINT_REQUIREMENTS = NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(sha: NO_SLASH_URL_PART_REGEX).freeze USER_REQUIREMENTS = { user_id: NO_SLASH_URL_PART_REGEX }.freeze + LOG_FILTERS = ::Rails.application.config.filter_parameters + [/^output$/] insert_before Grape::Middleware::Error, GrapeLogging::Middleware::RequestLogger, logger: Logger.new(LOG_FILENAME), formatter: Gitlab::GrapeLogging::Formatters::LogrageWithTimestamp.new, include: [ - GrapeLogging::Loggers::FilterParameters.new, - GrapeLogging::Loggers::ClientEnv.new, + GrapeLogging::Loggers::FilterParameters.new(LOG_FILTERS), + Gitlab::GrapeLogging::Loggers::ClientEnvLogger.new, Gitlab::GrapeLogging::Loggers::RouteLogger.new, Gitlab::GrapeLogging::Loggers::UserLogger.new, Gitlab::GrapeLogging::Loggers::QueueDurationLogger.new, @@ -52,7 +53,10 @@ module API rack_response({ 'message' => '404 Not found' }.to_json, 404) end - rescue_from ::Gitlab::ExclusiveLeaseHelpers::FailedToObtainLockError do + rescue_from( + ::ActiveRecord::StaleObjectError, + ::Gitlab::ExclusiveLeaseHelpers::FailedToObtainLockError + ) do rack_response({ 'message' => '409 Conflict: Resource lock' }.to_json, 409) end @@ -100,7 +104,6 @@ module API mount ::API::BroadcastMessages mount ::API::Commits mount ::API::CommitStatuses - mount ::API::ContainerRegistry mount ::API::DeployKeys mount ::API::Deployments mount ::API::Environments @@ -108,9 +111,11 @@ module API mount ::API::Features mount ::API::Files mount ::API::GroupBoards + mount ::API::GroupClusters mount ::API::GroupLabels mount ::API::GroupMilestones mount ::API::Groups + mount ::API::GroupContainerRepositories mount ::API::GroupVariables mount ::API::ImportGithub mount ::API::Internal @@ -133,6 +138,7 @@ module API mount ::API::Pipelines mount ::API::PipelineSchedules mount ::API::ProjectClusters + mount ::API::ProjectContainerRepositories mount ::API::ProjectEvents mount ::API::ProjectExport mount ::API::ProjectImport @@ -163,6 +169,7 @@ module API mount ::API::Templates mount ::API::Todos mount ::API::Triggers + mount ::API::UserCounts mount ::API::Users mount ::API::Variables mount ::API::Version diff --git a/lib/api/boards.rb b/lib/api/boards.rb index b7c77730afb..4e31f74f18a 100644 --- a/lib/api/boards.rb +++ b/lib/api/boards.rb @@ -27,7 +27,7 @@ module API end get '/' do authorize!(:read_board, user_project) - present paginate(board_parent.boards), with: Entities::Board + present paginate(board_parent.boards.with_associations), with: Entities::Board end desc 'Find a project board' do diff --git a/lib/api/boards_responses.rb b/lib/api/boards_responses.rb index 86d9b24802f..68497a08fb8 100644 --- a/lib/api/boards_responses.rb +++ b/lib/api/boards_responses.rb @@ -11,7 +11,7 @@ module API end def board_lists - board.lists.destroyable + board.destroyable_lists end def create_list diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 65d7f68bbf9..c3821630b6b 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -8,7 +8,10 @@ module API BRANCH_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(branch: API::NO_SLASH_URL_PART_REGEX) - before { authorize! :download_code, user_project } + before do + require_repository_enabled! + authorize! :download_code, user_project + end helpers do params :filter_params do diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index 08b4f8db8b0..d58a5e214ed 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -52,6 +52,7 @@ module API optional :name, type: String, desc: 'A string label to differentiate this status from the status of other systems. Default: "default"' optional :context, type: String, desc: 'A string label to differentiate this status from the status of other systems. Default: "default"' optional :coverage, type: Float, desc: 'The total code coverage' + optional :pipeline_id, type: Integer, desc: 'An existing pipeline ID, when multiple pipelines on the same commit SHA have been triggered' end # rubocop: disable CodeReuse/ActiveRecord post ':id/statuses/:sha' do @@ -73,7 +74,8 @@ module API name = params[:name] || params[:context] || 'default' - pipeline = @project.pipeline_for(ref, commit.sha) + pipeline = @project.pipeline_for(ref, commit.sha, params[:pipeline_id]) + unless pipeline pipeline = @project.ci_pipelines.create!( source: :external, diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 80913f4ca07..a2f3e87ebd2 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -6,7 +6,10 @@ module API class Commits < Grape::API include PaginationParams - before { authorize! :download_code, user_project } + before do + require_repository_enabled! + authorize! :download_code, user_project + end helpers do def user_access @@ -40,7 +43,7 @@ module API path = params[:path] before = params[:until] after = params[:since] - ref = params[:ref_name] || user_project.try(:default_branch) || 'master' unless params[:all] + ref = params[:ref_name].presence || user_project.try(:default_branch) || 'master' unless params[:all] offset = (params[:page] - 1) * params[:per_page] all = params[:all] with_stats = params[:with_stats] @@ -73,7 +76,7 @@ module API detail 'This feature was introduced in GitLab 8.13' end params do - requires :branch, type: String, desc: 'Name of the branch to commit into. To create a new branch, also provide `start_branch`.', allow_blank: false + requires :branch, type: String, desc: 'Name of the branch to commit into. To create a new branch, also provide either `start_branch` or `start_sha`, and optionally `start_project`.', allow_blank: false requires :commit_message, type: String, desc: 'Commit message' requires :actions, type: Array, desc: 'Actions to perform in commit' do requires :action, type: String, desc: 'The action to perform, `create`, `delete`, `move`, `update`, `chmod`', values: %w[create update move delete chmod].freeze @@ -95,12 +98,16 @@ module API requires :execute_filemode, type: Boolean, desc: 'When `true/false` enables/disables the execute flag on the file.' end end - optional :start_branch, type: String, desc: 'Name of the branch to start the new commit from' - optional :start_project, types: [Integer, String], desc: 'The ID or path of the project to start the commit from' + + optional :start_branch, type: String, desc: 'Name of the branch to start the new branch from' + optional :start_sha, type: String, desc: 'SHA of the commit to start the new branch from' + mutually_exclusive :start_branch, :start_sha + + optional :start_project, types: [Integer, String], desc: 'The ID or path of the project to start the new branch from' optional :author_email, type: String, desc: 'Author email for commit' optional :author_name, type: String, desc: 'Author name for commit' optional :stats, type: Boolean, default: true, desc: 'Include commit stats' - optional :force, type: Boolean, default: false, desc: 'When `true` overwrites the target branch with a new commit based on the `start_branch`' + optional :force, type: Boolean, default: false, desc: 'When `true` overwrites the target branch with a new commit based on the `start_branch` or `start_sha`' end post ':id/repository/commits' do if params[:start_project] @@ -115,7 +122,7 @@ module API attrs = declared_params attrs[:branch_name] = attrs.delete(:branch) - attrs[:start_branch] ||= attrs[:branch_name] + attrs[:start_branch] ||= attrs[:branch_name] unless attrs[:start_sha] attrs[:start_project] = start_project if start_project result = ::Files::MultiService.new(user_project, current_user, attrs).execute @@ -123,7 +130,7 @@ module API if result[:status] == :success commit_detail = user_project.repository.commit(result[:result]) - Gitlab::WebIdeCommitsCounter.increment if find_user_from_warden + Gitlab::UsageDataCounters::WebIdeCounter.increment_commits_count if find_user_from_warden present commit_detail, with: Entities::CommitDetail, stats: params[:stats] else diff --git a/lib/api/discussions.rb b/lib/api/discussions.rb index 693172b7d08..6c1acc3963f 100644 --- a/lib/api/discussions.rb +++ b/lib/api/discussions.rb @@ -4,6 +4,7 @@ module API class Discussions < Grape::API include PaginationParams helpers ::API::Helpers::NotesHelpers + helpers ::RendersNotes before { authenticate! } @@ -23,21 +24,15 @@ module API requires :noteable_id, types: [Integer, String], desc: 'The ID of the noteable' use :pagination end - # rubocop: disable CodeReuse/ActiveRecord - get ":id/#{noteables_path}/:noteable_id/discussions" do - noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) - notes = noteable.notes - .inc_relations_for_view - .includes(:noteable) - .fresh + get ":id/#{noteables_path}/:noteable_id/discussions" do + noteable = find_noteable(parent_type, params[:id], noteable_type, params[:noteable_id]) - notes = notes.reject { |n| n.cross_reference_not_visible_for?(current_user) } + notes = readable_discussion_notes(noteable) discussions = Kaminari.paginate_array(Discussion.build_collection(notes, noteable)) present paginate(discussions), with: Entities::Discussion end - # rubocop: enable CodeReuse/ActiveRecord desc "Get a single #{noteable_type.to_s.downcase} discussion" do success Entities::Discussion @@ -47,7 +42,7 @@ module API requires :noteable_id, types: [Integer, String], desc: 'The ID of the noteable' end get ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id" do - noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) + noteable = find_noteable(parent_type, params[:id], noteable_type, params[:noteable_id]) notes = readable_discussion_notes(noteable, params[:discussion_id]) if notes.empty? @@ -82,7 +77,7 @@ module API end end post ":id/#{noteables_path}/:noteable_id/discussions" do - noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) + noteable = find_noteable(parent_type, params[:id], noteable_type, params[:noteable_id]) type = params[:position] ? 'DiffNote' : 'DiscussionNote' id_key = noteable.is_a?(Commit) ? :commit_id : :noteable_id @@ -112,7 +107,7 @@ module API requires :noteable_id, types: [Integer, String], desc: 'The ID of the noteable' end get ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id/notes" do - noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) + noteable = find_noteable(parent_type, params[:id], noteable_type, params[:noteable_id]) notes = readable_discussion_notes(noteable, params[:discussion_id]) if notes.empty? @@ -132,7 +127,7 @@ module API optional :created_at, type: String, desc: 'The creation date of the note' end post ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id/notes" do - noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) + noteable = find_noteable(parent_type, params[:id], noteable_type, params[:noteable_id]) notes = readable_discussion_notes(noteable, params[:discussion_id]) first_note = notes.first @@ -166,7 +161,7 @@ module API requires :note_id, type: Integer, desc: 'The ID of a note' end get ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id/notes/:note_id" do - noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) + noteable = find_noteable(parent_type, params[:id], noteable_type, params[:noteable_id]) get_note(noteable, params[:note_id]) end @@ -183,7 +178,7 @@ module API exactly_one_of :body, :resolved end put ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id/notes/:note_id" do - noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) + noteable = find_noteable(parent_type, params[:id], noteable_type, params[:noteable_id]) if params[:resolved].nil? update_note(noteable, params[:note_id]) @@ -201,7 +196,7 @@ module API requires :note_id, type: Integer, desc: 'The ID of a note' end delete ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id/notes/:note_id" do - noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) + noteable = find_noteable(parent_type, params[:id], noteable_type, params[:noteable_id]) delete_note(noteable, params[:note_id]) end @@ -216,7 +211,7 @@ module API requires :resolved, type: Boolean, desc: 'Mark discussion resolved/unresolved' end put ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id" do - noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) + noteable = find_noteable(parent_type, params[:id], noteable_type, params[:noteable_id]) resolve_discussion(noteable, params[:discussion_id], params[:resolved]) end @@ -226,13 +221,24 @@ module API helpers do # rubocop: disable CodeReuse/ActiveRecord - def readable_discussion_notes(noteable, discussion_id) + def readable_discussion_notes(noteable, discussion_id = nil) notes = noteable.notes - .where(discussion_id: discussion_id) + notes = notes.where(discussion_id: discussion_id) if discussion_id + notes = notes .inc_relations_for_view .includes(:noteable) .fresh + # Without RendersActions#prepare_notes_for_rendering, + # Note#cross_reference_not_visible_for? will attempt to render + # Markdown references mentioned in the note to see whether they + # should be redacted. For notes that reference a commit, this + # would also incur a Gitaly call to verify the commit exists. + # + # With prepare_notes_for_rendering, we can avoid Gitaly calls + # because notes are redacted if they point to projects that + # cannot be accessed by the user. + notes = prepare_notes_for_rendering(notes) notes.reject { |n| n.cross_reference_not_visible_for?(current_user) } end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 8840accf675..5e66b4e76a5 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -2,6 +2,19 @@ module API module Entities + class BlameRangeCommit < Grape::Entity + expose :id + expose :parent_ids + expose :message + expose :authored_date, :author_name, :author_email + expose :committed_date, :committer_name, :committer_email + end + + class BlameRange < Grape::Entity + expose :commit, using: BlameRangeCommit + expose :lines + end + class WikiPageBasic < Grape::Entity expose :format expose :slug @@ -64,6 +77,11 @@ module API expose :last_activity_on, as: :last_activity_at # Back-compat end + class UserStarsProject < Grape::Entity + expose :starred_since + expose :user, using: Entities::UserBasic + end + class Identity < Grape::Entity expose :provider, :extern_uid end @@ -201,6 +219,7 @@ module API # MR describing the solution: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/20555 projects_relation.preload(:project_feature, :route) .preload(:import_state, :tags) + .preload(:auto_devops) .preload(namespace: [:route, :owner]) end # rubocop: enable CodeReuse/ActiveRecord @@ -247,12 +266,20 @@ module API expose :container_registry_enabled # Expose old field names with the new permissions methods to keep API compatible + # TODO: remove in API v5, replaced by *_access_level expose(:issues_enabled) { |project, options| project.feature_available?(:issues, options[:current_user]) } expose(:merge_requests_enabled) { |project, options| project.feature_available?(:merge_requests, options[:current_user]) } expose(:wiki_enabled) { |project, options| project.feature_available?(:wiki, options[:current_user]) } expose(:jobs_enabled) { |project, options| project.feature_available?(:builds, options[:current_user]) } expose(:snippets_enabled) { |project, options| project.feature_available?(:snippets, options[:current_user]) } + expose(:issues_access_level) { |project, options| project.project_feature.string_access_level(:issues) } + expose(:repository_access_level) { |project, options| project.project_feature.string_access_level(:repository) } + expose(:merge_requests_access_level) { |project, options| project.project_feature.string_access_level(:merge_requests) } + expose(:wiki_access_level) { |project, options| project.project_feature.string_access_level(:wiki) } + expose(:builds_access_level) { |project, options| project.project_feature.string_access_level(:builds) } + expose(:snippets_access_level) { |project, options| project.project_feature.string_access_level(:snippets) } + expose :shared_runners_enabled expose :lfs_enabled?, as: :lfs_enabled expose :creator_id @@ -267,6 +294,12 @@ module API expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } expose :ci_default_git_depth expose :public_builds, as: :public_jobs + expose :build_git_strategy, if: lambda { |project, options| options[:user_can_admin_project] } do |project, options| + project.build_allow_git_fetch ? 'fetch' : 'clone' + end + expose :build_timeout + expose :auto_cancel_pending_pipelines + expose :build_coverage_regex expose :ci_config_path, if: -> (project, options) { Ability.allowed?(options[:current_user], :download_code, project) } expose :shared_with_groups do |project, options| SharedGroup.represent(project.project_group_links, options) @@ -279,7 +312,10 @@ module API expose :statistics, using: 'API::Entities::ProjectStatistics', if: -> (project, options) { options[:statistics] && Ability.allowed?(options[:current_user], :read_statistics, project) } - expose :external_authorization_classification_label + expose :auto_devops_enabled?, as: :auto_devops_enabled + expose :auto_devops_deploy_strategy do |project, options| + project.auto_devops.nil? ? 'continuous' : project.auto_devops.deploy_strategy + end # rubocop: disable CodeReuse/ActiveRecord def self.preload_relation(projects_relation, options = {}) @@ -289,6 +325,7 @@ module API # MR describing the solution: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/20555 super(projects_relation).preload(:group) .preload(:ci_cd_settings) + .preload(:auto_devops) .preload(project_group_links: { group: :route }, fork_network: :root_project, fork_network_member: :forked_from_project, @@ -347,10 +384,7 @@ module API end expose :request_access_enabled expose :full_name, :full_path - - if ::Group.supports_nested_objects? - expose :parent_id - end + expose :parent_id expose :custom_attributes, using: 'API::Entities::CustomAttribute', if: :with_custom_attributes @@ -491,16 +525,16 @@ module API end end - class ProjectEntity < Grape::Entity + class IssuableEntity < Grape::Entity expose :id, :iid expose(:project_id) { |entity| entity&.project.try(:id) } expose :title, :description expose :state, :created_at, :updated_at # Avoids an N+1 query when metadata is included - def issuable_metadata(subject, options, method) + def issuable_metadata(subject, options, method, args = nil) cached_subject = options.dig(:issuable_metadata, subject.id) - (cached_subject || subject).public_send(method) # rubocop: disable GitlabSecurity/PublicSend + (cached_subject || subject).public_send(method, *args) # rubocop: disable GitlabSecurity/PublicSend end end @@ -544,7 +578,7 @@ module API end end - class IssueBasic < ProjectEntity + class IssueBasic < IssuableEntity expose :closed_at expose :closed_by, using: Entities::UserBasic @@ -564,7 +598,7 @@ module API end expose(:user_notes_count) { |issue, options| issuable_metadata(issue, options, :user_notes_count) } - expose(:merge_requests_count) { |issue, options| issuable_metadata(issue, options, :merge_requests_count) } + expose(:merge_requests_count) { |issue, options| issuable_metadata(issue, options, :merge_requests_count, options[:current_user]) } expose(:upvotes) { |issue, options| issuable_metadata(issue, options, :upvotes) } expose(:downvotes) { |issue, options| issuable_metadata(issue, options, :downvotes) } expose :due_date @@ -611,7 +645,10 @@ module API end end - expose :subscribed do |issue, options| + # Calculating the value of subscribed field triggers Markdown + # processing. We can't do that for multiple issues / merge + # requests in a single API request. + expose :subscribed, if: -> (_, options) { options.fetch(:include_subscribed, true) } do |issue, options| issue.subscribed?(options[:current_user], options[:project] || issue.project) end end @@ -650,14 +687,14 @@ module API end end - class MergeRequestSimple < ProjectEntity + class MergeRequestSimple < IssuableEntity expose :title expose :web_url do |merge_request, options| Gitlab::UrlBuilder.build(merge_request) end end - class MergeRequestBasic < ProjectEntity + class MergeRequestBasic < IssuableEntity expose :merged_by, using: Entities::UserBasic do |merge_request, _options| merge_request.metrics&.merged_by end @@ -703,7 +740,7 @@ module API # See https://gitlab.com/gitlab-org/gitlab-ce/issues/42344 for more # information. expose :merge_status do |merge_request| - merge_request.check_if_can_be_merged + merge_request.check_mergeability merge_request.merge_status end expose :diff_head_sha, as: :sha @@ -757,7 +794,9 @@ module API merge_request.metrics&.pipeline end - expose :head_pipeline, using: 'API::Entities::Pipeline' + expose :head_pipeline, using: 'API::Entities::Pipeline', if: -> (_, options) do + Ability.allowed?(options[:current_user], :read_pipeline, options[:project]) + end expose :diff_refs, using: Entities::DiffRefs @@ -997,7 +1036,7 @@ module API class ProjectService < Grape::Entity expose :id, :title, :created_at, :updated_at, :active - expose :push_events, :issues_events, :confidential_issues_events + expose :commit_events, :push_events, :issues_events, :confidential_issues_events expose :merge_requests_events, :tag_push_events, :note_events expose :confidential_note_events, :pipeline_events, :wiki_page_events expose :job_events @@ -1031,15 +1070,8 @@ module API # rubocop: disable CodeReuse/ActiveRecord def self.preload_relation(projects_relation, options = {}) relation = super(projects_relation, options) - - # MySQL doesn't support LIMIT inside an IN subquery - if Gitlab::Database.mysql? - project_ids = relation.pluck('projects.id') - namespace_ids = relation.pluck(:namespace_id) - else - project_ids = relation.select('projects.id') - namespace_ids = relation.select(:namespace_id) - end + project_ids = relation.select('projects.id') + namespace_ids = relation.select(:namespace_id) options[:project_members] = options[:current_user] .project_members @@ -1061,16 +1093,18 @@ module API end class Label < LabelBasic - expose :open_issues_count do |label, options| - label.open_issues_count(options[:current_user]) - end + with_options if: lambda { |_, options| options[:with_counts] } do + expose :open_issues_count do |label, options| + label.open_issues_count(options[:current_user]) + end - expose :closed_issues_count do |label, options| - label.closed_issues_count(options[:current_user]) - end + expose :closed_issues_count do |label, options| + label.closed_issues_count(options[:current_user]) + end - expose :open_merge_requests_count do |label, options| - label.open_merge_requests_count(options[:current_user]) + expose :open_merge_requests_count do |label, options| + label.open_merge_requests_count(options[:current_user]) + end end expose :subscribed do |label, options| @@ -1101,7 +1135,7 @@ module API expose :project, using: Entities::BasicProjectDetails expose :lists, using: Entities::List do |board| - board.lists.destroyable + board.destroyable_lists end end @@ -1138,6 +1172,7 @@ module API attributes = ::ApplicationSettingsHelper.visible_attributes attributes.delete(:performance_bar_allowed_group_path) attributes.delete(:performance_bar_enabled) + attributes.delete(:allow_local_requests_from_hooks_and_services) attributes end @@ -1156,6 +1191,7 @@ module API # support legacy names, can be removed in v5 expose :password_authentication_enabled_for_web, as: :password_authentication_enabled expose :password_authentication_enabled_for_web, as: :signin_enabled + expose :allow_local_requests_from_web_hooks_and_services, as: :allow_local_requests_from_hooks_and_services end # deprecated old Release representation @@ -1186,8 +1222,10 @@ module API MarkupHelper.markdown_field(entity, :description) end expose :created_at + expose :released_at expose :author, using: Entities::UserBasic, if: -> (release, _) { release.author.present? } expose :commit, using: Entities::Commit, if: lambda { |_, _| can_download_code? } + expose :upcoming_release?, as: :upcoming_release expose :assets do expose :assets_count, as: :count do |release, _| @@ -1313,6 +1351,7 @@ module API expose :variable_type, :key, :value expose :protected?, as: :protected, if: -> (entity, _) { entity.respond_to?(:protected?) } expose :masked?, as: :masked, if: -> (entity, _) { entity.respond_to?(:masked?) } + expose :environment_scope, if: -> (entity, _) { entity.respond_to?(:environment_scope) } end class Pipeline < PipelineBasic @@ -1662,5 +1701,9 @@ module API class ClusterProject < Cluster expose :project, using: Entities::BasicProjectDetails end + + class ClusterGroup < Cluster + expose :group, using: Entities::BasicGroupDetails + end end end diff --git a/lib/api/entities/container_registry.rb b/lib/api/entities/container_registry.rb index 00833ca7480..6250f35c7cb 100644 --- a/lib/api/entities/container_registry.rb +++ b/lib/api/entities/container_registry.rb @@ -3,18 +3,20 @@ module API module Entities module ContainerRegistry - class Repository < Grape::Entity - expose :id + class Tag < Grape::Entity expose :name expose :path expose :location - expose :created_at end - class Tag < Grape::Entity + class Repository < Grape::Entity + expose :id expose :name expose :path + expose :project_id expose :location + expose :created_at + expose :tags, using: Tag, if: -> (_, options) { options[:tags] } end class TagDetails < Tag diff --git a/lib/api/environments.rb b/lib/api/environments.rb index 6cd43923559..ec58b3b7bb9 100644 --- a/lib/api/environments.rb +++ b/lib/api/environments.rb @@ -18,11 +18,16 @@ module API end params do use :pagination + optional :name, type: String, desc: 'Returns the environment with this name' + optional :search, type: String, desc: 'Returns list of environments matching the search criteria' + mutually_exclusive :name, :search, message: 'cannot be used together' end get ':id/environments' do authorize! :read_environment, user_project - present paginate(user_project.environments), with: Entities::Environment, current_user: current_user + environments = ::EnvironmentsFinder.new(user_project, current_user, params).find + + present paginate(environments), with: Entities::Environment, current_user: current_user end desc 'Creates a new environment' do diff --git a/lib/api/files.rb b/lib/api/files.rb index ca59d330e1c..0b438fb5bbc 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -83,6 +83,31 @@ module API resource :projects, requirements: FILE_ENDPOINT_REQUIREMENTS do allow_access_with_scope :read_repository, if: -> (request) { request.get? || request.head? } + desc 'Get blame file metadata from repository' + params do + requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' + requires :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false + end + head ":id/repository/files/:file_path/blame", requirements: FILE_ENDPOINT_REQUIREMENTS do + assign_file_vars! + + set_http_headers(blob_data) + end + + desc 'Get blame file from the repository' + params do + requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' + requires :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false + end + get ":id/repository/files/:file_path/blame", requirements: FILE_ENDPOINT_REQUIREMENTS do + assign_file_vars! + + set_http_headers(blob_data) + + blame_ranges = Gitlab::Blame.new(@blob, @commit).groups(highlight: false) + present blame_ranges, with: Entities::BlameRange + end + desc 'Get raw file metadata from repository' params do requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' diff --git a/lib/api/group_boards.rb b/lib/api/group_boards.rb index 9a20ee8c8b9..feb2254963e 100644 --- a/lib/api/group_boards.rb +++ b/lib/api/group_boards.rb @@ -37,7 +37,7 @@ module API use :pagination end get '/' do - present paginate(board_parent.boards), with: Entities::Board + present paginate(board_parent.boards.with_associations), with: Entities::Board end end diff --git a/lib/api/group_clusters.rb b/lib/api/group_clusters.rb new file mode 100644 index 00000000000..db0f8081140 --- /dev/null +++ b/lib/api/group_clusters.rb @@ -0,0 +1,140 @@ +# frozen_string_literal: true + +module API + class GroupClusters < Grape::API + include PaginationParams + + before { authenticate! } + + # EE::API::GroupClusters will + # override these methods + helpers do + params :create_params_ee do + end + + params :update_params_ee do + end + end + + params do + requires :id, type: String, desc: 'The ID of the group' + end + resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + desc 'Get all clusters from the group' do + success Entities::Cluster + end + params do + use :pagination + end + get ':id/clusters' do + authorize! :read_cluster, user_group + + present paginate(clusters_for_current_user), with: Entities::Cluster + end + + desc 'Get specific cluster for the group' do + success Entities::ClusterGroup + end + params do + requires :cluster_id, type: Integer, desc: 'The cluster ID' + end + get ':id/clusters/:cluster_id' do + authorize! :read_cluster, cluster + + present cluster, with: Entities::ClusterGroup + end + + desc 'Adds an existing cluster' do + success Entities::ClusterGroup + end + params do + requires :name, type: String, desc: 'Cluster name' + optional :enabled, type: Boolean, default: true, desc: 'Determines if cluster is active or not, defaults to true' + optional :domain, type: String, desc: 'Cluster base domain' + optional :managed, type: Boolean, default: true, desc: 'Determines if GitLab will manage namespaces and service accounts for this cluster, defaults to true' + requires :platform_kubernetes_attributes, type: Hash, desc: %q(Platform Kubernetes data) do + requires :api_url, type: String, allow_blank: false, desc: 'URL to access the Kubernetes API' + requires :token, type: String, desc: 'Token to authenticate against Kubernetes' + optional :ca_cert, type: String, desc: 'TLS certificate (needed if API is using a self-signed TLS certificate)' + optional :namespace, type: String, desc: 'Unique namespace related to Group' + optional :authorization_type, type: String, values: Clusters::Platforms::Kubernetes.authorization_types.keys, default: 'rbac', desc: 'Cluster authorization type, defaults to RBAC' + end + use :create_params_ee + end + post ':id/clusters/user' do + authorize! :add_cluster, user_group + + user_cluster = ::Clusters::CreateService + .new(current_user, create_cluster_user_params) + .execute + + if user_cluster.persisted? + present user_cluster, with: Entities::ClusterGroup + else + render_validation_error!(user_cluster) + end + end + + desc 'Update an existing cluster' do + success Entities::ClusterGroup + end + params do + requires :cluster_id, type: Integer, desc: 'The cluster ID' + optional :name, type: String, desc: 'Cluster name' + optional :domain, type: String, desc: 'Cluster base domain' + optional :platform_kubernetes_attributes, type: Hash, desc: %q(Platform Kubernetes data) do + optional :api_url, type: String, desc: 'URL to access the Kubernetes API' + optional :token, type: String, desc: 'Token to authenticate against Kubernetes' + optional :ca_cert, type: String, desc: 'TLS certificate (needed if API is using a self-signed TLS certificate)' + optional :namespace, type: String, desc: 'Unique namespace related to Group' + end + use :update_params_ee + end + put ':id/clusters/:cluster_id' do + authorize! :update_cluster, cluster + + update_service = Clusters::UpdateService.new(current_user, update_cluster_params) + + if update_service.execute(cluster) + present cluster, with: Entities::ClusterGroup + else + render_validation_error!(cluster) + end + end + + desc 'Remove a cluster' do + success Entities::ClusterGroup + end + params do + requires :cluster_id, type: Integer, desc: 'The Cluster ID' + end + delete ':id/clusters/:cluster_id' do + authorize! :admin_cluster, cluster + + destroy_conditionally!(cluster) + end + end + + helpers do + def clusters_for_current_user + @clusters_for_current_user ||= ClustersFinder.new(user_group, current_user, :all).execute + end + + def cluster + @cluster ||= clusters_for_current_user.find(params[:cluster_id]) + end + + def create_cluster_user_params + declared_params.merge({ + provider_type: :user, + platform_type: :kubernetes, + clusterable: user_group + }) + end + + def update_cluster_params + declared_params(include_missing: false).without(:cluster_id) + end + end + end +end diff --git a/lib/api/group_container_repositories.rb b/lib/api/group_container_repositories.rb new file mode 100644 index 00000000000..fd24662cc9a --- /dev/null +++ b/lib/api/group_container_repositories.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module API + class GroupContainerRepositories < Grape::API + include PaginationParams + + before { authorize_read_group_container_images! } + + REPOSITORY_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge( + tag_name: API::NO_SLASH_URL_PART_REGEX) + + params do + requires :id, type: String, desc: "Group's ID or path" + end + resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + desc 'Get a list of all repositories within a group' do + detail 'This feature was introduced in GitLab 12.2.' + success Entities::ContainerRegistry::Repository + end + params do + use :pagination + optional :tags, type: Boolean, default: false, desc: 'Determines if tags should be included' + end + get ':id/registry/repositories' do + repositories = ContainerRepositoriesFinder.new( + id: user_group.id, container_type: :group + ).execute + + present paginate(repositories), with: Entities::ContainerRegistry::Repository, tags: params[:tags] + end + end + + helpers do + def authorize_read_group_container_images! + authorize! :read_container_image, user_group + end + end + end +end diff --git a/lib/api/group_labels.rb b/lib/api/group_labels.rb index 0dbc5f45a68..79a44941c81 100644 --- a/lib/api/group_labels.rb +++ b/lib/api/group_labels.rb @@ -16,6 +16,8 @@ module API success Entities::GroupLabel end params do + optional :with_counts, type: Boolean, default: false, + desc: 'Include issue and merge request counts' use :pagination end get ':id/labels' do diff --git a/lib/api/groups.rb b/lib/api/groups.rb index ec1020c7c78..f545f33c06b 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -114,10 +114,7 @@ module API params do requires :name, type: String, desc: 'The name of the group' requires :path, type: String, desc: 'The path of the group' - - if ::Group.supports_nested_objects? - optional :parent_id, type: Integer, desc: 'The parent group id for creating nested group' - end + optional :parent_id, type: Integer, desc: 'The parent group id for creating nested group' use :optional_params end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 00bcf6b055b..1aa6dc44bf7 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -183,11 +183,6 @@ module API user_project.commit_by(oid: id) end - def find_project_snippet(id) - finder_params = { project: user_project } - SnippetsFinder.new(current_user, finder_params).find(id) - end - # rubocop: disable CodeReuse/ActiveRecord def find_merge_request_with_access(iid, access_level = :read_merge_request) merge_request = user_project.merge_requests.find_by!(iid: iid) @@ -235,6 +230,10 @@ module API authorize! :push_code, user_project end + def authorize_admin_tag + authorize! :admin_tag, user_project + end + def authorize_admin_project authorize! :admin_project, user_project end @@ -251,6 +250,10 @@ module API authorize! :update_build, user_project end + def require_repository_enabled!(subject = :global) + not_found!("Repository") unless user_project.feature_available?(:repository, current_user) + end + def require_gitlab_workhorse! unless env['HTTP_GITLAB_WORKHORSE'].present? forbidden!('Request should be executed via GitLab Workhorse') @@ -541,5 +544,9 @@ module API params[:archived] end + + def ip_address + env["action_dispatch.remote_ip"].to_s || request.ip + end end end diff --git a/lib/api/helpers/graphql_helpers.rb b/lib/api/helpers/graphql_helpers.rb index 94010ab1bc2..bd60470fbd6 100644 --- a/lib/api/helpers/graphql_helpers.rb +++ b/lib/api/helpers/graphql_helpers.rb @@ -7,8 +7,6 @@ module API # should be in app/graphql/ or lib/gitlab/graphql/ module GraphqlHelpers def conditionally_graphql!(fallback:, query:, context: {}, transform: nil) - return fallback.call unless Feature.enabled?(:graphql) - result = GitlabSchema.execute(query, context: context) if transform diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index c318f5b9127..9afe6c5b027 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -72,7 +72,7 @@ module API result == 'PONG' rescue => e - Rails.logger.warn("GitLab: An unexpected error occurred in pinging to Redis: #{e}") + Rails.logger.warn("GitLab: An unexpected error occurred in pinging to Redis: #{e}") # rubocop:disable Gitlab/RailsLogger false end diff --git a/lib/api/helpers/label_helpers.rb b/lib/api/helpers/label_helpers.rb index c11e7d614ab..896b0aba52b 100644 --- a/lib/api/helpers/label_helpers.rb +++ b/lib/api/helpers/label_helpers.rb @@ -19,7 +19,11 @@ module API end def get_labels(parent, entity) - present paginate(available_labels_for(parent)), with: entity, current_user: current_user, parent: parent + present paginate(available_labels_for(parent)), + with: entity, + current_user: current_user, + parent: parent, + with_counts: params[:with_counts] end def create_label(parent, entity) diff --git a/lib/api/helpers/notes_helpers.rb b/lib/api/helpers/notes_helpers.rb index a068de4361c..6bf9057fad7 100644 --- a/lib/api/helpers/notes_helpers.rb +++ b/lib/api/helpers/notes_helpers.rb @@ -73,14 +73,29 @@ module API "read_#{noteable.class.to_s.underscore}".to_sym end - def find_noteable(parent, noteables_str, noteable_id) - noteable = public_send("find_#{parent}_#{noteables_str.singularize}", noteable_id) # rubocop:disable GitlabSecurity/PublicSend + def find_noteable(parent_type, parent_id, noteable_type, noteable_id) + params = finder_params_by_noteable_type_and_id(noteable_type, noteable_id, parent_id) - readable = can?(current_user, noteable_read_ability_name(noteable), noteable) + noteable = NotesFinder.new(current_user, params).target + noteable = nil unless can?(current_user, noteable_read_ability_name(noteable), noteable) + noteable || not_found!(noteable_type) + end + + def finder_params_by_noteable_type_and_id(type, id, parent_id) + target_type = type.name.underscore + { target_type: target_type }.tap do |h| + if %w(issue merge_request).include?(target_type) + h[:target_iid] = id + else + h[:target_id] = id + end - return not_found!(noteables_str) unless readable + add_parent_to_finder_params(h, type, parent_id) + end + end - noteable + def add_parent_to_finder_params(finder_params, noteable_type, parent_id) + finder_params[:project] = user_project end def noteable_parent(noteable) diff --git a/lib/api/helpers/pagination.rb b/lib/api/helpers/pagination.rb index 2a9b17ad22a..71bbc218f94 100644 --- a/lib/api/helpers/pagination.rb +++ b/lib/api/helpers/pagination.rb @@ -205,7 +205,9 @@ module API limited_total_count = pagination_data.total_count_with_limit if limited_total_count > Kaminari::ActiveRecordRelationMethods::MAX_COUNT_LIMIT - pagination_data.without_count + # The call to `total_count_with_limit` memoizes `@arel` because of a call to `references_eager_loaded_tables?` + # We need to call `reset` because `without_count` relies on `@arel` being unmemoized + pagination_data.reset.without_count else pagination_data end diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb index f242f1fea0e..51b7cf05c8f 100644 --- a/lib/api/helpers/projects_helpers.rb +++ b/lib/api/helpers/projects_helpers.rb @@ -8,12 +8,26 @@ module API params :optional_project_params_ce do optional :description, type: String, desc: 'The description of the project' + optional :build_git_strategy, type: String, values: %w(fetch clone), desc: 'The Git strategy. Defaults to `fetch`' + optional :build_timeout, type: Integer, desc: 'Build timeout' + optional :auto_cancel_pending_pipelines, type: String, values: %w(disabled enabled), desc: 'Auto-cancel pending pipelines' + optional :build_coverage_regex, type: String, desc: 'Test coverage parsing' optional :ci_config_path, type: String, desc: 'The path to CI config file. Defaults to `.gitlab-ci.yml`' + + # TODO: remove in API v5, replaced by *_access_level optional :issues_enabled, type: Boolean, desc: 'Flag indication if the issue tracker is enabled' optional :merge_requests_enabled, type: Boolean, desc: 'Flag indication if merge requests are enabled' optional :wiki_enabled, type: Boolean, desc: 'Flag indication if the wiki is enabled' optional :jobs_enabled, type: Boolean, desc: 'Flag indication if jobs are enabled' optional :snippets_enabled, type: Boolean, desc: 'Flag indication if snippets are enabled' + + optional :issues_access_level, type: String, values: %w(disabled private enabled), desc: 'Issues access level. One of `disabled`, `private` or `enabled`' + optional :repository_access_level, type: String, values: %w(disabled private enabled), desc: 'Repository access level. One of `disabled`, `private` or `enabled`' + optional :merge_requests_access_level, type: String, values: %w(disabled private enabled), desc: 'Merge requests access level. One of `disabled`, `private` or `enabled`' + optional :wiki_access_level, type: String, values: %w(disabled private enabled), desc: 'Wiki access level. One of `disabled`, `private` or `enabled`' + optional :builds_access_level, type: String, values: %w(disabled private enabled), desc: 'Builds access level. One of `disabled`, `private` or `enabled`' + optional :snippets_access_level, type: String, values: %w(disabled private enabled), desc: 'Snippets access level. One of `disabled`, `private` or `enabled`' + optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project' optional :resolve_outdated_diff_discussions, type: Boolean, desc: 'Automatically resolve merge request diffs discussions on lines changed with a push' optional :container_registry_enabled, type: Boolean, desc: 'Flag indication if the container registry is enabled for that project' @@ -28,8 +42,9 @@ module API optional :printing_merge_request_link_enabled, type: Boolean, desc: 'Show link to create/view merge request when pushing from the command line' optional :merge_method, type: String, values: %w(ff rebase_merge merge), desc: 'The merge method used when merging merge requests' optional :initialize_with_readme, type: Boolean, desc: "Initialize a project with a README.md" - optional :external_authorization_classification_label, type: String, desc: 'The classification label for the project' optional :ci_default_git_depth, type: Integer, desc: 'Default number of revisions for shallow cloning' + optional :auto_devops_enabled, type: Boolean, desc: 'Flag indication if Auto DevOps is enabled' + optional :auto_devops_deploy_strategy, type: String, values: %w(continuous manual timed_incremental), desc: 'Auto Deploy strategy' end params :optional_project_params_ee do @@ -48,15 +63,21 @@ module API def self.update_params_at_least_one_of [ - :jobs_enabled, - :resolve_outdated_diff_discussions, + :auto_devops_enabled, + :auto_devops_deploy_strategy, + :auto_cancel_pending_pipelines, + :build_coverage_regex, + :build_git_strategy, + :build_timeout, + :builds_access_level, :ci_config_path, + :ci_default_git_depth, :container_registry_enabled, :default_branch, :description, - :issues_enabled, + :issues_access_level, :lfs_enabled, - :merge_requests_enabled, + :merge_requests_access_level, :merge_method, :name, :only_allow_merge_if_all_discussions_are_resolved, @@ -64,16 +85,28 @@ module API :path, :printing_merge_request_link_enabled, :public_builds, + :repository_access_level, :request_access_enabled, + :resolve_outdated_diff_discussions, :shared_runners_enabled, - :snippets_enabled, + :snippets_access_level, :tag_list, :visibility, - :wiki_enabled, + :wiki_access_level, :avatar, - :external_authorization_classification_label + + # TODO: remove in API v5, replaced by *_access_level + :issues_enabled, + :jobs_enabled, + :merge_requests_enabled, + :wiki_enabled, + :jobs_enabled, + :snippets_enabled ] end + + def filter_attributes_using_license!(attrs) + end end end end diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb index ff73a49d5e8..5b87eccf860 100644 --- a/lib/api/helpers/runner.rb +++ b/lib/api/helpers/runner.rb @@ -7,8 +7,7 @@ module API JOB_TOKEN_PARAM = :token def runner_registration_token_valid? - ActiveSupport::SecurityUtils.variable_size_secure_compare(params[:token], - Gitlab::CurrentSettings.runners_registration_token) + ActiveSupport::SecurityUtils.secure_compare(params[:token], Gitlab::CurrentSettings.runners_registration_token) end def authenticate_runner! @@ -26,7 +25,7 @@ module API end def get_runner_ip - { ip_address: env["action_dispatch.remote_ip"].to_s || request.ip } + { ip_address: ip_address } end def current_runner diff --git a/lib/api/helpers/services_helpers.rb b/lib/api/helpers/services_helpers.rb index 44c577204b8..422db5c7a50 100644 --- a/lib/api/helpers/services_helpers.rb +++ b/lib/api/helpers/services_helpers.rb @@ -462,57 +462,31 @@ module API required: true, name: :url, type: String, - desc: 'The base URL to the JIRA instance web interface which is being linked to this GitLab project. E.g., https://jira.example.com' + desc: 'The base URL to the Jira instance web interface which is being linked to this GitLab project. E.g., https://jira.example.com' }, { required: false, name: :api_url, type: String, - desc: 'The base URL to the JIRA instance API. Web URL value will be used if not set. E.g., https://jira-api.example.com' + desc: 'The base URL to the Jira instance API. Web URL value will be used if not set. E.g., https://jira-api.example.com' }, { required: true, name: :username, type: String, - desc: 'The username of the user created to be used with GitLab/JIRA' + desc: 'The username of the user created to be used with GitLab/Jira' }, { required: true, name: :password, type: String, - desc: 'The password of the user created to be used with GitLab/JIRA' + desc: 'The password of the user created to be used with GitLab/Jira' }, { required: false, name: :jira_issue_transition_id, type: String, - desc: 'The ID of a transition that moves issues to a closed state. You can find this number under the JIRA workflow administration (**Administration > Issues > Workflows**) by selecting **View** under **Operations** of the desired workflow of your project. The ID of each state can be found inside the parenthesis of each transition name under the **Transitions (id)** column ([see screenshot][trans]). By default, this ID is set to `2`' - } - ], - 'kubernetes' => [ - { - required: true, - name: :namespace, - type: String, - desc: 'The Kubernetes namespace to use' - }, - { - required: true, - name: :api_url, - type: String, - desc: 'The URL to the Kubernetes cluster API, e.g., https://kubernetes.example.com' - }, - { - required: true, - name: :token, - type: String, - desc: 'The service token to authenticate against the Kubernetes cluster with' - }, - { - required: false, - name: :ca_pem, - type: String, - desc: 'A custom certificate authority bundle to verify the Kubernetes cluster with (PEM format)' + desc: 'The ID of a transition that moves issues to a closed state. You can find this number under the Jira workflow administration (**Administration > Issues > Workflows**) by selecting **View** under **Operations** of the desired workflow of your project. The ID of each state can be found inside the parenthesis of each transition name under the **Transitions (id)** column ([see screenshot][trans]). By default, this ID is set to `2`' } ], 'mattermost-slash-commands' => [ @@ -683,8 +657,9 @@ module API name: :webhook, type: String, desc: 'The Microsoft Teams webhook. e.g. https://outlook.office.com/webhook/…' - } - ], + }, + chat_notification_flags + ].flatten, 'mattermost' => [ chat_notification_settings, chat_notification_flags, @@ -738,7 +713,6 @@ module API ::HipchatService, ::IrkerService, ::JiraService, - ::KubernetesService, ::MattermostSlashCommandsService, ::SlackSlashCommandsService, ::PackagistService, diff --git a/lib/api/helpers/variables_helpers.rb b/lib/api/helpers/variables_helpers.rb deleted file mode 100644 index 78a92d0f5a6..00000000000 --- a/lib/api/helpers/variables_helpers.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module API - module Helpers - module VariablesHelpers - extend ActiveSupport::Concern - extend Grape::API::Helpers - - params :optional_params_ee do - end - end - end -end diff --git a/lib/api/import_github.rb b/lib/api/import_github.rb index e7504051808..21d4928193e 100644 --- a/lib/api/import_github.rb +++ b/lib/api/import_github.rb @@ -28,7 +28,7 @@ module API desc 'Import a GitHub project' do detail 'This feature was introduced in GitLab 11.3.4.' - success Entities::ProjectEntity + success ::ProjectEntity end params do requires :personal_access_token, type: String, desc: 'GitHub personal access token' diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 039ebf92187..7819c2de515 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -96,7 +96,8 @@ module API with: Entities::Issue, with_labels_details: declared_params[:with_labels_details], current_user: current_user, - issuable_metadata: issuable_meta_data(issues, 'Issue') + issuable_metadata: issuable_meta_data(issues, 'Issue', current_user), + include_subscribed: false } present issues, options @@ -122,7 +123,8 @@ module API with: Entities::Issue, with_labels_details: declared_params[:with_labels_details], current_user: current_user, - issuable_metadata: issuable_meta_data(issues, 'Issue') + issuable_metadata: issuable_meta_data(issues, 'Issue', current_user), + include_subscribed: false } present issues, options @@ -161,7 +163,7 @@ module API with_labels_details: declared_params[:with_labels_details], current_user: current_user, project: user_project, - issuable_metadata: issuable_meta_data(issues, 'Issue') + issuable_metadata: issuable_meta_data(issues, 'Issue', current_user) } present issues, options diff --git a/lib/api/job_artifacts.rb b/lib/api/job_artifacts.rb index e7fed55170e..b35aa952f81 100644 --- a/lib/api/job_artifacts.rb +++ b/lib/api/job_artifacts.rb @@ -27,7 +27,7 @@ module API requirements: { ref_name: /.+/ } do authorize_download_artifacts! - latest_build = user_project.latest_successful_build_for!(params[:job], params[:ref_name]) + latest_build = user_project.latest_successful_build_for_ref!(params[:job], params[:ref_name]) present_carrierwave_file!(latest_build.artifacts_file) end @@ -45,7 +45,7 @@ module API requirements: { ref_name: /.+/ } do authorize_download_artifacts! - build = user_project.latest_successful_build_for!(params[:job], params[:ref_name]) + build = user_project.latest_successful_build_for_ref!(params[:job], params[:ref_name]) path = Gitlab::Ci::Build::Artifacts::Path .new(params[:artifact_path]) diff --git a/lib/api/labels.rb b/lib/api/labels.rb index d729d3ee625..c183198d3c6 100644 --- a/lib/api/labels.rb +++ b/lib/api/labels.rb @@ -15,6 +15,8 @@ module API success Entities::ProjectLabel end params do + optional :with_counts, type: Boolean, default: false, + desc: 'Include issue and merge request counts' use :pagination end get ':id/labels' do diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 955624404f1..64ee82cd775 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -72,7 +72,7 @@ module API if params[:view] == 'simple' options[:with] = Entities::MergeRequestSimple else - options[:issuable_metadata] = issuable_meta_data(merge_requests, 'MergeRequest') + options[:issuable_metadata] = issuable_meta_data(merge_requests, 'MergeRequest', current_user) end options @@ -397,28 +397,16 @@ module API present merge_request, with: Entities::MergeRequest, current_user: current_user, project: user_project end - desc 'Merge a merge request to its default temporary merge ref path' - params do - optional :merge_commit_message, type: String, desc: 'Custom merge commit message' - end - put ':id/merge_requests/:merge_request_iid/merge_to_ref' do + desc 'Returns the up to date merge-ref HEAD commit' + get ':id/merge_requests/:merge_request_iid/merge_ref' do merge_request = find_project_merge_request(params[:merge_request_iid]) - authorize! :admin_merge_request, user_project - - merge_params = { - commit_message: params[:merge_commit_message] - } - - result = ::MergeRequests::MergeToRefService - .new(merge_request.target_project, current_user, merge_params) - .execute(merge_request) + result = ::MergeRequests::MergeabilityCheckService.new(merge_request).execute(recheck: true) - if result[:status] == :success - present result.slice(:commit_id), 200 + if result.success? + present :commit_id, result.payload.dig(:merge_ref_head, :commit_id) else - http_status = result[:http_status] || 400 - render_api_error!(result[:message], http_status) + render_api_error!(result.message, 400) end end @@ -441,9 +429,10 @@ module API authorize_push_to_merge_request!(merge_request) - RebaseWorker.perform_async(merge_request.id, current_user.id) + merge_request.rebase_async(current_user.id) status :accepted + present rebase_in_progress: merge_request.rebase_in_progress? end desc 'List issues that will be closed on merge' do diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 416cf39d3ec..9381f045144 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -15,8 +15,6 @@ module API requires :id, type: String, desc: "The ID of a #{parent_type}" end resource parent_type.pluralize.to_sym, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - noteables_str = noteable_type.to_s.underscore.pluralize - desc "Get a list of #{noteable_type.to_s.downcase} notes" do success Entities::Note end @@ -30,7 +28,7 @@ module API end # rubocop: disable CodeReuse/ActiveRecord get ":id/#{noteables_str}/:noteable_id/notes" do - noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) + noteable = find_noteable(parent_type, params[:id], noteable_type, params[:noteable_id]) # We exclude notes that are cross-references and that cannot be viewed # by the current user. By doing this exclusion at this level and not @@ -56,7 +54,7 @@ module API requires :noteable_id, type: Integer, desc: 'The ID of the noteable' end get ":id/#{noteables_str}/:noteable_id/notes/:note_id" do - noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) + noteable = find_noteable(parent_type, params[:id], noteable_type, params[:noteable_id]) get_note(noteable, params[:note_id]) end @@ -69,7 +67,7 @@ module API optional :created_at, type: String, desc: 'The creation date of the note' end post ":id/#{noteables_str}/:noteable_id/notes" do - noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) + noteable = find_noteable(parent_type, params[:id], noteable_type, params[:noteable_id]) opts = { note: params[:body], @@ -96,7 +94,7 @@ module API requires :body, type: String, desc: 'The content of a note' end put ":id/#{noteables_str}/:noteable_id/notes/:note_id" do - noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) + noteable = find_noteable(parent_type, params[:id], noteable_type, params[:noteable_id]) update_note(noteable, params[:note_id]) end @@ -109,7 +107,7 @@ module API requires :note_id, type: Integer, desc: 'The ID of a note' end delete ":id/#{noteables_str}/:noteable_id/notes/:note_id" do - noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) + noteable = find_noteable(parent_type, params[:id], noteable_type, params[:noteable_id]) delete_note(noteable, params[:note_id]) end diff --git a/lib/api/pages_domains.rb b/lib/api/pages_domains.rb index 78442f465bd..4227a106a95 100644 --- a/lib/api/pages_domains.rb +++ b/lib/api/pages_domains.rb @@ -90,14 +90,15 @@ module API end params do requires :domain, type: String, desc: 'The domain' - optional :certificate, allow_blank: false, types: [File, String], desc: 'The certificate' - optional :key, allow_blank: false, types: [File, String], desc: 'The key' - all_or_none_of :certificate, :key + optional :certificate, allow_blank: false, types: [File, String], desc: 'The certificate', as: :user_provided_certificate + optional :key, allow_blank: false, types: [File, String], desc: 'The key', as: :user_provided_key + all_or_none_of :user_provided_certificate, :user_provided_key end post ":id/pages/domains" do authorize! :update_pages, user_project pages_domain_params = declared(params, include_parent_namespaces: false) + pages_domain = user_project.pages_domains.create(pages_domain_params) if pages_domain.persisted? @@ -110,8 +111,8 @@ module API desc 'Updates a pages domain' params do requires :domain, type: String, desc: 'The domain' - optional :certificate, allow_blank: false, types: [File, String], desc: 'The certificate' - optional :key, allow_blank: false, types: [File, String], desc: 'The key' + optional :certificate, allow_blank: false, types: [File, String], desc: 'The certificate', as: :user_provided_certificate + optional :key, allow_blank: false, types: [File, String], desc: 'The key', as: :user_provided_key end put ":id/pages/domains/:domain", requirements: PAGES_DOMAINS_ENDPOINT_REQUIREMENTS do authorize! :update_pages, user_project @@ -119,8 +120,8 @@ module API pages_domain_params = declared(params, include_parent_namespaces: false) # Remove empty private key if certificate is not empty. - if pages_domain_params[:certificate] && !pages_domain_params[:key] - pages_domain_params.delete(:key) + if pages_domain_params[:user_provided_certificate] && !pages_domain_params[:user_provided_key] + pages_domain_params.delete(:user_provided_key) end if pages_domain.update(pages_domain_params) diff --git a/lib/api/project_clusters.rb b/lib/api/project_clusters.rb index dcc8d94fb79..4f093e9be08 100644 --- a/lib/api/project_clusters.rb +++ b/lib/api/project_clusters.rb @@ -65,7 +65,7 @@ module API use :create_params_ee end post ':id/clusters/user' do - authorize! :add_cluster, user_project, 'Instance does not support multiple Kubernetes clusters' + authorize! :add_cluster, user_project user_cluster = ::Clusters::CreateService .new(current_user, create_cluster_user_params) diff --git a/lib/api/container_registry.rb b/lib/api/project_container_repositories.rb index e4493910196..6d53abcc500 100644 --- a/lib/api/container_registry.rb +++ b/lib/api/project_container_repositories.rb @@ -1,10 +1,10 @@ # frozen_string_literal: true module API - class ContainerRegistry < Grape::API + class ProjectContainerRepositories < Grape::API include PaginationParams - REGISTRY_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge( + REPOSITORY_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge( tag_name: API::NO_SLASH_URL_PART_REGEX) before { error!('404 Not Found', 404) unless Feature.enabled?(:container_registry_api, user_project, default_enabled: true) } @@ -20,11 +20,14 @@ module API end params do use :pagination + optional :tags, type: Boolean, default: false, desc: 'Determines if tags should be included' end get ':id/registry/repositories' do - repositories = user_project.container_repositories.ordered + repositories = ContainerRepositoriesFinder.new( + id: user_project.id, container_type: :project + ).execute - present paginate(repositories), with: Entities::ContainerRegistry::Repository + present paginate(repositories), with: Entities::ContainerRegistry::Repository, tags: params[:tags] end desc 'Delete repository' do @@ -33,7 +36,7 @@ module API params do requires :repository_id, type: Integer, desc: 'The ID of the repository' end - delete ':id/registry/repositories/:repository_id', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do + delete ':id/registry/repositories/:repository_id', requirements: REPOSITORY_ENDPOINT_REQUIREMENTS do authorize_admin_container_image! DeleteContainerRepositoryWorker.perform_async(current_user.id, repository.id) @@ -49,7 +52,7 @@ module API requires :repository_id, type: Integer, desc: 'The ID of the repository' use :pagination end - get ':id/registry/repositories/:repository_id/tags', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do + get ':id/registry/repositories/:repository_id/tags', requirements: REPOSITORY_ENDPOINT_REQUIREMENTS do authorize_read_container_image! tags = Kaminari.paginate_array(repository.tags) @@ -65,9 +68,12 @@ module API optional :keep_n, type: Integer, desc: 'Keep n of latest tags with matching name' optional :older_than, type: String, desc: 'Delete older than: 1h, 1d, 1month' end - delete ':id/registry/repositories/:repository_id/tags', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do + delete ':id/registry/repositories/:repository_id/tags', requirements: REPOSITORY_ENDPOINT_REQUIREMENTS do authorize_admin_container_image! + message = 'This request has already been made. You can run this at most once an hour for a given container repository' + render_api_error!(message, 400) unless obtain_new_cleanup_container_lease + CleanupContainerRepositoryWorker.perform_async(current_user.id, repository.id, declared_params.except(:repository_id)) # rubocop: disable CodeReuse/ActiveRecord @@ -82,7 +88,7 @@ module API requires :repository_id, type: Integer, desc: 'The ID of the repository' requires :tag_name, type: String, desc: 'The name of the tag' end - get ':id/registry/repositories/:repository_id/tags/:tag_name', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do + get ':id/registry/repositories/:repository_id/tags/:tag_name', requirements: REPOSITORY_ENDPOINT_REQUIREMENTS do authorize_read_container_image! validate_tag! @@ -96,7 +102,7 @@ module API requires :repository_id, type: Integer, desc: 'The ID of the repository' requires :tag_name, type: String, desc: 'The name of the tag' end - delete ':id/registry/repositories/:repository_id/tags/:tag_name', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do + delete ':id/registry/repositories/:repository_id/tags/:tag_name', requirements: REPOSITORY_ENDPOINT_REQUIREMENTS do authorize_destroy_container_image! validate_tag! @@ -115,18 +121,21 @@ module API authorize! :read_container_image, repository end - def authorize_update_container_image! - authorize! :update_container_image, repository - end - def authorize_destroy_container_image! - authorize! :admin_container_image, repository + authorize! :destroy_container_image, repository end def authorize_admin_container_image! authorize! :admin_container_image, repository end + def obtain_new_cleanup_container_lease + Gitlab::ExclusiveLease + .new("container_repository:cleanup_tags:#{repository.id}", + timeout: 1.hour) + .try_obtain + end + def repository @repository ||= user_project.container_repositories.find(params[:repository_id]) end diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb index 71891e43dcc..bb1b037c08f 100644 --- a/lib/api/project_import.rb +++ b/lib/api/project_import.rb @@ -59,6 +59,7 @@ module API } override_params = import_params.delete(:override_params) + filter_attributes_using_license!(override_params) if override_params project = ::Projects::GitlabProjectsImportService.new( current_user, project_params, override_params diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 1e14c77b5be..996205d4b7b 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -115,6 +115,22 @@ module API present_projects load_projects end + + desc 'Get projects starred by a user' do + success Entities::BasicProjectDetails + end + params do + requires :user_id, type: String, desc: 'The ID or username of the user' + use :collection_params + use :statistics_params + end + get ":user_id/starred_projects" do + user = find_user(params[:user_id]) + not_found!('User') unless user + + starred_projects = StarredProjectsFinder.new(user, params: project_finder_params, current_user: current_user).execute + present_projects starred_projects + end end resource :projects do @@ -145,6 +161,7 @@ module API post do attrs = declared_params(include_missing: false) attrs = translate_params_for_compatibility(attrs) + filter_attributes_using_license!(attrs) project = ::Projects::CreateService.new(current_user, attrs).execute if project.saved? @@ -179,6 +196,7 @@ module API attrs = declared_params(include_missing: false) attrs = translate_params_for_compatibility(attrs) + filter_attributes_using_license!(attrs) project = ::Projects::CreateService.new(user, attrs).execute if project.saved? @@ -292,7 +310,7 @@ module API authorize! :change_visibility_level, user_project if attrs[:visibility].present? attrs = translate_params_for_compatibility(attrs) - + filter_attributes_using_license!(attrs) verify_update_project_attrs!(user_project, attrs) result = ::Projects::UpdateService.new(user_project, current_user, attrs).execute @@ -356,6 +374,19 @@ module API end end + desc 'Get the users who starred a project' do + success Entities::UserBasic + end + params do + optional :search, type: String, desc: 'Return list of users matching the search criteria' + use :pagination + end + get ':id/starrers' do + starrers = UsersStarProjectsFinder.new(user_project, params, current_user: current_user).execute + + present paginate(starrers), with: Entities::UserStarsProject + end + desc 'Get languages in project repository' get ':id/languages' do ::Projects::RepositoryLanguagesService @@ -474,7 +505,7 @@ module API authorize_admin_project begin - ::Projects::HousekeepingService.new(user_project).execute + ::Projects::HousekeepingService.new(user_project, :gc).execute rescue ::Projects::HousekeepingService::LeaseTaken => error conflict!(error.message) end diff --git a/lib/api/releases.rb b/lib/api/releases.rb index 6b17f4317db..7a3d804c30c 100644 --- a/lib/api/releases.rb +++ b/lib/api/releases.rb @@ -54,6 +54,7 @@ module API requires :url, type: String end end + optional :released_at, type: DateTime, desc: 'The date when the release will be/was ready. Defaults to the current time.' end post ':id/releases' do authorize_create_release! @@ -77,6 +78,7 @@ module API requires :tag_name, type: String, desc: 'The name of the tag', as: :tag optional :name, type: String, desc: 'The name of the release' optional :description, type: String, desc: 'Release notes with markdown support' + optional :released_at, type: DateTime, desc: 'The date when the release will be/was ready.' end put ':id/releases/:tag_name', requirements: RELEASE_ENDPOINT_REQUIREMETS do authorize_update_release! diff --git a/lib/api/resource_label_events.rb b/lib/api/resource_label_events.rb index 448bef12cec..505a6c68c9c 100644 --- a/lib/api/resource_label_events.rb +++ b/lib/api/resource_label_events.rb @@ -26,7 +26,7 @@ module API # rubocop: disable CodeReuse/ActiveRecord get ":id/#{eventables_str}/:eventable_id/resource_label_events" do - eventable = find_noteable(parent_type, eventables_str, params[:eventable_id]) + eventable = find_noteable(parent_type, params[:id], eventable_type, params[:eventable_id]) events = eventable.resource_label_events.includes(:label, :user) present paginate(events), with: Entities::ResourceLabelEvent @@ -42,7 +42,7 @@ module API requires :eventable_id, types: [Integer, String], desc: 'The ID of the eventable' end get ":id/#{eventables_str}/:eventable_id/resource_label_events/:event_id" do - eventable = find_noteable(parent_type, eventables_str, params[:eventable_id]) + eventable = find_noteable(parent_type, params[:id], eventable_type, params[:eventable_id]) event = eventable.resource_label_events.find(params[:event_id]) present event, with: Entities::ResourceLabelEvent diff --git a/lib/api/runners.rb b/lib/api/runners.rb index f3fea463e7f..c2d371b6867 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -115,6 +115,8 @@ module API params do requires :id, type: Integer, desc: 'The ID of the runner' optional :status, type: String, desc: 'Status of the job', values: Ci::Build::AVAILABLE_STATUSES + optional :order_by, type: String, desc: 'Order by `id` or not', values: RunnerJobsFinder::ALLOWED_INDEXED_COLUMNS + optional :sort, type: String, values: %w[asc desc], default: 'desc', desc: 'Sort by asc (ascending) or desc (descending)' use :pagination end get ':id/jobs' do diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 6767ef882cb..c36ee5af63f 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -36,10 +36,6 @@ module API given akismet_enabled: ->(val) { val } do requires :akismet_api_key, type: String, desc: 'Generate API key at http://www.akismet.com' end - optional :clientside_sentry_enabled, type: Boolean, desc: 'Sentry can also be used for reporting and logging clientside exceptions. https://sentry.io/for/javascript/' - given clientside_sentry_enabled: ->(val) { val } do - requires :clientside_sentry_dsn, type: String, desc: 'Clientside Sentry Data Source Name' - end optional :container_registry_token_expire_delay, type: Integer, desc: 'Authorization token duration (minutes)' optional :default_artifacts_expire_in, type: String, desc: "Set the default expiration time for each job's artifacts" optional :default_project_creation, type: Integer, values: ::Gitlab::Access.project_creation_values, desc: 'Determine if developers can create projects in the group' @@ -59,9 +55,11 @@ module API optional :gitaly_timeout_default, type: Integer, desc: 'Default Gitaly timeout, in seconds. Set to 0 to disable timeouts.' optional :gitaly_timeout_fast, type: Integer, desc: 'Gitaly fast operation timeout, in seconds. Set to 0 to disable timeouts.' optional :gitaly_timeout_medium, type: Integer, desc: 'Medium Gitaly timeout, in seconds. Set to 0 to disable timeouts.' + optional :grafana_enabled, type: Boolean, desc: 'Enable Grafana' + optional :grafana_url, type: String, desc: 'Grafana URL' optional :gravatar_enabled, type: Boolean, desc: 'Flag indicating if the Gravatar service is enabled' optional :help_page_hide_commercial_content, type: Boolean, desc: 'Hide marketing-related entries from help' - optional :help_page_support_url, type: String, desc: 'Alternate support URL for help page' + optional :help_page_support_url, type: String, desc: 'Alternate support URL for help page and help dropdown' optional :help_page_text, type: String, desc: 'Custom text displayed on the help page' optional :home_page_url, type: String, desc: 'We will redirect non-logged in users to this page' optional :housekeeping_enabled, type: Boolean, desc: 'Enable automatic repository housekeeping (git repack, git gc)' @@ -114,10 +112,6 @@ module API end optional :restricted_visibility_levels, type: Array[String], desc: 'Selected levels cannot be used by non-admin users for groups, projects or snippets. If the public level is restricted, user profiles are only visible to logged in users.' optional :send_user_confirmation_email, type: Boolean, desc: 'Send confirmation email on sign-up' - optional :sentry_enabled, type: Boolean, desc: 'Sentry is an error reporting and logging tool which is currently not shipped with GitLab, get it here: https://getsentry.com' - given sentry_enabled: ->(val) { val } do - requires :sentry_dsn, type: String, desc: 'Sentry Data Source Name' - end optional :session_expire_delay, type: Integer, desc: 'Session duration in minutes. GitLab restart is required to apply changes.' optional :shared_runners_enabled, type: Boolean, desc: 'Enable shared runners for new projects' given shared_runners_enabled: ->(val) { val } do @@ -130,6 +124,13 @@ module API optional :usage_ping_enabled, type: Boolean, desc: 'Every week GitLab will report license usage back to GitLab, Inc.' optional :instance_statistics_visibility_private, type: Boolean, desc: 'When set to `true` Instance statistics will only be available to admins' optional :local_markdown_version, type: Integer, desc: "Local markdown version, increase this value when any cached markdown should be invalidated" + optional :allow_local_requests_from_hooks_and_services, type: Boolean, desc: 'Deprecated: Use :allow_local_requests_from_web_hooks_and_services instead. Allow requests to the local network from hooks and services.' # support legacy names, can be removed in v5 + optional :snowplow_enabled, type: Grape::API::Boolean, desc: 'Enable Snowplow tracking' + given snowplow_enabled: ->(val) { val } do + requires :snowplow_collector_hostname, type: String, desc: 'The Snowplow collector hostname' + optional :snowplow_cookie_domain, type: String, desc: 'The Snowplow cookie domain' + optional :snowplow_site_id, type: String, desc: 'The Snowplow site name / application ic' + end ApplicationSetting::SUPPORTED_KEY_TYPES.each do |type| optional :"#{type}_key_restriction", @@ -164,6 +165,11 @@ module API attrs[:password_authentication_enabled_for_web] = attrs.delete(:password_authentication_enabled) end + # support legacy names, can be removed in v5 + if attrs.has_key?(:allow_local_requests_from_hooks_and_services) + attrs[:allow_local_requests_from_web_hooks_and_services] = attrs.delete(:allow_local_requests_from_hooks_and_services) + end + attrs = filter_attributes_using_license(attrs) if ApplicationSettings::UpdateService.new(current_settings, current_user, attrs).execute diff --git a/lib/api/tags.rb b/lib/api/tags.rb index f5359fd316c..796b1450602 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -55,7 +55,7 @@ module API optional :release_description, type: String, desc: 'Specifying release notes stored in the GitLab database (deprecated in GitLab 11.7)' end post ':id/repository/tags' do - authorize_push_project + authorize_admin_tag result = ::Tags::CreateService.new(user_project, current_user) .execute(params[:tag_name], params[:ref], params[:message]) @@ -87,7 +87,7 @@ module API requires :tag_name, type: String, desc: 'The name of the tag' end delete ':id/repository/tags/:tag_name', requirements: TAG_ENDPOINT_REQUIREMENTS do - authorize_push_project + authorize_admin_tag tag = user_project.repository.find_tag(params[:tag_name]) not_found!('Tag') unless tag diff --git a/lib/api/todos.rb b/lib/api/todos.rb index d2196f05173..404675bfaec 100644 --- a/lib/api/todos.rb +++ b/lib/api/todos.rb @@ -13,6 +13,13 @@ module API 'issues' => ->(iid) { find_project_issue(iid) } }.freeze + helpers do + # EE::API::Todos would override this method + def find_todos + TodosFinder.new(current_user, params).execute + end + end + params do requires :id, type: String, desc: 'The ID of a project' end @@ -41,10 +48,6 @@ module API resource :todos do helpers do - def find_todos - TodosFinder.new(current_user, params).execute - end - def issuable_and_awardable?(type) obj_type = Object.const_get(type) @@ -65,7 +68,7 @@ module API next unless collection targets = collection.map(&:target) - options[type] = { issuable_metadata: issuable_meta_data(targets, type) } + options[type] = { issuable_metadata: issuable_meta_data(targets, type, current_user) } end end end @@ -77,7 +80,7 @@ module API use :pagination end get do - todos = paginate(find_todos.with_api_entity_associations) + todos = paginate(find_todos.with_entity_associations) options = { with: Entities::Todo, current_user: current_user } batch_load_issuable_metadata(todos, options) @@ -107,3 +110,5 @@ module API end end end + +API::Todos.prepend_if_ee('EE::API::Todos') diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index 0e829c5699b..eeecc390256 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -112,27 +112,6 @@ module API end end - desc 'Take ownership of trigger' do - success Entities::Trigger - end - params do - requires :trigger_id, type: Integer, desc: 'The trigger ID' - end - post ':id/triggers/:trigger_id/take_ownership' do - authenticate! - authorize! :admin_build, user_project - - trigger = user_project.triggers.find(params.delete(:trigger_id)) - break not_found!('Trigger') unless trigger - - if trigger.update(owner: current_user) - status :ok - present trigger, with: Entities::Trigger, current_user: current_user - else - render_validation_error!(trigger) - end - end - desc 'Delete a trigger' do success Entities::Trigger end diff --git a/lib/api/user_counts.rb b/lib/api/user_counts.rb new file mode 100644 index 00000000000..8df4b381bbf --- /dev/null +++ b/lib/api/user_counts.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module API + class UserCounts < Grape::API + resource :user_counts do + desc 'Return the user specific counts' do + detail 'Open MR Count' + end + get do + unauthorized! unless current_user + + { + merge_requests: current_user.assigned_open_merge_requests_count + } + end + end + end +end diff --git a/lib/api/users.rb b/lib/api/users.rb index 6afeebb6890..a4ac5b629b8 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -51,7 +51,7 @@ module API optional :can_create_group, type: Boolean, desc: 'Flag indicating the user can create groups' optional :external, type: Boolean, desc: 'Flag indicating the user is an external user' optional :avatar, type: File, desc: 'Avatar image for user' - optional :private_profile, type: Boolean, desc: 'Flag indicating the user has a private profile' + optional :private_profile, type: Boolean, default: false, desc: 'Flag indicating the user has a private profile' all_or_none_of :extern_uid, :provider use :optional_params_ee @@ -148,7 +148,7 @@ module API end desc 'Create a user. Available only for admins.' do - success Entities::UserPublic + success Entities::UserWithAdmin end params do requires :email, type: String, desc: 'The email of the user' @@ -158,6 +158,7 @@ module API at_least_one_of :password, :reset_password requires :name, type: String, desc: 'The name of the user' requires :username, type: String, desc: 'The username of the user' + optional :force_random_password, type: Boolean, desc: 'Flag indicating a random password will be set' use :optional_attributes end post do @@ -167,7 +168,7 @@ module API user = ::Users::CreateService.new(current_user, params).execute(skip_authorization: true) if user.persisted? - present user, with: Entities::UserPublic, current_user: current_user + present user, with: Entities::UserWithAdmin, current_user: current_user else conflict!('Email has already been taken') if User .by_any_email(user.email.downcase) @@ -182,7 +183,7 @@ module API end desc 'Update a user. Available only for admins.' do - success Entities::UserPublic + success Entities::UserWithAdmin end params do requires :id, type: Integer, desc: 'The ID of the user' @@ -209,25 +210,12 @@ module API .where.not(id: user.id).count > 0 user_params = declared_params(include_missing: false) - identity_attrs = user_params.slice(:provider, :extern_uid) - - if identity_attrs.any? - identity = user.identities.find_by(provider: identity_attrs[:provider]) - - if identity - identity.update(identity_attrs) - else - identity = user.identities.build(identity_attrs) - identity.save - end - end user_params[:password_expires_at] = Time.now if user_params[:password].present? - - result = ::Users::UpdateService.new(current_user, user_params.except(:extern_uid, :provider).merge(user: user)).execute + result = ::Users::UpdateService.new(current_user, user_params.merge(user: user)).execute if result[:status] == :success - present user, with: Entities::UserPublic, current_user: current_user + present user, with: Entities::UserWithAdmin, current_user: current_user else render_validation_error!(user) end diff --git a/lib/api/validations/types/labels_list.rb b/lib/api/validations/types/labels_list.rb index 47cd83c29cf..60277b99106 100644 --- a/lib/api/validations/types/labels_list.rb +++ b/lib/api/validations/types/labels_list.rb @@ -10,7 +10,7 @@ module API when String value.split(',').map(&:strip) when Array - value.map { |v| v.to_s.split(',').map(&:strip) }.flatten + value.flat_map { |v| v.to_s.split(',').map(&:strip) } when LabelsList value else diff --git a/lib/api/variables.rb b/lib/api/variables.rb index af1d7936556..f022b9e665a 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -7,8 +7,6 @@ module API before { authenticate! } before { authorize! :admin_build, user_project } - helpers Helpers::VariablesHelpers - helpers do def filter_variable_parameters(params) # This method exists so that EE can more easily filter out certain @@ -59,8 +57,7 @@ module API optional :protected, type: Boolean, desc: 'Whether the variable is protected' optional :masked, type: Boolean, desc: 'Whether the variable is masked' optional :variable_type, type: String, values: Ci::Variable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file. Defaults to env_var' - - use :optional_params_ee + optional :environment_scope, type: String, desc: 'The environment_scope of the variable' end post ':id/variables' do variable_params = declared_params(include_missing: false) @@ -84,8 +81,7 @@ module API optional :protected, type: Boolean, desc: 'Whether the variable is protected' optional :masked, type: Boolean, desc: 'Whether the variable is masked' optional :variable_type, type: String, values: Ci::Variable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file' - - use :optional_params_ee + optional :environment_scope, type: String, desc: 'The environment_scope of the variable' end # rubocop: disable CodeReuse/ActiveRecord put ':id/variables/:key' do diff --git a/lib/backup/database.rb b/lib/backup/database.rb index cd8e29d14d3..7e457c4982d 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -23,11 +23,6 @@ module Backup dump_pid = case config["adapter"] - when /^mysql/ then - progress.print "Dumping MySQL database #{config['database']} ... " - # Workaround warnings from MySQL 5.6 about passwords on cmd line - ENV['MYSQL_PWD'] = config["password"].to_s if config["password"] - spawn('mysqldump', *mysql_args, config['database'], out: compress_wr) when "postgresql" then progress.print "Dumping PostgreSQL database #{config['database']} ... " pg_env @@ -57,11 +52,6 @@ module Backup restore_pid = case config["adapter"] - when /^mysql/ then - progress.print "Restoring MySQL database #{config['database']} ... " - # Workaround warnings from MySQL 5.6 about passwords on cmd line - ENV['MYSQL_PWD'] = config["password"].to_s if config["password"] - spawn('mysql', *mysql_args, config['database'], in: decompress_rd) when "postgresql" then progress.print "Restoring PostgreSQL database #{config['database']} ... " pg_env @@ -80,23 +70,6 @@ module Backup protected - def mysql_args - args = { - 'host' => '--host', - 'port' => '--port', - 'socket' => '--socket', - 'username' => '--user', - 'encoding' => '--default-character-set', - # SSL - 'sslkey' => '--ssl-key', - 'sslcert' => '--ssl-cert', - 'sslca' => '--ssl-ca', - 'sslcapath' => '--ssl-capath', - 'sslcipher' => '--ssl-cipher' - } - args.map { |opt, arg| "#{arg}=#{config[opt]}" if config[opt] }.compact - end - def pg_env args = { 'username' => 'PGUSER', diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 0224dd8fcd1..52af28ce8ec 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -337,6 +337,24 @@ module Banzai @current_project_namespace_path ||= project&.namespace&.full_path end + def records_per_parent + @_records_per_project ||= {} + + @_records_per_project[object_class.to_s.underscore] ||= begin + hash = Hash.new { |h, k| h[k] = {} } + + parent_per_reference.each do |path, parent| + record_ids = references_per_parent[path] + + parent_records(parent, record_ids).each do |record| + hash[parent][record_identifier(record)] = record + end + end + + hash + end + end + private def full_project_path(namespace, project_ref) diff --git a/lib/banzai/filter/ascii_doc_sanitization_filter.rb b/lib/banzai/filter/ascii_doc_sanitization_filter.rb new file mode 100644 index 00000000000..9105e86ad04 --- /dev/null +++ b/lib/banzai/filter/ascii_doc_sanitization_filter.rb @@ -0,0 +1,111 @@ +# frozen_string_literal: true + +module Banzai + module Filter + # Sanitize HTML produced by AsciiDoc/Asciidoctor. + # + # Extends Banzai::Filter::BaseSanitizationFilter with specific rules. + class AsciiDocSanitizationFilter < Banzai::Filter::BaseSanitizationFilter + # Section anchor link pattern + SECTION_LINK_REF_PATTERN = /\A#{Gitlab::Asciidoc::DEFAULT_ADOC_ATTRS['idprefix']}(:?[[:alnum:]]|-|_)+\z/.freeze + SECTION_HEADINGS = %w(h2 h3 h4 h5 h6).freeze + + # Footnote link patterns + FOOTNOTE_LINK_ID_PATTERNS = { + a: /\A_footnoteref_\d+\z/, + div: /\A_footnotedef_\d+\z/ + }.freeze + + # Classes used by Asciidoctor to style components + ADMONITION_CLASSES = %w(fa icon-note icon-tip icon-warning icon-caution icon-important).freeze + CALLOUT_CLASSES = ['conum'].freeze + CHECKLIST_CLASSES = %w(fa fa-check-square-o fa-square-o).freeze + LIST_CLASSES = %w(checklist none no-bullet unnumbered unstyled).freeze + + ELEMENT_CLASSES_WHITELIST = { + span: %w(big small underline overline line-through).freeze, + div: ['admonitionblock'].freeze, + td: ['icon'].freeze, + i: ADMONITION_CLASSES + CALLOUT_CLASSES + CHECKLIST_CLASSES, + ul: LIST_CLASSES, + ol: LIST_CLASSES, + a: ['anchor'].freeze + }.freeze + + def customize_whitelist(whitelist) + # Allow marks + whitelist[:elements].push('mark') + + # Allow any classes in `span`, `i`, `div`, `td`, `ul`, `ol` and `a` elements + # but then remove any unknown classes + whitelist[:attributes]['span'] = %w(class) + whitelist[:attributes]['div'].push('class') + whitelist[:attributes]['td'] = %w(class) + whitelist[:attributes]['i'] = %w(class) + whitelist[:attributes]['ul'] = %w(class) + whitelist[:attributes]['ol'] = %w(class) + whitelist[:attributes]['a'].push('class') + whitelist[:transformers].push(self.class.remove_element_classes) + + # Allow `id` in heading elements for section anchors + SECTION_HEADINGS.each do |header| + whitelist[:attributes][header] = %w(id) + end + whitelist[:transformers].push(self.class.remove_non_heading_ids) + + # Allow `id` in footnote elements + FOOTNOTE_LINK_ID_PATTERNS.keys.each do |element| + whitelist[:attributes][element.to_s].push('id') + end + whitelist[:transformers].push(self.class.remove_non_footnote_ids) + + whitelist + end + + class << self + def remove_non_footnote_ids + lambda do |env| + node = env[:node] + + return unless (pattern = FOOTNOTE_LINK_ID_PATTERNS[node.name.to_sym]) + return unless node.has_attribute?('id') + + return if node['id'] =~ pattern + + node.remove_attribute('id') + end + end + + def remove_non_heading_ids + lambda do |env| + node = env[:node] + + return unless SECTION_HEADINGS.any?(node.name) + return unless node.has_attribute?('id') + + return if node['id'] =~ SECTION_LINK_REF_PATTERN + + node.remove_attribute('id') + end + end + + def remove_element_classes + lambda do |env| + node = env[:node] + + return unless (classes_whitelist = ELEMENT_CLASSES_WHITELIST[node.name.to_sym]) + return unless node.has_attribute?('class') + + classes = node['class'].strip.split(' ') + allowed_classes = (classes & classes_whitelist) + if allowed_classes.empty? + node.remove_attribute('class') + else + node['class'] = allowed_classes.join(' ') + end + end + end + end + end + end +end diff --git a/lib/banzai/filter/autolink_filter.rb b/lib/banzai/filter/autolink_filter.rb index 56214043d87..5f2cbc24c60 100644 --- a/lib/banzai/filter/autolink_filter.rb +++ b/lib/banzai/filter/autolink_filter.rb @@ -18,6 +18,7 @@ module Banzai # class AutolinkFilter < HTML::Pipeline::Filter include ActionView::Helpers::TagHelper + include Gitlab::Utils::SanitizeNodeLink # Pattern to match text that should be autolinked. # @@ -72,19 +73,11 @@ module Banzai private - # Return true if any of the UNSAFE_PROTOCOLS strings are included in the URI scheme - def contains_unsafe?(scheme) - return false unless scheme - - scheme = scheme.strip.downcase - Banzai::Filter::SanitizationFilter::UNSAFE_PROTOCOLS.any? { |protocol| scheme.include?(protocol) } - end - def autolink_match(match) # start by stripping out dangerous links begin uri = Addressable::URI.parse(match) - return match if contains_unsafe?(uri.scheme) + return match unless safe_protocol?(uri.scheme) rescue Addressable::URI::InvalidURIError return match end diff --git a/lib/banzai/filter/base_sanitization_filter.rb b/lib/banzai/filter/base_sanitization_filter.rb new file mode 100644 index 00000000000..2dabca3552d --- /dev/null +++ b/lib/banzai/filter/base_sanitization_filter.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module Banzai + module Filter + # Sanitize HTML produced by markup languages (Markdown, AsciiDoc...). + # Specific rules are implemented in dedicated filters: + # + # - Banzai::Filter::SanitizationFilter (Markdown) + # - Banzai::Filter::AsciiDocSanitizationFilter (AsciiDoc/Asciidoctor) + # + # Extends HTML::Pipeline::SanitizationFilter with common rules. + class BaseSanitizationFilter < HTML::Pipeline::SanitizationFilter + include Gitlab::Utils::StrongMemoize + extend Gitlab::Utils::SanitizeNodeLink + + UNSAFE_PROTOCOLS = %w(data javascript vbscript).freeze + + def whitelist + strong_memoize(:whitelist) do + whitelist = super.deep_dup + + # Allow span elements + whitelist[:elements].push('span') + + # Allow data-math-style attribute in order to support LaTeX formatting + whitelist[:attributes]['code'] = %w(data-math-style) + whitelist[:attributes]['pre'] = %w(data-math-style) + + # Allow html5 details/summary elements + whitelist[:elements].push('details') + whitelist[:elements].push('summary') + + # Allow abbr elements with title attribute + whitelist[:elements].push('abbr') + whitelist[:attributes]['abbr'] = %w(title) + + # Disallow `name` attribute globally, allow on `a` + whitelist[:attributes][:all].delete('name') + whitelist[:attributes]['a'].push('name') + + # Allow any protocol in `a` elements + # and then remove links with unsafe protocols + whitelist[:protocols].delete('a') + whitelist[:transformers].push(self.class.method(:remove_unsafe_links)) + + # Remove `rel` attribute from `a` elements + whitelist[:transformers].push(self.class.remove_rel) + + customize_whitelist(whitelist) + end + end + + def customize_whitelist(whitelist) + raise NotImplementedError + end + + class << self + def remove_rel + lambda do |env| + if env[:node_name] == 'a' + env[:node].remove_attribute('rel') + end + end + end + end + end + end +end diff --git a/lib/banzai/filter/commit_reference_filter.rb b/lib/banzai/filter/commit_reference_filter.rb index c3e5ac41cb8..e1d7b36b9a2 100644 --- a/lib/banzai/filter/commit_reference_filter.rb +++ b/lib/banzai/filter/commit_reference_filter.rb @@ -19,12 +19,11 @@ module Banzai end def find_object(project, id) - return unless project.is_a?(Project) + return unless project.is_a?(Project) && project.valid_repo? - if project && project.valid_repo? - # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/43894 - Gitlab::GitalyClient.allow_n_plus_1_calls { project.commit(id) } - end + _, record = records_per_parent[project].detect { |k, _v| Gitlab::Git.shas_eql?(k, id) } + + record end def referenced_merge_request_commit_shas @@ -66,6 +65,14 @@ module Banzai private + def record_identifier(record) + record.id + end + + def parent_records(parent, ids) + parent.commits_by(oids: ids.to_a) + end + def noteable context[:noteable] end diff --git a/lib/banzai/filter/inline_embeds_filter.rb b/lib/banzai/filter/inline_embeds_filter.rb new file mode 100644 index 00000000000..9f1ef0796f0 --- /dev/null +++ b/lib/banzai/filter/inline_embeds_filter.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +module Banzai + module Filter + # HTML filter that inserts a node for each occurence of + # a given link format. To transform references to DB + # resources in place, prefer to inherit from AbstractReferenceFilter. + class InlineEmbedsFilter < HTML::Pipeline::Filter + # Find every relevant link, create a new node based on + # the link, and insert this node after any html content + # surrounding the link. + def call + doc.xpath(xpath_search).each do |node| + next unless element = element_to_embed(node) + + # We want this to follow any surrounding content. For example, + # if a link is inline in a paragraph. + node.parent.children.last.add_next_sibling(element) + end + + doc + end + + # Implement in child class. + # + # Return a Nokogiri::XML::Element to embed in the + # markdown. + def create_element(params) + end + + # Implement in child class unless overriding #embed_params + # + # Returns the regex pattern used to filter + # to only matching urls. + def link_pattern + end + + # Returns the xpath query string used to select nodes + # from the html document on which the embed is based. + # + # Override to select nodes other than links. + def xpath_search + 'descendant-or-self::a[@href]' + end + + # Creates a new element based on the parameters + # obtained from the target link + def element_to_embed(node) + return unless params = embed_params(node) + + create_element(params) + end + + # Returns a hash of named parameters based on the + # provided regex with string keys. + # + # Override to select nodes other than links. + def embed_params(node) + url = node['href'] + + link_pattern.match(url) { |m| m.named_captures } + end + end + end +end diff --git a/lib/banzai/filter/inline_metrics_filter.rb b/lib/banzai/filter/inline_metrics_filter.rb new file mode 100644 index 00000000000..c5a328c21b2 --- /dev/null +++ b/lib/banzai/filter/inline_metrics_filter.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Banzai + module Filter + # HTML filter that inserts a placeholder element for each + # reference to a metrics dashboard. + class InlineMetricsFilter < Banzai::Filter::InlineEmbedsFilter + # Placeholder element for the frontend to use as an + # injection point for charts. + def create_element(params) + doc.document.create_element( + 'div', + class: 'js-render-metrics', + 'data-dashboard-url': metrics_dashboard_url(params) + ) + end + + # Search params for selecting metrics links. A few + # simple checks is enough to boost performance without + # the cost of doing a full regex match. + def xpath_search + "descendant-or-self::a[contains(@href,'metrics') and \ + starts-with(@href, '#{Gitlab.config.gitlab.url}')]" + end + + # Regular expression matching metrics urls + def link_pattern + Gitlab::Metrics::Dashboard::Url.regex + end + + private + + # Endpoint FE should hit to collect the appropriate + # chart information + def metrics_dashboard_url(params) + Gitlab::Metrics::Dashboard::Url.build_dashboard_url( + params['namespace'], + params['project'], + params['environment'], + embedded: true, + **query_params(params['url']) + ) + end + + # Parses query params out from full url string into hash. + # + # Ex) 'https://<root>/<project>/<environment>/metrics?title=Title&group=Group' + # --> { title: 'Title', group: 'Group' } + def query_params(url) + Gitlab::Metrics::Dashboard::Url.parse_query(url) + end + end + end +end diff --git a/lib/banzai/filter/inline_metrics_redactor_filter.rb b/lib/banzai/filter/inline_metrics_redactor_filter.rb new file mode 100644 index 00000000000..4d8a5028898 --- /dev/null +++ b/lib/banzai/filter/inline_metrics_redactor_filter.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: true + +module Banzai + module Filter + # HTML filter that removes embeded elements that the current user does + # not have permission to view. + class InlineMetricsRedactorFilter < HTML::Pipeline::Filter + include Gitlab::Utils::StrongMemoize + + METRICS_CSS_CLASS = '.js-render-metrics' + + # Finds all embeds based on the css class the FE + # uses to identify the embedded content, removing + # only unnecessary nodes. + def call + nodes.each do |node| + path = paths_by_node[node] + user_has_access = user_access_by_path[path] + + node.remove unless user_has_access + end + + doc + end + + private + + def user + context[:current_user] + end + + # Returns all nodes which the FE will identify as + # a metrics dashboard placeholder element + # + # @return [Nokogiri::XML::NodeSet] + def nodes + @nodes ||= doc.css(METRICS_CSS_CLASS) + end + + # Maps a node to the full path of a project. + # Memoized so we only need to run the regex to get + # the project full path from the url once per node. + # + # @return [Hash<Nokogiri::XML::Node, String>] + def paths_by_node + strong_memoize(:paths_by_node) do + nodes.each_with_object({}) do |node, paths| + paths[node] = path_for_node(node) + end + end + end + + # Gets a project's full_path from the dashboard url + # in the placeholder node. The FE will use the attr + # `data-dashboard-url`, so we want to check against that + # attribute directly in case a user has manually + # created a metrics element (rather than supporting + # an alternate attr in InlineMetricsFilter). + # + # @return [String] + def path_for_node(node) + url = node.attribute('data-dashboard-url').to_s + + Gitlab::Metrics::Dashboard::Url.regex.match(url) do |m| + "#{$~[:namespace]}/#{$~[:project]}" + end + end + + # Maps a project's full path to a Project object. + # Contains all of the Projects referenced in the + # metrics placeholder elements of the current document + # + # @return [Hash<String, Project>] + def projects_by_path + strong_memoize(:projects_by_path) do + Project.eager_load(:route, namespace: [:route]) + .where_full_path_in(paths_by_node.values.uniq) + .index_by(&:full_path) + end + end + + # Returns a mapping representing whether the current user + # has permission to view the metrics for the project. + # Determined in a batch + # + # @return [Hash<Project, Boolean>] + def user_access_by_path + strong_memoize(:user_access_by_path) do + projects_by_path.each_with_object({}) do |(path, project), access| + access[path] = Ability.allowed?(user, :read_environment, project) + end + end + end + end + end +end diff --git a/lib/banzai/filter/issuable_reference_filter.rb b/lib/banzai/filter/issuable_reference_filter.rb index 2963cba91e8..b91ba9f7256 100644 --- a/lib/banzai/filter/issuable_reference_filter.rb +++ b/lib/banzai/filter/issuable_reference_filter.rb @@ -3,22 +3,8 @@ module Banzai module Filter class IssuableReferenceFilter < AbstractReferenceFilter - def records_per_parent - @records_per_project ||= {} - - @records_per_project[object_class.to_s.underscore] ||= begin - hash = Hash.new { |h, k| h[k] = {} } - - parent_per_reference.each do |path, parent| - record_ids = references_per_parent[path] - - parent_records(parent, record_ids).each do |record| - hash[parent][record.iid.to_i] = record - end - end - - hash - end + def record_identifier(record) + record.iid.to_i end def find_object(parent, iid) diff --git a/lib/banzai/filter/redactor_filter.rb b/lib/banzai/filter/reference_redactor_filter.rb index 1f091f594f8..485d3fd5fc7 100644 --- a/lib/banzai/filter/redactor_filter.rb +++ b/lib/banzai/filter/reference_redactor_filter.rb @@ -7,12 +7,12 @@ module Banzai # # Expected to be run in its own post-processing pipeline. # - class RedactorFilter < HTML::Pipeline::Filter + class ReferenceRedactorFilter < HTML::Pipeline::Filter def call unless context[:skip_redaction] context = RenderContext.new(project, current_user) - Redactor.new(context).redact([doc]) + ReferenceRedactor.new(context).redact([doc]) end doc diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb index 199b3533cf4..86f18679496 100644 --- a/lib/banzai/filter/relative_link_filter.rb +++ b/lib/banzai/filter/relative_link_filter.rb @@ -17,6 +17,8 @@ module Banzai include Gitlab::Utils::StrongMemoize def call + return doc if context[:system_note] + @uri_types = {} clear_memoization(:linkable_files) @@ -54,10 +56,10 @@ module Banzai def process_link_to_upload_attr(html_attr) path_parts = [Addressable::URI.unescape(html_attr.value)] - if group - path_parts.unshift(relative_url_root, 'groups', group.full_path, '-') - elsif project + if project path_parts.unshift(relative_url_root, project.full_path) + elsif group + path_parts.unshift(relative_url_root, 'groups', group.full_path, '-') else path_parts.unshift(relative_url_root) end @@ -100,7 +102,7 @@ module Banzai end def relative_file_path(uri) - path = Addressable::URI.unescape(uri.path) + path = Addressable::URI.unescape(uri.path).delete("\0") request_path = Addressable::URI.unescape(context[:requested_path]) nested_path = build_relative_path(path, request_path) file_exists?(nested_path) ? nested_path : path diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index a4a06eae7b7..f57e57890f8 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -2,23 +2,13 @@ module Banzai module Filter - # Sanitize HTML + # Sanitize HTML produced by Markdown. # - # Extends HTML::Pipeline::SanitizationFilter with a custom whitelist. - class SanitizationFilter < HTML::Pipeline::SanitizationFilter - include Gitlab::Utils::StrongMemoize - - UNSAFE_PROTOCOLS = %w(data javascript vbscript).freeze + # Extends Banzai::Filter::BaseSanitizationFilter with specific rules. + class SanitizationFilter < Banzai::Filter::BaseSanitizationFilter + # Styles used by Markdown for table alignment TABLE_ALIGNMENT_PATTERN = /text-align: (?<alignment>center|left|right)/.freeze - def whitelist - strong_memoize(:whitelist) do - customize_whitelist(super.deep_dup) - end - end - - private - def customize_whitelist(whitelist) # Allow table alignment; we whitelist specific text-align values in a # transformer below @@ -26,36 +16,9 @@ module Banzai whitelist[:attributes]['td'] = %w(style) whitelist[:css] = { properties: ['text-align'] } - # Allow span elements - whitelist[:elements].push('span') - - # Allow data-math-style attribute in order to support LaTeX formatting - whitelist[:attributes]['code'] = %w(data-math-style) - whitelist[:attributes]['pre'] = %w(data-math-style) - - # Allow html5 details/summary elements - whitelist[:elements].push('details') - whitelist[:elements].push('summary') - - # Allow abbr elements with title attribute - whitelist[:elements].push('abbr') - whitelist[:attributes]['abbr'] = %w(title) - # Allow the 'data-sourcepos' from CommonMark on all elements whitelist[:attributes][:all].push('data-sourcepos') - # Disallow `name` attribute globally, allow on `a` - whitelist[:attributes][:all].delete('name') - whitelist[:attributes]['a'].push('name') - - # Allow any protocol in `a` elements - # and then remove links with unsafe protocols - whitelist[:protocols].delete('a') - whitelist[:transformers].push(self.class.remove_unsafe_links) - - # Remove `rel` attribute from `a` elements - whitelist[:transformers].push(self.class.remove_rel) - # Remove any `style` properties not required for table alignment whitelist[:transformers].push(self.class.remove_unsafe_table_style) @@ -69,43 +32,6 @@ module Banzai end class << self - def remove_unsafe_links - lambda do |env| - node = env[:node] - - return unless node.name == 'a' - return unless node.has_attribute?('href') - - begin - node['href'] = node['href'].strip - uri = Addressable::URI.parse(node['href']) - - return unless uri.scheme - - # Remove all invalid scheme characters before checking against the - # list of unsafe protocols. - # - # See https://tools.ietf.org/html/rfc3986#section-3.1 - scheme = uri.scheme - .strip - .downcase - .gsub(/[^A-Za-z0-9\+\.\-]+/, '') - - node.remove_attribute('href') if UNSAFE_PROTOCOLS.include?(scheme) - rescue Addressable::URI::InvalidURIError - node.remove_attribute('href') - end - end - end - - def remove_rel - lambda do |env| - if env[:node_name] == 'a' - env[:node].remove_attribute('rel') - end - end - end - def remove_unsafe_table_style lambda do |env| node = env[:node] diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index fe56f9a1e33..9b66759a5fb 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -14,7 +14,7 @@ module Banzai LANG_PARAMS_ATTR = 'data-lang-params'.freeze def call - doc.search('pre > code').each do |node| + doc.search('pre:not([data-math-style]) > code').each do |node| highlight_node(node) end diff --git a/lib/banzai/filter/wiki_link_filter.rb b/lib/banzai/filter/wiki_link_filter.rb index 1728a442533..18947679b69 100644 --- a/lib/banzai/filter/wiki_link_filter.rb +++ b/lib/banzai/filter/wiki_link_filter.rb @@ -8,15 +8,19 @@ module Banzai # Context options: # :project_wiki class WikiLinkFilter < HTML::Pipeline::Filter + include Gitlab::Utils::SanitizeNodeLink + def call return doc unless project_wiki? - doc.search('a:not(.gfm)').each { |el| process_link_attr(el.attribute('href')) } - doc.search('video').each { |el| process_link_attr(el.attribute('src')) } + doc.search('a:not(.gfm)').each { |el| process_link(el.attribute('href'), el) } + + doc.search('video').each { |el| process_link(el.attribute('src'), el) } + doc.search('img').each do |el| attr = el.attribute('data-src') || el.attribute('src') - process_link_attr(attr) + process_link(attr, el) end doc @@ -24,6 +28,11 @@ module Banzai protected + def process_link(link_attr, node) + process_link_attr(link_attr) + remove_unsafe_links({ node: node }, remove_invalid_links: false) + end + def project_wiki? !context[:project_wiki].nil? end diff --git a/lib/banzai/filter/wiki_link_filter/rewriter.rb b/lib/banzai/filter/wiki_link_filter/rewriter.rb index 77b5053f38c..f4cc8beeb52 100644 --- a/lib/banzai/filter/wiki_link_filter/rewriter.rb +++ b/lib/banzai/filter/wiki_link_filter/rewriter.rb @@ -4,8 +4,6 @@ module Banzai module Filter class WikiLinkFilter < HTML::Pipeline::Filter class Rewriter - UNSAFE_SLUG_REGEXES = [/\Ajavascript:/i].freeze - def initialize(link_string, wiki:, slug:) @uri = Addressable::URI.parse(link_string) @wiki_base_path = wiki && wiki.wiki_base_path @@ -37,8 +35,6 @@ module Banzai # Of the form `./link`, `../link`, or similar def apply_hierarchical_link_rules! - return if slug_considered_unsafe? - @uri = Addressable::URI.join(@slug, @uri) if @uri.to_s[0] == '.' end @@ -58,10 +54,6 @@ module Banzai def repository_upload? @uri.relative? && @uri.path.starts_with?(Wikis::CreateAttachmentService::ATTACHMENT_PATH) end - - def slug_considered_unsafe? - UNSAFE_SLUG_REGEXES.any? { |r| r.match?(@slug) } - end end end end diff --git a/lib/banzai/object_renderer.rb b/lib/banzai/object_renderer.rb index 75661ffa233..d6d29f4bfab 100644 --- a/lib/banzai/object_renderer.rb +++ b/lib/banzai/object_renderer.rb @@ -72,7 +72,7 @@ module Banzai # # Returns an Array containing the redacted documents. def redact_documents(documents) - redactor = Redactor.new(context) + redactor = ReferenceRedactor.new(context) redactor.redact(documents) end diff --git a/lib/banzai/pipeline/ascii_doc_pipeline.rb b/lib/banzai/pipeline/ascii_doc_pipeline.rb index cc4af280872..d25b74b23b2 100644 --- a/lib/banzai/pipeline/ascii_doc_pipeline.rb +++ b/lib/banzai/pipeline/ascii_doc_pipeline.rb @@ -5,7 +5,8 @@ module Banzai class AsciiDocPipeline < BasePipeline def self.filters FilterArray[ - Filter::SanitizationFilter, + Filter::AsciiDocSanitizationFilter, + Filter::SyntaxHighlightFilter, Filter::ExternalLinkFilter, Filter::PlantumlFilter, Filter::AsciiDocPostProcessingFilter diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index d67f461be57..2c1006f708a 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -25,6 +25,7 @@ module Banzai Filter::VideoLinkFilter, Filter::ImageLazyLoadFilter, Filter::ImageLinkFilter, + Filter::InlineMetricsFilter, Filter::TableOfContentsFilter, Filter::AutolinkFilter, Filter::ExternalLinkFilter, diff --git a/lib/banzai/pipeline/post_process_pipeline.rb b/lib/banzai/pipeline/post_process_pipeline.rb index 7eaad6d7560..54af26b41be 100644 --- a/lib/banzai/pipeline/post_process_pipeline.rb +++ b/lib/banzai/pipeline/post_process_pipeline.rb @@ -12,7 +12,8 @@ module Banzai def self.internal_link_filters [ - Filter::RedactorFilter, + Filter::ReferenceRedactorFilter, + Filter::InlineMetricsRedactorFilter, Filter::RelativeLinkFilter, Filter::IssuableStateFilter, Filter::SuggestionFilter diff --git a/lib/banzai/redactor.rb b/lib/banzai/reference_redactor.rb index c2da7fec7cc..936436982e7 100644 --- a/lib/banzai/redactor.rb +++ b/lib/banzai/reference_redactor.rb @@ -3,7 +3,7 @@ module Banzai # Class for removing Markdown references a certain user is not allowed to # view. - class Redactor + class ReferenceRedactor attr_reader :context # context - An instance of `Banzai::RenderContext`. @@ -33,7 +33,7 @@ module Banzai # # data - An Array of a Hashes mapping an HTML document to nodes to redact. def redact_document_nodes(all_document_nodes) - all_nodes = all_document_nodes.map { |x| x[:nodes] }.flatten + all_nodes = all_document_nodes.flat_map { |x| x[:nodes] } visible = nodes_visible_to_user(all_nodes) metadata = [] diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index c7239a5eaa6..3cb9ec21e8f 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -55,11 +55,16 @@ module Banzai # Perform multiple render from an Array of Markdown String into an # Array of HTML-safe String of HTML. # - # As the rendered Markdown String can be already cached read all the data - # from the cache using Rails.cache.read_multi operation. If the Markdown String - # is not in the cache or it's not cacheable (no cache_key entry is provided in - # the context) the Markdown String is rendered and stored in the cache so the - # next render call gets the rendered HTML-safe String from the cache. + # The redis cache is completely obviated if we receive a `:rendered` key in the + # context, as it is assumed the item has been pre-rendered somewhere else and there + # is no need to cache it. + # + # If no `:rendered` key is present in the context, as the rendered Markdown String + # can be already cached, read all the data from the cache using + # Rails.cache.read_multi operation. If the Markdown String is not in the cache + # or it's not cacheable (no cache_key entry is provided in the context) the + # Markdown String is rendered and stored in the cache so the next render call + # gets the rendered HTML-safe String from the cache. # # For further explanation see #render method comments. # @@ -76,19 +81,34 @@ module Banzai # => [{ text: '### Hello', # context: { cache_key: [note, :note] } }] def self.cache_collection_render(texts_and_contexts) - items_collection = texts_and_contexts.each_with_index do |item, index| + items_collection = texts_and_contexts.each do |item| context = item[:context] - cache_key = full_cache_multi_key(context.delete(:cache_key), context[:pipeline]) - item[:cache_key] = cache_key if cache_key + if context.key?(:rendered) + item[:rendered] = context.delete(:rendered) + else + # If the attribute didn't come in pre-rendered, let's prepare it for caching it in redis + cache_key = full_cache_multi_key(context.delete(:cache_key), context[:pipeline]) + item[:cache_key] = cache_key if cache_key + end end - cacheable_items, non_cacheable_items = items_collection.partition { |item| item.key?(:cache_key) } + cacheable_items, non_cacheable_items = items_collection.group_by do |item| + if item.key?(:rendered) + # We're not really doing anything here as these don't need any processing, but leaving it just in case + # as they could have a cache_key and we don't want them to be re-rendered + :rendered + elsif item.key?(:cache_key) + :cacheable + else + :non_cacheable + end + end.values_at(:cacheable, :non_cacheable) items_in_cache = [] items_not_in_cache = [] - unless cacheable_items.empty? + if cacheable_items.present? items_in_cache = Rails.cache.read_multi(*cacheable_items.map { |item| item[:cache_key] }) items_not_in_cache = cacheable_items.reject do |item| item[:rendered] = items_in_cache[item[:cache_key]] @@ -96,7 +116,7 @@ module Banzai end end - (items_not_in_cache + non_cacheable_items).each do |item| + (items_not_in_cache + Array.wrap(non_cacheable_items)).each do |item| item[:rendered] = render(item[:text], item[:context]) Rails.cache.write(item[:cache_key], item[:rendered]) if item[:cache_key] end @@ -114,7 +134,7 @@ module Banzai # # This method is used to perform state-dependent changes to a String of # HTML, such as removing references that the current user doesn't have - # permission to make (`RedactorFilter`). + # permission to make (`ReferenceRedactorFilter`). # # html - String to process # context - Hash of options to customize output diff --git a/lib/bitbucket/representation/comment.rb b/lib/bitbucket/representation/comment.rb index 1b8dc27793a..127598a76ab 100644 --- a/lib/bitbucket/representation/comment.rb +++ b/lib/bitbucket/representation/comment.rb @@ -4,7 +4,7 @@ module Bitbucket module Representation class Comment < Representation::Base def author - user['username'] + user['nickname'] end def note diff --git a/lib/bitbucket/representation/issue.rb b/lib/bitbucket/representation/issue.rb index a88797cdab9..3f6db9cb75b 100644 --- a/lib/bitbucket/representation/issue.rb +++ b/lib/bitbucket/representation/issue.rb @@ -14,7 +14,7 @@ module Bitbucket end def author - raw.dig('reporter', 'username') + raw.dig('reporter', 'nickname') end def description diff --git a/lib/bitbucket/representation/pull_request.rb b/lib/bitbucket/representation/pull_request.rb index 6a0e8b354bf..a498c9bc213 100644 --- a/lib/bitbucket/representation/pull_request.rb +++ b/lib/bitbucket/representation/pull_request.rb @@ -4,7 +4,7 @@ module Bitbucket module Representation class PullRequest < Representation::Base def author - raw.fetch('author', {}).fetch('username', nil) + raw.fetch('author', {}).fetch('nickname', nil) end def description diff --git a/lib/bitbucket_server/client.rb b/lib/bitbucket_server/client.rb index 6a608058813..cf55c692271 100644 --- a/lib/bitbucket_server/client.rb +++ b/lib/bitbucket_server/client.rb @@ -23,8 +23,9 @@ module BitbucketServer BitbucketServer::Representation::Repo.new(parsed_response) end - def repos(page_offset: 0, limit: nil) + def repos(page_offset: 0, limit: nil, filter: nil) path = "/repos" + path += "?name=#{filter}" if filter get_collection(path, :repo, page_offset: page_offset, limit: limit) end diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb index c80f49f5ae0..82810ea4076 100644 --- a/lib/container_registry/client.rb +++ b/lib/container_registry/client.rb @@ -7,7 +7,9 @@ module ContainerRegistry class Client attr_accessor :uri - MANIFEST_VERSION = 'application/vnd.docker.distribution.manifest.v2+json'.freeze + DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE = 'application/vnd.docker.distribution.manifest.v2+json' + OCI_MANIFEST_V1_TYPE = 'application/vnd.oci.image.manifest.v1+json' + ACCEPTED_TYPES = [DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE, OCI_MANIFEST_V1_TYPE].freeze # Taken from: FaradayMiddleware::FollowRedirects REDIRECT_CODES = Set.new [301, 302, 303, 307] @@ -60,12 +62,13 @@ module ContainerRegistry end def accept_manifest(conn) - conn.headers['Accept'] = MANIFEST_VERSION + conn.headers['Accept'] = ACCEPTED_TYPES conn.response :json, content_type: 'application/json' conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+prettyjws' conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+json' - conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v2+json' + conn.response :json, content_type: DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE + conn.response :json, content_type: OCI_MANIFEST_V1_TYPE end def response_body(response, allow_redirect: false) @@ -79,7 +82,10 @@ module ContainerRegistry def redirect_response(location) return unless location - faraday_redirect.get(location) + uri = URI(@base_uri).merge(location) + raise ArgumentError, "Invalid scheme for #{location}" unless %w[http https].include?(uri.scheme) + + faraday_redirect.get(uri) end def faraday diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb index ef41dc560c9..ebea84fa1ca 100644 --- a/lib/container_registry/tag.rb +++ b/lib/container_registry/tag.rb @@ -6,6 +6,9 @@ module ContainerRegistry attr_reader :repository, :name + # https://github.com/docker/distribution/commit/3150937b9f2b1b5b096b2634d0e7c44d4a0f89fb + TAG_NAME_REGEX = /^[\w][\w.-]{0,127}$/.freeze + delegate :registry, :client, to: :repository delegate :revision, :short_revision, to: :config_blob, allow_nil: true @@ -13,6 +16,10 @@ module ContainerRegistry @repository, @name = repository, name end + def valid_name? + !name.match(TAG_NAME_REGEX).nil? + end + def valid? manifest.present? end diff --git a/lib/expand_variables.rb b/lib/expand_variables.rb index c83cec9dc4a..45af30f46dc 100644 --- a/lib/expand_variables.rb +++ b/lib/expand_variables.rb @@ -3,6 +3,20 @@ module ExpandVariables class << self def expand(value, variables) + variables_hash = nil + + value.gsub(/\$([a-zA-Z_][a-zA-Z0-9_]*)|\${\g<1>}|%\g<1>%/) do + variables_hash ||= transform_variables(variables) + variables_hash[$1 || $2] + end + end + + private + + def transform_variables(variables) + # Lazily initialise variables + variables = variables.call if variables.is_a?(Proc) + # Convert hash array to variables if variables.is_a?(Array) variables = variables.reduce({}) do |hash, variable| @@ -11,9 +25,7 @@ module ExpandVariables end end - value.gsub(/\$([a-zA-Z_][a-zA-Z0-9_]*)|\${\g<1>}|%\g<1>%/) do - variables[$1 || $2] - end + variables end end end diff --git a/lib/feature.rb b/lib/feature.rb index 749c861d740..c70a6980f19 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -30,7 +30,14 @@ class Feature end def persisted_names - Gitlab::SafeRequestStore[:flipper_persisted_names] ||= FlipperFeature.feature_names + Gitlab::SafeRequestStore[:flipper_persisted_names] ||= + begin + # We saw on GitLab.com, this database request was called 2300 + # times/s. Let's cache it for a minute to avoid that load. + Gitlab::ThreadMemoryCache.cache_backend.fetch('flipper:persisted_names', expires_in: 1.minute) do + FlipperFeature.feature_names + end + end end def persisted?(feature) @@ -73,6 +80,13 @@ class Feature get(key).disable_group(group) end + def remove(key) + feature = get(key) + return unless persisted?(feature) + + feature.remove + end + def flipper if Gitlab::SafeRequestStore.active? Gitlab::SafeRequestStore[:flipper] ||= build_flipper_instance @@ -96,10 +110,27 @@ class Feature feature_class: FlipperFeature, gate_class: FlipperGate) + # Redis L2 cache + redis_cache_adapter = + Flipper::Adapters::ActiveSupportCacheStore.new( + active_record_adapter, + l2_cache_backend, + expires_in: 1.hour) + + # Thread-local L1 cache: use a short timeout since we don't have a + # way to expire this cache all at once Flipper::Adapters::ActiveSupportCacheStore.new( - active_record_adapter, - Rails.cache, - expires_in: 1.hour) + redis_cache_adapter, + l1_cache_backend, + expires_in: 1.minute) + end + + def l1_cache_backend + Gitlab::ThreadMemoryCache.cache_backend + end + + def l2_cache_backend + Rails.cache end end diff --git a/lib/feature/gitaly.rb b/lib/feature/gitaly.rb new file mode 100644 index 00000000000..9ded1aed4e3 --- /dev/null +++ b/lib/feature/gitaly.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +require 'set' + +class Feature + class Gitaly + # Server feature flags should use '_' to separate words. + SERVER_FEATURE_FLAGS = + [ + # 'get_commit_signatures'.freeze + ].freeze + + DEFAULT_ON_FLAGS = Set.new([]).freeze + + class << self + def enabled?(feature_flag) + return false unless Feature::FlipperFeature.table_exists? + + default_on = DEFAULT_ON_FLAGS.include?(feature_flag) + Feature.enabled?("gitaly_#{feature_flag}", default_enabled: default_on) + rescue ActiveRecord::NoDatabaseError + false + end + + def server_feature_flags + SERVER_FEATURE_FLAGS.map do |f| + ["gitaly-feature-#{f.tr('_', '-')}", enabled?(f).to_s] + end.to_h + end + end + end +end diff --git a/lib/forever.rb b/lib/forever.rb index 0a37118fe68..3f923557441 100644 --- a/lib/forever.rb +++ b/lib/forever.rb @@ -1,15 +1,9 @@ # frozen_string_literal: true class Forever - POSTGRESQL_DATE = DateTime.new(3000, 1, 1) - MYSQL_DATE = DateTime.new(2038, 01, 19) + DATE = DateTime.new(3000, 1, 1) - # MySQL timestamp has a range of '1970-01-01 00:00:01' UTC to '2038-01-19 03:14:07' UTC def self.date - if Gitlab::Database.postgresql? - POSTGRESQL_DATE - else - MYSQL_DATE - end + DATE end end diff --git a/lib/gitaly/server.rb b/lib/gitaly/server.rb index 7b238623418..907c6e1b605 100644 --- a/lib/gitaly/server.rb +++ b/lib/gitaly/server.rb @@ -2,8 +2,18 @@ module Gitaly class Server - def self.all - Gitlab.config.repositories.storages.keys.map { |s| Gitaly::Server.new(s) } + class << self + def all + Gitlab.config.repositories.storages.keys.map { |s| Gitaly::Server.new(s) } + end + + def count + all.size + end + + def filesystems + all.map(&:filesystem_type).compact.uniq + end end attr_reader :storage @@ -36,6 +46,10 @@ module Gitaly storage_status&.writeable end + def filesystem_type + storage_status&.fs_type + end + def address Gitlab::GitalyClient.address(@storage) rescue RuntimeError => e diff --git a/lib/gitlab.rb b/lib/gitlab.rb index ccaf06c5d6a..d9d8dcf7900 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_dependency 'gitlab/popen' +require 'pathname' module Gitlab def self.root @@ -60,11 +60,19 @@ module Gitlab end def self.ee? - if ENV['IS_GITLAB_EE'].present? - Gitlab::Utils.to_boolean(ENV['IS_GITLAB_EE']) - else - Object.const_defined?(:License) - end + @is_ee ||= + if ENV['IS_GITLAB_EE'] && !ENV['IS_GITLAB_EE'].empty? + Gitlab::Utils.to_boolean(ENV['IS_GITLAB_EE']) + else + # We may use this method when the Rails environment is not loaded. This + # means that checking the presence of the License class could result in + # this method returning `false`, even for an EE installation. + root.join('ee/app/models/license.rb').exist? + end + end + + def self.ee + yield if ee? end def self.http_proxy_env? diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb index 6eb08f674c2..7ef9f7ef630 100644 --- a/lib/gitlab/access.rb +++ b/lib/gitlab/access.rb @@ -29,6 +29,10 @@ module Gitlab MAINTAINER_PROJECT_ACCESS = 1 DEVELOPER_MAINTAINER_PROJECT_ACCESS = 2 + # Default subgroup creation level + OWNER_SUBGROUP_ACCESS = 0 + MAINTAINER_SUBGROUP_ACCESS = 1 + class << self delegate :values, to: :options @@ -106,6 +110,13 @@ module Gitlab def project_creation_level_name(name) project_creation_options.key(name) end + + def subgroup_creation_options + { + s_('SubgroupCreationlevel|Owners') => OWNER_SUBGROUP_ACCESS, + s_('SubgroupCreationlevel|Maintainers') => MAINTAINER_SUBGROUP_ACCESS + } + end end def human_access diff --git a/lib/gitlab/action_rate_limiter.rb b/lib/gitlab/action_rate_limiter.rb index c442211e073..0e8707af631 100644 --- a/lib/gitlab/action_rate_limiter.rb +++ b/lib/gitlab/action_rate_limiter.rb @@ -33,16 +33,48 @@ module Gitlab # Increments the given key and returns true if the action should # be throttled. # - # key - An array of ActiveRecord instances - # threshold_value - The maximum number of times this action should occur in the given time interval + # key - An array of ActiveRecord instances or strings + # threshold_value - The maximum number of times this action should occur in the given time interval. If number is zero is considered disabled. def throttled?(key, threshold_value) - self.increment(key) > threshold_value + threshold_value > 0 && + self.increment(key) > threshold_value + end + + # Logs request into auth.log + # + # request - Web request to be logged + # type - A symbol key that represents the request. + # current_user - Current user of the request, it can be nil. + def log_request(request, type, current_user) + request_information = { + message: 'Action_Rate_Limiter_Request', + env: type, + remote_ip: request.ip, + request_method: request.request_method, + path: request.fullpath + } + + if current_user + request_information.merge!({ + user_id: current_user.id, + username: current_user.username + }) + end + + Gitlab::AuthLogger.error(request_information) end private def action_key(key) - serialized = key.map { |obj| "#{obj.class.model_name.to_s.underscore}:#{obj.id}" }.join(":") + serialized = key.map do |obj| + if obj.is_a?(String) + "#{obj}" + else + "#{obj.class.model_name.to_s.underscore}:#{obj.id}" + end + end.join(":") + "action_rate_limiter:#{action}:#{serialized}" end end diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index df8f0470063..da65caa6c9c 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -1,27 +1,43 @@ # frozen_string_literal: true require 'asciidoctor' -require 'asciidoctor/converter/html5' -require "asciidoctor-plantuml" +require 'asciidoctor-plantuml' +require 'asciidoctor/extensions' +require 'gitlab/asciidoc/html5_converter' +require 'gitlab/asciidoc/syntax_highlighter/html_pipeline_adapter' module Gitlab # Parser/renderer for the AsciiDoc format that uses Asciidoctor and filters # the resulting HTML through HTML pipeline filters. module Asciidoc - DEFAULT_ADOC_ATTRS = [ - 'showtitle', 'idprefix=user-content-', 'idseparator=-', 'env=gitlab', - 'env-gitlab', 'source-highlighter=html-pipeline', 'icons=font', - 'outfilesuffix=.adoc' - ].freeze + MAX_INCLUDE_DEPTH = 5 + DEFAULT_ADOC_ATTRS = { + 'showtitle' => true, + 'sectanchors' => true, + 'idprefix' => 'user-content-', + 'idseparator' => '-', + 'env' => 'gitlab', + 'env-gitlab' => '', + 'source-highlighter' => 'gitlab-html-pipeline', + 'icons' => 'font', + 'outfilesuffix' => '.adoc', + 'max-include-depth' => MAX_INCLUDE_DEPTH + }.freeze # Public: Converts the provided Asciidoc markup into HTML. # # input - the source text in Asciidoc format + # context - :commit, :project, :ref, :requested_path # def self.render(input, context) + extensions = proc do + include_processor ::Gitlab::Asciidoc::IncludeProcessor.new(context) + end + asciidoc_opts = { safe: :secure, backend: :gitlab_html5, - attributes: DEFAULT_ADOC_ATTRS } + attributes: DEFAULT_ADOC_ATTRS, + extensions: extensions } context[:pipeline] = :ascii_doc @@ -40,29 +56,5 @@ module Gitlab conf.txt_enable = false end end - - class Html5Converter < Asciidoctor::Converter::Html5Converter - extend Asciidoctor::Converter::Config - - register_for 'gitlab_html5' - - def stem(node) - return super unless node.style.to_sym == :latexmath - - %(<pre#{id_attribute(node)} data-math-style="display"><code>#{node.content}</code></pre>) - end - - def inline_quoted(node) - return super unless node.type.to_sym == :latexmath - - %(<code#{id_attribute(node)} data-math-style="inline">#{node.text}</code>) - end - - private - - def id_attribute(node) - node.id ? %( id="#{node.id}") : nil - end - end end end diff --git a/lib/gitlab/asciidoc/html5_converter.rb b/lib/gitlab/asciidoc/html5_converter.rb new file mode 100644 index 00000000000..e5163e1954c --- /dev/null +++ b/lib/gitlab/asciidoc/html5_converter.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require 'asciidoctor' + +module Gitlab + module Asciidoc + class Html5Converter < (Asciidoctor::Converter.for 'html5') + register_for 'gitlab_html5' + + def convert_stem(node) + return super unless node.style.to_sym == :latexmath + + %(<pre#{id_attribute(node)} data-math-style="display"><code>#{node.content}</code></pre>) + end + + def convert_inline_quoted(node) + return super unless node.type.to_sym == :latexmath + + %(<code#{id_attribute(node)} data-math-style="inline">#{node.text}</code>) + end + + private + + def id_attribute(node) + node.id ? %( id="#{node.id}") : nil + end + end + end +end diff --git a/lib/gitlab/asciidoc/include_processor.rb b/lib/gitlab/asciidoc/include_processor.rb new file mode 100644 index 00000000000..c6fbf540e9c --- /dev/null +++ b/lib/gitlab/asciidoc/include_processor.rb @@ -0,0 +1,126 @@ +# frozen_string_literal: true + +require 'asciidoctor/include_ext/include_processor' + +module Gitlab + module Asciidoc + # Asciidoctor extension for processing includes (macro include::[]) within + # documents inside the same repository. + class IncludeProcessor < Asciidoctor::IncludeExt::IncludeProcessor + extend ::Gitlab::Utils::Override + + def initialize(context) + super(logger: Gitlab::AppLogger) + + @context = context + @repository = context[:project].try(:repository) + + # Note: Asciidoctor calls #freeze on extensions, so we can't set new + # instance variables after initialization. + @cache = { + uri_types: {} + } + end + + protected + + override :include_allowed? + def include_allowed?(target, reader) + doc = reader.document + + return false if doc.attributes.fetch('max-include-depth').to_i < 1 + return false if target_uri?(target) + + true + end + + override :resolve_target_path + def resolve_target_path(target, reader) + return unless repository.try(:exists?) + + base_path = reader.include_stack.empty? ? requested_path : reader.file + path = resolve_relative_path(target, base_path) + + path if Gitlab::Git::Blob.find(repository, ref, path) + end + + override :read_lines + def read_lines(filename, selector) + blob = read_blob(ref, filename) + + if selector + blob.data.each_line.select.with_index(1, &selector) + else + blob.data + end + end + + override :unresolved_include! + def unresolved_include!(target, reader) + reader.unshift_line("*[ERROR: include::#{target}[] - unresolved directive]*") + end + + private + + attr_accessor :context, :repository, :cache + + # Gets a Blob at a path for a specific revision. + # This method will check that the Blob exists and contains readable text. + # + # revision - The String SHA1. + # path - The String file path. + # + # Returns a Blob + def read_blob(ref, filename) + blob = repository&.blob_at(ref, filename) + + raise 'Blob not found' unless blob + raise 'File is not readable' unless blob.readable_text? + + blob + end + + # Resolves the given relative path of file in repository into canonical + # path based on the specified base_path. + # + # Examples: + # + # # File in the same directory as the current path + # resolve_relative_path("users.adoc", "doc/api/README.adoc") + # # => "doc/api/users.adoc" + # + # # File in the same directory, which is also the current path + # resolve_relative_path("users.adoc", "doc/api") + # # => "doc/api/users.adoc" + # + # # Going up one level to a different directory + # resolve_relative_path("../update/7.14-to-8.0.adoc", "doc/api/README.adoc") + # # => "doc/update/7.14-to-8.0.adoc" + # + # Returns a String + def resolve_relative_path(path, base_path) + p = Pathname(base_path) + p = p.dirname unless p.extname.empty? + p += path + + p.cleanpath.to_s + end + + def current_commit + cache[:current_commit] ||= context[:commit] || repository&.commit(ref) + end + + def ref + context[:ref] || context[:project].default_branch + end + + def requested_path + cache[:requested_path] ||= Addressable::URI.unescape(context[:requested_path]) + end + + def uri_type(path) + cache[:uri_types][path] ||= current_commit&.uri_type(path) + end + end + end +end diff --git a/lib/gitlab/asciidoc/syntax_highlighter/html_pipeline_adapter.rb b/lib/gitlab/asciidoc/syntax_highlighter/html_pipeline_adapter.rb new file mode 100644 index 00000000000..5fc3323f0fd --- /dev/null +++ b/lib/gitlab/asciidoc/syntax_highlighter/html_pipeline_adapter.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Asciidoc + module SyntaxHighlighter + class HtmlPipelineAdapter < Asciidoctor::SyntaxHighlighter::Base + register_for 'gitlab-html-pipeline' + + def format(node, lang, opts) + %(<pre><code #{lang ? %[ lang="#{lang}"] : ''}>#{node.content}</code></pre>) + end + end + end + end +end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 4317992d933..82e0c7ceeaa 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -94,6 +94,7 @@ module Gitlab end end + # rubocop:disable Gitlab/RailsLogger def rate_limit!(ip, success:, login:) rate_limiter = Gitlab::Auth::IpRateLimiter.new(ip) return unless rate_limiter.enabled? @@ -114,6 +115,7 @@ module Gitlab end end end + # rubocop:enable Gitlab/RailsLogger private diff --git a/lib/gitlab/auth/activity.rb b/lib/gitlab/auth/activity.rb index 558628b5422..988ff196193 100644 --- a/lib/gitlab/auth/activity.rb +++ b/lib/gitlab/auth/activity.rb @@ -37,14 +37,17 @@ module Gitlab def user_authenticated! self.class.user_authenticated_counter_increment! + + case @opts[:message] + when :two_factor_authenticated + self.class.user_two_factor_authenticated_counter_increment! + end end def user_session_override! self.class.user_session_override_counter_increment! case @opts[:message] - when :two_factor_authenticated - self.class.user_two_factor_authenticated_counter_increment! when :sessionless_sign_in self.class.user_sessionless_authentication_counter_increment! end diff --git a/lib/gitlab/auth/ip_rate_limiter.rb b/lib/gitlab/auth/ip_rate_limiter.rb index 81e616fa20a..0b7055b3256 100644 --- a/lib/gitlab/auth/ip_rate_limiter.rb +++ b/lib/gitlab/auth/ip_rate_limiter.rb @@ -3,6 +3,8 @@ module Gitlab module Auth class IpRateLimiter + include ::Gitlab::Utils::StrongMemoize + attr_reader :ip def initialize(ip) @@ -37,7 +39,20 @@ module Gitlab end def ip_can_be_banned? - config.ip_whitelist.exclude?(ip) + !trusted_ip? + end + + def trusted_ip? + trusted_ips.any? { |netmask| netmask.include?(ip) } + end + + def trusted_ips + strong_memoize(:trusted_ips) do + config.ip_whitelist.map do |proxy| + IPAddr.new(proxy) + rescue IPAddr::InvalidAddressError + end.compact + end end end end diff --git a/lib/gitlab/auth/ldap/adapter.rb b/lib/gitlab/auth/ldap/adapter.rb index 15b9d5ad6e9..bcb0ecccdf9 100644 --- a/lib/gitlab/auth/ldap/adapter.rb +++ b/lib/gitlab/auth/ldap/adapter.rb @@ -55,7 +55,7 @@ module Gitlab response = ldap.get_operation_result unless response.code.zero? - Rails.logger.warn("LDAP search error: #{response.message}") + Rails.logger.warn("LDAP search error: #{response.message}") # rubocop:disable Gitlab/RailsLogger end [] @@ -67,7 +67,7 @@ module Gitlab retries += 1 error_message = connection_error_message(error) - Rails.logger.warn(error_message) + Rails.logger.warn(error_message) # rubocop:disable Gitlab/RailsLogger if retries < MAX_SEARCH_RETRIES renew_connection_adapter diff --git a/lib/gitlab/auth/ldap/config.rb b/lib/gitlab/auth/ldap/config.rb index 47d63eb53cf..354f91306f9 100644 --- a/lib/gitlab/auth/ldap/config.rb +++ b/lib/gitlab/auth/ldap/config.rb @@ -240,7 +240,7 @@ module Gitlab begin custom_options[:cert] = OpenSSL::X509::Certificate.new(custom_options[:cert]) rescue OpenSSL::X509::CertificateError => e - Rails.logger.error "LDAP TLS Options 'cert' is invalid for provider #{provider}: #{e.message}" + Rails.logger.error "LDAP TLS Options 'cert' is invalid for provider #{provider}: #{e.message}" # rubocop:disable Gitlab/RailsLogger end end @@ -248,7 +248,7 @@ module Gitlab begin custom_options[:key] = OpenSSL::PKey.read(custom_options[:key]) rescue OpenSSL::PKey::PKeyError => e - Rails.logger.error "LDAP TLS Options 'key' is invalid for provider #{provider}: #{e.message}" + Rails.logger.error "LDAP TLS Options 'key' is invalid for provider #{provider}: #{e.message}" # rubocop:disable Gitlab/RailsLogger end end diff --git a/lib/gitlab/auth/ldap/person.rb b/lib/gitlab/auth/ldap/person.rb index c1517222956..11a4052a109 100644 --- a/lib/gitlab/auth/ldap/person.rb +++ b/lib/gitlab/auth/ldap/person.rb @@ -45,7 +45,7 @@ module Gitlab def self.normalize_dn(dn) ::Gitlab::Auth::LDAP::DN.new(dn).to_normalized_s rescue ::Gitlab::Auth::LDAP::DN::FormatError => e - Rails.logger.info("Returning original DN \"#{dn}\" due to error during normalization attempt: #{e.message}") + Rails.logger.info("Returning original DN \"#{dn}\" due to error during normalization attempt: #{e.message}") # rubocop:disable Gitlab/RailsLogger dn end @@ -57,13 +57,13 @@ module Gitlab def self.normalize_uid(uid) ::Gitlab::Auth::LDAP::DN.normalize_value(uid) rescue ::Gitlab::Auth::LDAP::DN::FormatError => e - Rails.logger.info("Returning original UID \"#{uid}\" due to error during normalization attempt: #{e.message}") + Rails.logger.info("Returning original UID \"#{uid}\" due to error during normalization attempt: #{e.message}") # rubocop:disable Gitlab/RailsLogger uid end def initialize(entry, provider) - Rails.logger.debug { "Instantiating #{self.class.name} with LDIF:\n#{entry.to_ldif}" } + Rails.logger.debug { "Instantiating #{self.class.name} with LDIF:\n#{entry.to_ldif}" } # rubocop:disable Gitlab/RailsLogger @entry = entry @provider = provider end diff --git a/lib/gitlab/auth/o_auth/auth_hash.rb b/lib/gitlab/auth/o_auth/auth_hash.rb index 72a187377d0..91b9ddc0d00 100644 --- a/lib/gitlab/auth/o_auth/auth_hash.rb +++ b/lib/gitlab/auth/o_auth/auth_hash.rb @@ -60,8 +60,7 @@ module Gitlab def get_info(key) value = info[key] - Gitlab::Utils.force_utf8(value) if value - value + value.is_a?(String) ? Gitlab::Utils.force_utf8(value) : value end def username_and_email diff --git a/lib/gitlab/auth/o_auth/provider.rb b/lib/gitlab/auth/o_auth/provider.rb index 9fdf3324db3..3d44c83736a 100644 --- a/lib/gitlab/auth/o_auth/provider.rb +++ b/lib/gitlab/auth/o_auth/provider.rb @@ -40,6 +40,10 @@ module Gitlab name.to_s.start_with?('ldap') end + def self.ultraauth_provider?(name) + name.to_s.eql?('ultraauth') + end + def self.sync_profile_from_provider?(provider) return true if ldap_provider?(provider) diff --git a/lib/gitlab/auth/user_auth_finders.rb b/lib/gitlab/auth/user_auth_finders.rb index a5efe33bdc6..bba7e2cbb3c 100644 --- a/lib/gitlab/auth/user_auth_finders.rb +++ b/lib/gitlab/auth/user_auth_finders.rb @@ -90,8 +90,8 @@ module Gitlab def find_personal_access_token token = current_request.params[PRIVATE_TOKEN_PARAM].presence || - current_request.env[PRIVATE_TOKEN_HEADER].presence - + current_request.env[PRIVATE_TOKEN_HEADER].presence || + parsed_oauth_token return unless token # Expiration, revocation and scopes are verified in `validate_access_token!` @@ -99,9 +99,12 @@ module Gitlab end def find_oauth_access_token - token = Doorkeeper::OAuth::Token.from_request(current_request, *Doorkeeper.configuration.access_token_methods) + token = parsed_oauth_token return unless token + # PATs with OAuth headers are not handled by OauthAccessToken + return if matches_personal_access_token_length?(token) + # Expiration, revocation and scopes are verified in `validate_access_token!` oauth_token = OauthAccessToken.by_token(token) raise UnauthorizedError unless oauth_token @@ -110,6 +113,14 @@ module Gitlab oauth_token end + def parsed_oauth_token + Doorkeeper::OAuth::Token.from_request(current_request, *Doorkeeper.configuration.access_token_methods) + end + + def matches_personal_access_token_length?(token) + token.length == PersonalAccessToken::TOKEN_LENGTH + end + # Check if the request is GET/HEAD, or if CSRF token is valid. def verified_request? Gitlab::RequestForgeryProtection.verified?(current_request.env) diff --git a/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb b/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb index cb2bdea755c..c912628d0fc 100644 --- a/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb +++ b/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb @@ -9,7 +9,7 @@ module Gitlab end def perform(start_id, stop_id) - Rails.logger.info("Setting commits_count for merge request diffs: #{start_id} - #{stop_id}") + Rails.logger.info("Setting commits_count for merge request diffs: #{start_id} - #{stop_id}") # rubocop:disable Gitlab/RailsLogger update = ' commits_count = ( diff --git a/lib/gitlab/background_migration/archive_legacy_traces.rb b/lib/gitlab/background_migration/archive_legacy_traces.rb index 7ee783b8489..3c26982729d 100644 --- a/lib/gitlab/background_migration/archive_legacy_traces.rb +++ b/lib/gitlab/background_migration/archive_legacy_traces.rb @@ -14,7 +14,7 @@ module Gitlab build.trace.archive! rescue => e - Rails.logger.error "Failed to archive live trace. id: #{build.id} message: #{e.message}" + Rails.logger.error "Failed to archive live trace. id: #{build.id} message: #{e.message}" # rubocop:disable Gitlab/RailsLogger end end end diff --git a/lib/gitlab/background_migration/backfill_project_repositories.rb b/lib/gitlab/background_migration/backfill_project_repositories.rb index c8d83cc1803..1d9aa050041 100644 --- a/lib/gitlab/background_migration/backfill_project_repositories.rb +++ b/lib/gitlab/background_migration/backfill_project_repositories.rb @@ -40,7 +40,7 @@ module Gitlab end def reload! - @shards = Hash[*Shard.all.map { |shard| [shard.name, shard.id] }.flatten] + @shards = Hash[*Shard.all.flat_map { |shard| [shard.name, shard.id] }] end end diff --git a/lib/gitlab/background_migration/calculate_wiki_sizes.rb b/lib/gitlab/background_migration/calculate_wiki_sizes.rb index 886c41a2b9d..e62f5edd0e7 100644 --- a/lib/gitlab/background_migration/calculate_wiki_sizes.rb +++ b/lib/gitlab/background_migration/calculate_wiki_sizes.rb @@ -10,7 +10,7 @@ module Gitlab .includes(project: [:route, :group, namespace: [:owner]]).find_each do |statistics| statistics.refresh!(only: [:wiki_size]) rescue => e - Rails.logger.error "Failed to update wiki statistics. id: #{statistics.id} message: #{e.message}" + Rails.logger.error "Failed to update wiki statistics. id: #{statistics.id} message: #{e.message}" # rubocop:disable Gitlab/RailsLogger end end end diff --git a/lib/gitlab/background_migration/create_fork_network_memberships_range.rb b/lib/gitlab/background_migration/create_fork_network_memberships_range.rb deleted file mode 100644 index ccd1f9b4dba..00000000000 --- a/lib/gitlab/background_migration/create_fork_network_memberships_range.rb +++ /dev/null @@ -1,85 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class CreateForkNetworkMembershipsRange - RESCHEDULE_DELAY = 15 - - class ForkedProjectLink < ActiveRecord::Base - self.table_name = 'forked_project_links' - end - - def perform(start_id, end_id) - log("Creating memberships for forks: #{start_id} - #{end_id}") - - insert_members(start_id, end_id) - - if missing_members?(start_id, end_id) - BackgroundMigrationWorker.perform_in(RESCHEDULE_DELAY, "CreateForkNetworkMembershipsRange", [start_id, end_id]) - end - end - - def insert_members(start_id, end_id) - ActiveRecord::Base.connection.execute <<~INSERT_MEMBERS - INSERT INTO fork_network_members (fork_network_id, project_id, forked_from_project_id) - - SELECT fork_network_members.fork_network_id, - forked_project_links.forked_to_project_id, - forked_project_links.forked_from_project_id - - FROM forked_project_links - - INNER JOIN fork_network_members - ON forked_project_links.forked_from_project_id = fork_network_members.project_id - - WHERE forked_project_links.id BETWEEN #{start_id} AND #{end_id} - AND NOT EXISTS ( - SELECT true - FROM fork_network_members existing_members - WHERE existing_members.project_id = forked_project_links.forked_to_project_id - ) - INSERT_MEMBERS - rescue ActiveRecord::RecordNotUnique => e - # `fork_network_member` was created concurrently in another migration - log(e.message) - end - - def missing_members?(start_id, end_id) - count_sql = <<~MISSING_MEMBERS - SELECT COUNT(*) - - FROM forked_project_links - - WHERE NOT EXISTS ( - SELECT true - FROM fork_network_members - WHERE fork_network_members.project_id = forked_project_links.forked_to_project_id - ) - AND EXISTS ( - SELECT true - FROM projects - WHERE forked_project_links.forked_from_project_id = projects.id - ) - AND NOT EXISTS ( - SELECT true - FROM forked_project_links AS parent_links - WHERE parent_links.forked_to_project_id = forked_project_links.forked_from_project_id - AND NOT EXISTS ( - SELECT true - FROM projects - WHERE parent_links.forked_from_project_id = projects.id - ) - ) - AND forked_project_links.id BETWEEN #{start_id} AND #{end_id} - MISSING_MEMBERS - - ForkedProjectLink.count_by_sql(count_sql) > 0 - end - - def log(message) - Rails.logger.info("#{self.class.name} - #{message}") - end - end - end -end diff --git a/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb b/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb deleted file mode 100644 index da8265a3a5f..00000000000 --- a/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb +++ /dev/null @@ -1,56 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -class Gitlab::BackgroundMigration::CreateGpgKeySubkeysFromGpgKeys - class GpgKey < ActiveRecord::Base - self.table_name = 'gpg_keys' - - include EachBatch - include ShaAttribute - - sha_attribute :primary_keyid - sha_attribute :fingerprint - - has_many :subkeys, class_name: 'GpgKeySubkey' - end - - class GpgKeySubkey < ActiveRecord::Base - self.table_name = 'gpg_key_subkeys' - - include ShaAttribute - - sha_attribute :keyid - sha_attribute :fingerprint - end - - def perform(gpg_key_id) - gpg_key = GpgKey.find_by(id: gpg_key_id) - - return if gpg_key.nil? - return if gpg_key.subkeys.any? - - create_subkeys(gpg_key) - update_signatures(gpg_key) - end - - private - - def create_subkeys(gpg_key) - gpg_subkeys = Gitlab::Gpg.subkeys_from_key(gpg_key.key) - - gpg_subkeys[gpg_key.primary_keyid.upcase]&.each do |subkey_data| - gpg_key.subkeys.build(keyid: subkey_data[:keyid], fingerprint: subkey_data[:fingerprint]) - end - - # Improve latency by doing all INSERTs in a single call - GpgKey.transaction do - gpg_key.save! - end - end - - def update_signatures(gpg_key) - return unless gpg_key.subkeys.exists? - - InvalidGpgSignatureUpdateWorker.perform_async(gpg_key.id) - end -end diff --git a/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb b/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb deleted file mode 100644 index 21b626dde56..00000000000 --- a/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class DeleteConflictingRedirectRoutesRange - def perform(start_id, end_id) - # No-op. - # See https://gitlab.com/gitlab-com/infrastructure/issues/3460#note_53223252 - end - end - end -end diff --git a/lib/gitlab/background_migration/delete_diff_files.rb b/lib/gitlab/background_migration/delete_diff_files.rb deleted file mode 100644 index 664ead1af44..00000000000 --- a/lib/gitlab/background_migration/delete_diff_files.rb +++ /dev/null @@ -1,81 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class DeleteDiffFiles - class MergeRequestDiff < ActiveRecord::Base - self.table_name = 'merge_request_diffs' - - belongs_to :merge_request - has_many :merge_request_diff_files - end - - class MergeRequestDiffFile < ActiveRecord::Base - self.table_name = 'merge_request_diff_files' - end - - DEAD_TUPLES_THRESHOLD = 50_000 - VACUUM_WAIT_TIME = 5.minutes - - def perform(ids) - @ids = ids - - # We should reschedule until deadtuples get in a desirable - # state (e.g. < 50_000). That may take more than one reschedule. - # - if should_wait_deadtuple_vacuum? - reschedule - return - end - - prune_diff_files - end - - def should_wait_deadtuple_vacuum? - return false unless Gitlab::Database.postgresql? - - diff_files_dead_tuples_count >= DEAD_TUPLES_THRESHOLD - end - - private - - def reschedule - BackgroundMigrationWorker.perform_in(VACUUM_WAIT_TIME, self.class.name.demodulize, [@ids]) - end - - def diffs_collection - MergeRequestDiff.where(id: @ids) - end - - def diff_files_dead_tuples_count - dead_tuple = - execute_statement("SELECT n_dead_tup FROM pg_stat_all_tables "\ - "WHERE relname = 'merge_request_diff_files'")[0] - - dead_tuple&.fetch('n_dead_tup', 0).to_i - end - - def prune_diff_files - removed = 0 - updated = 0 - - MergeRequestDiff.transaction do - updated = diffs_collection.update_all(state: 'without_files') - removed = MergeRequestDiffFile.where(merge_request_diff_id: @ids).delete_all - end - - log_info("Removed #{removed} merge_request_diff_files rows, "\ - "updated #{updated} merge_request_diffs rows") - end - - def execute_statement(sql) - ActiveRecord::Base.connection.execute(sql) - end - - def log_info(message) - Rails.logger.info("BackgroundMigration::DeleteDiffFiles - #{message}") - end - end - end -end diff --git a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb deleted file mode 100644 index 58df74cfa9b..00000000000 --- a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb +++ /dev/null @@ -1,149 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Metrics/MethodLength -# rubocop:disable Metrics/AbcSize -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class DeserializeMergeRequestDiffsAndCommits - attr_reader :diff_ids, :commit_rows, :file_rows - - class Error < StandardError - def backtrace - cause.backtrace - end - end - - class MergeRequestDiff < ActiveRecord::Base - self.table_name = 'merge_request_diffs' - end - - BUFFER_ROWS = 1000 - DIFF_FILE_BUFFER_ROWS = 100 - - def perform(start_id, stop_id) - merge_request_diffs = MergeRequestDiff - .select(:id, :st_commits, :st_diffs) - .where('st_commits IS NOT NULL OR st_diffs IS NOT NULL') - .where(id: start_id..stop_id) - - reset_buffers! - - merge_request_diffs.each do |merge_request_diff| - commits, files = single_diff_rows(merge_request_diff) - - diff_ids << merge_request_diff.id - commit_rows.concat(commits) - file_rows.concat(files) - - if diff_ids.length > BUFFER_ROWS || - commit_rows.length > BUFFER_ROWS || - file_rows.length > DIFF_FILE_BUFFER_ROWS - - flush_buffers! - end - end - - flush_buffers! - rescue => e - Rails.logger.info("#{self.class.name}: failed for IDs #{merge_request_diffs.map(&:id)} with #{e.class.name}") - - raise Error.new(e.inspect) - end - - private - - def reset_buffers! - @diff_ids = [] - @commit_rows = [] - @file_rows = [] - end - - def flush_buffers! - if diff_ids.any? - commit_rows.each_slice(BUFFER_ROWS).each do |commit_rows_slice| - bulk_insert('merge_request_diff_commits', commit_rows_slice) - end - - file_rows.each_slice(DIFF_FILE_BUFFER_ROWS).each do |file_rows_slice| - bulk_insert('merge_request_diff_files', file_rows_slice) - end - - MergeRequestDiff.where(id: diff_ids).update_all(st_commits: nil, st_diffs: nil) - end - - reset_buffers! - end - - def bulk_insert(table, rows) - Gitlab::Database.bulk_insert(table, rows) - rescue ActiveRecord::RecordNotUnique - ids = rows.map { |row| row[:merge_request_diff_id] }.uniq.sort - - Rails.logger.info("#{self.class.name}: rows inserted twice for IDs #{ids}") - end - - def single_diff_rows(merge_request_diff) - sha_attribute = Gitlab::Database::ShaAttribute.new - commits = YAML.load(merge_request_diff.st_commits) rescue [] - commits ||= [] - - commit_rows = commits.map.with_index do |commit, index| - commit_hash = commit.to_hash.with_indifferent_access.except(:parent_ids) - sha = commit_hash.delete(:id) - - commit_hash.merge( - merge_request_diff_id: merge_request_diff.id, - relative_order: index, - sha: sha_attribute.serialize(sha) - ) - end - - diffs = YAML.load(merge_request_diff.st_diffs) rescue [] - diffs = [] unless valid_raw_diffs?(diffs) - - file_rows = diffs.map.with_index do |diff, index| - diff_hash = diff.to_hash.with_indifferent_access.merge( - binary: false, - merge_request_diff_id: merge_request_diff.id, - relative_order: index - ) - - diff_hash.tap do |hash| - diff_text = hash[:diff] - - hash[:too_large] = !!hash[:too_large] - - hash[:a_mode] ||= guess_mode(hash[:new_file], hash[:diff]) - hash[:b_mode] ||= guess_mode(hash[:deleted_file], hash[:diff]) - - # Compatibility with old diffs created with Psych. - if diff_text.encoding == Encoding::BINARY && !diff_text.ascii_only? - hash[:binary] = true - hash[:diff] = [diff_text].pack('m0') - end - end - end - - [commit_rows, file_rows] - end - - # This doesn't have to be 100% accurate, because it's only used for - # display - it won't change file modes in the repository. Submodules are - # created as 600, regular files as 644. - def guess_mode(file_missing, diff) - return '0' if file_missing - - diff.include?('Subproject commit') ? '160000' : '100644' - end - - # Unlike MergeRequestDiff#valid_raw_diff?, don't count Rugged objects as - # valid, because we don't render them usefully anyway. - def valid_raw_diffs?(diffs) - return false unless diffs.respond_to?(:each) - - diffs.all? { |diff| diff.is_a?(Hash) } - end - end - end -end diff --git a/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb b/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb index 0e93b2cb2fa..4016b807f21 100644 --- a/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb +++ b/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb @@ -19,20 +19,13 @@ module Gitlab def perform(start_id, stop_id) PagesDomain.where(id: start_id..stop_id).find_each do |domain| - if Gitlab::Database.mysql? - domain.update_columns( - certificate_valid_not_before: domain.x509&.not_before, - certificate_valid_not_after: domain.x509&.not_after - ) - else - # for some reason activerecord doesn't append timezone, iso8601 forces this - domain.update_columns( - certificate_valid_not_before: domain.x509&.not_before&.iso8601, - certificate_valid_not_after: domain.x509&.not_after&.iso8601 - ) - end + # for some reason activerecord doesn't append timezone, iso8601 forces this + domain.update_columns( + certificate_valid_not_before: domain.x509&.not_before&.iso8601, + certificate_valid_not_after: domain.x509&.not_after&.iso8601 + ) rescue => e - Rails.logger.error "Failed to update pages domain certificate valid time. id: #{domain.id}, message: #{e.message}" + Rails.logger.error "Failed to update pages domain certificate valid time. id: #{domain.id}, message: #{e.message}" # rubocop:disable Gitlab/RailsLogger end end end diff --git a/lib/gitlab/background_migration/fix_cross_project_label_links.rb b/lib/gitlab/background_migration/fix_cross_project_label_links.rb index bf5d7f5f322..20a98c8e141 100644 --- a/lib/gitlab/background_migration/fix_cross_project_label_links.rb +++ b/lib/gitlab/background_migration/fix_cross_project_label_links.rb @@ -108,7 +108,7 @@ module Gitlab next unless matching_label - Rails.logger.info "#{resource.class.name.demodulize} #{resource.id}: replacing #{label.label_id} with #{matching_label.id}" + Rails.logger.info "#{resource.class.name.demodulize} #{resource.id}: replacing #{label.label_id} with #{matching_label.id}" # rubocop:disable Gitlab/RailsLogger LabelLink.update(label.label_link_id, label_id: matching_label.id) end end diff --git a/lib/gitlab/background_migration/fix_pages_access_level.rb b/lib/gitlab/background_migration/fix_pages_access_level.rb new file mode 100644 index 00000000000..0d49f3dd8c5 --- /dev/null +++ b/lib/gitlab/background_migration/fix_pages_access_level.rb @@ -0,0 +1,128 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # corrects stored pages access level on db depending on project visibility + class FixPagesAccessLevel + # Copy routable here to avoid relying on application logic + module Routable + def build_full_path + if parent && path + parent.build_full_path + '/' + path + else + path + end + end + end + + # Namespace + class Namespace < ApplicationRecord + self.table_name = 'namespaces' + self.inheritance_column = :_type_disabled + + include Routable + + belongs_to :parent, class_name: "Namespace" + end + + # Project + class Project < ActiveRecord::Base + self.table_name = 'projects' + self.inheritance_column = :_type_disabled + + include Routable + + belongs_to :namespace + alias_method :parent, :namespace + alias_attribute :parent_id, :namespace_id + + PRIVATE = 0 + INTERNAL = 10 + PUBLIC = 20 + + def pages_deployed? + Dir.exist?(public_pages_path) + end + + def public_pages_path + File.join(pages_path, 'public') + end + + def pages_path + # TODO: when we migrate Pages to work with new storage types, change here to use disk_path + File.join(Settings.pages.path, build_full_path) + end + end + + # ProjectFeature + class ProjectFeature < ActiveRecord::Base + include ::EachBatch + + self.table_name = 'project_features' + + belongs_to :project + + PRIVATE = 10 + ENABLED = 20 + PUBLIC = 30 + end + + def perform(start_id, stop_id) + fix_public_access_level(start_id, stop_id) + + make_internal_projects_public(start_id, stop_id) + + fix_private_access_level(start_id, stop_id) + end + + private + + def access_control_is_enabled + @access_control_is_enabled = Gitlab.config.pages.access_control + end + + # Public projects are allowed to have only enabled pages_access_level + # which is equivalent to public + def fix_public_access_level(start_id, stop_id) + project_features(start_id, stop_id, ProjectFeature::PUBLIC, Project::PUBLIC).each_batch do |features| + features.update_all(pages_access_level: ProjectFeature::ENABLED) + end + end + + # If access control is disabled and project has pages deployed + # project will become unavailable when access control will become enabled + # we make these projects public to avoid negative surprise to user + def make_internal_projects_public(start_id, stop_id) + return if access_control_is_enabled + + project_features(start_id, stop_id, ProjectFeature::ENABLED, Project::INTERNAL).find_each do |project_feature| + next unless project_feature.project.pages_deployed? + + project_feature.update(pages_access_level: ProjectFeature::PUBLIC) + end + end + + # Private projects are not allowed to have enabled access level, only `private` and `public` + # If access control is enabled, these projects currently behave as if the have `private` pages_access_level + # if access control is disabled, these projects currently behave as if the have `public` pages_access_level + # so we preserve this behaviour for projects with pages already deployed + # for project without pages we always set `private` access_level + def fix_private_access_level(start_id, stop_id) + project_features(start_id, stop_id, ProjectFeature::ENABLED, Project::PRIVATE).find_each do |project_feature| + if access_control_is_enabled + project_feature.update!(pages_access_level: ProjectFeature::PRIVATE) + else + fixed_access_level = project_feature.project.pages_deployed? ? ProjectFeature::PUBLIC : ProjectFeature::PRIVATE + project_feature.update!(pages_access_level: fixed_access_level) + end + end + end + + def project_features(start_id, stop_id, pages_access_level, project_visibility_level) + ProjectFeature.where(id: start_id..stop_id).joins(:project) + .where(pages_access_level: pages_access_level) + .where(projects: { visibility_level: project_visibility_level }) + end + end + end +end diff --git a/lib/gitlab/background_migration/fix_user_namespace_names.rb b/lib/gitlab/background_migration/fix_user_namespace_names.rb new file mode 100644 index 00000000000..1a207121be0 --- /dev/null +++ b/lib/gitlab/background_migration/fix_user_namespace_names.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This migration fixes the namespaces.name for all user-namespaces that have names + # that aren't equal to the users name. + # Then it uses the updated names of the namespaces to update the associated routes + # For more info see https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/23272 + class FixUserNamespaceNames + def perform(from_id, to_id) + fix_namespace_names(from_id, to_id) + fix_namespace_route_names(from_id, to_id) + end + + def fix_namespace_names(from_id, to_id) + ActiveRecord::Base.connection.execute <<~UPDATE_NAMESPACES + WITH namespaces_to_update AS ( + SELECT + namespaces.id, + users.name AS correct_name + FROM + namespaces + INNER JOIN users ON namespaces.owner_id = users.id + WHERE + namespaces.type IS NULL + AND namespaces.id BETWEEN #{from_id} AND #{to_id} + AND namespaces.name != users.name + ) + UPDATE + namespaces + SET + name = correct_name + FROM + namespaces_to_update + WHERE + namespaces.id = namespaces_to_update.id + UPDATE_NAMESPACES + end + + def fix_namespace_route_names(from_id, to_id) + ActiveRecord::Base.connection.execute <<~ROUTES_UPDATE + WITH routes_to_update AS ( + SELECT + routes.id, + users.name AS correct_name + FROM + routes + INNER JOIN namespaces ON routes.source_id = namespaces.id + INNER JOIN users ON namespaces.owner_id = users.id + WHERE + namespaces.type IS NULL + AND routes.source_type = 'Namespace' + AND namespaces.id BETWEEN #{from_id} AND #{to_id} + AND (routes.name != users.name OR routes.name IS NULL) + ) + UPDATE + routes + SET + name = correct_name + FROM + routes_to_update + WHERE + routes_to_update.id = routes.id + ROUTES_UPDATE + end + end + end +end diff --git a/lib/gitlab/background_migration/fix_user_project_route_names.rb b/lib/gitlab/background_migration/fix_user_project_route_names.rb new file mode 100644 index 00000000000..b84ff32e712 --- /dev/null +++ b/lib/gitlab/background_migration/fix_user_project_route_names.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This migration fixes the routes.name for all user-projects that have names + # that don't start with the users name. + # For more info see https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/23272 + class FixUserProjectRouteNames + def perform(from_id, to_id) + ActiveRecord::Base.connection.execute <<~ROUTES_UPDATE + WITH routes_to_update AS ( + SELECT + routes.id, + users.name || ' / ' || projects.name AS correct_name + FROM + routes + INNER JOIN projects ON routes.source_id = projects.id + INNER JOIN namespaces ON projects.namespace_id = namespaces.id + INNER JOIN users ON namespaces.owner_id = users.id + WHERE + routes.source_type = 'Project' + AND routes.id BETWEEN #{from_id} AND #{to_id} + AND namespaces.type IS NULL + AND (routes.name NOT LIKE users.name || '%' OR routes.name IS NULL) + ) + UPDATE + routes + SET + name = routes_to_update.correct_name + FROM + routes_to_update + WHERE + routes_to_update.id = routes.id + ROUTES_UPDATE + end + end + end +end diff --git a/lib/gitlab/background_migration/legacy_upload_mover.rb b/lib/gitlab/background_migration/legacy_upload_mover.rb new file mode 100644 index 00000000000..051c1176edb --- /dev/null +++ b/lib/gitlab/background_migration/legacy_upload_mover.rb @@ -0,0 +1,140 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class takes a legacy upload and migrates it to the correct location + class LegacyUploadMover + include Gitlab::Utils::StrongMemoize + + attr_reader :upload, :project, :note + attr_accessor :logger + + def initialize(upload) + @upload = upload + @note = Note.find_by(id: upload.model_id) + @project = note&.project + @logger = Gitlab::BackgroundMigration::Logger.build + end + + def execute + return unless upload + + if !project + # if we don't have models associated with the upload we can not move it + warn('Deleting upload due to model not found.') + + destroy_legacy_upload + elsif note.is_a?(LegacyDiffNote) + return unless move_legacy_diff_file + + migrate_upload + elsif !legacy_file_exists? + warn('Deleting upload due to file not found.') + destroy_legacy_upload + else + migrate_upload + end + end + + private + + def migrate_upload + return unless copy_upload_to_project + + add_upload_link_to_note_text + destroy_legacy_file + destroy_legacy_upload + end + + # we should proceed and log whenever one upload copy fails, no matter the reasons + # rubocop: disable Lint/RescueException + def copy_upload_to_project + @uploader = FileUploader.copy_to(legacy_file_uploader, project) + + logger.info( + message: 'MigrateLegacyUploads: File copied successfully', + old_path: legacy_file_uploader.file.path, new_path: @uploader.file.path + ) + true + rescue Exception => e + warn( + 'File could not be copied to project uploads', + file_path: legacy_file_uploader.file.path, error: e.message + ) + false + end + # rubocop: enable Lint/RescueException + + def destroy_legacy_upload + if note + note.remove_attachment = true + note.save + end + + if upload.destroy + logger.info(message: 'MigrateLegacyUploads: Upload was destroyed.', upload: upload.inspect) + else + warn('MigrateLegacyUploads: Upload destroy failed.') + end + end + + def destroy_legacy_file + legacy_file_uploader.file.delete + end + + def add_upload_link_to_note_text + new_text = "#{note.note} \n #{@uploader.markdown_link}" + # Bypass validations because old data may have invalid + # noteable values. If we fail hard here, we may kill the + # entire background migration, which affects a range of notes. + note.update_attribute(:note, new_text) + end + + def legacy_file_uploader + strong_memoize(:legacy_file_uploader) do + uploader = upload.build_uploader + uploader.retrieve_from_store!(File.basename(upload.path)) + uploader + end + end + + def legacy_file_exists? + legacy_file_uploader.file.exists? + end + + # we should proceed and log whenever one upload copy fails, no matter the reasons + # rubocop: disable Lint/RescueException + def move_legacy_diff_file + old_path = upload.absolute_path + old_path_sub = '-/system/note/attachment' + + if !File.exist?(old_path) || !old_path.include?(old_path_sub) + log_legacy_diff_note_problem(old_path) + return false + end + + new_path = upload.absolute_path.sub(old_path_sub, '-/system/legacy_diff_note/attachment') + new_dir = File.dirname(new_path) + FileUtils.mkdir_p(new_dir) + + FileUtils.mv(old_path, new_path) + rescue Exception => e + log_legacy_diff_note_problem(old_path, new_path, e) + false + end + + def warn(message, params = {}) + logger.warn( + params.merge(message: "MigrateLegacyUploads: #{message}", upload: upload.inspect) + ) + end + + def log_legacy_diff_note_problem(old_path, new_path = nil, error = nil) + warn('LegacyDiffNote upload could not be moved to a new path', + old_path: old_path, new_path: new_path, error: error&.message + ) + end + # rubocop: enable Lint/RescueException + end + end +end diff --git a/lib/gitlab/background_migration/legacy_uploads_migrator.rb b/lib/gitlab/background_migration/legacy_uploads_migrator.rb new file mode 100644 index 00000000000..a9d38a27e0c --- /dev/null +++ b/lib/gitlab/background_migration/legacy_uploads_migrator.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This migration takes all legacy uploads (that were uploaded using AttachmentUploader) + # and migrate them to the new (FileUploader) location (=under projects). + # + # We have dependencies (uploaders) in this migration because extracting code would add a lot of complexity + # and possible errors could appear as the logic in the uploaders is not trivial. + # + # This migration will be removed in 13.0 in order to get rid of a migration that depends on + # the application code. + class LegacyUploadsMigrator + include Database::MigrationHelpers + + def perform(start_id, end_id) + Upload.where(id: start_id..end_id, uploader: 'AttachmentUploader').find_each do |upload| + LegacyUploadMover.new(upload).execute + end + end + end + end +end diff --git a/lib/gitlab/background_migration/logger.rb b/lib/gitlab/background_migration/logger.rb new file mode 100644 index 00000000000..4ea89771eff --- /dev/null +++ b/lib/gitlab/background_migration/logger.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Logger that can be used for migrations logging + class Logger < ::Gitlab::JsonLogger + def self.file_name_noext + 'migrations' + end + end + end +end diff --git a/lib/gitlab/background_migration/merge_request_assignees_migration_progress_check.rb b/lib/gitlab/background_migration/merge_request_assignees_migration_progress_check.rb new file mode 100644 index 00000000000..e948cedaad5 --- /dev/null +++ b/lib/gitlab/background_migration/merge_request_assignees_migration_progress_check.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Style/Documentation + class MergeRequestAssigneesMigrationProgressCheck + include Gitlab::Utils::StrongMemoize + + RESCHEDULE_DELAY = 3.hours + WORKER = 'PopulateMergeRequestAssigneesTable'.freeze + DeadJobsError = Class.new(StandardError) + + def perform + raise DeadJobsError, "Only dead background jobs in the queue for #{WORKER}" if !ongoing? && dead_jobs? + + if ongoing? + BackgroundMigrationWorker.perform_in(RESCHEDULE_DELAY, self.class.name) + else + Feature.enable(:multiple_merge_request_assignees) + end + end + + private + + def dead_jobs? + strong_memoize(:dead_jobs) do + migration_klass.dead_jobs?(WORKER) + end + end + + def ongoing? + strong_memoize(:ongoing) do + migration_klass.exists?(WORKER) || migration_klass.retrying_jobs?(WORKER) + end + end + + def migration_klass + Gitlab::BackgroundMigration + end + end + # rubocop: enable Style/Documentation + end +end diff --git a/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb b/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb deleted file mode 100644 index 42fcaa87e66..00000000000 --- a/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb +++ /dev/null @@ -1,179 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - # Class that migrates events for the new push event payloads setup. All - # events are copied to a shadow table, and push events will also have a row - # created in the push_event_payloads table. - class MigrateEventsToPushEventPayloads - class Event < ActiveRecord::Base - self.table_name = 'events' - - serialize :data - - BLANK_REF = ('0' * 40).freeze - TAG_REF_PREFIX = 'refs/tags/'.freeze - MAX_INDEX = 69 - PUSHED = 5 - - def push_event? - action == PUSHED && data.present? - end - - def commit_title - commit = commits.last - - return unless commit && commit[:message] - - index = commit[:message].index("\n") - message = index ? commit[:message][0..index] : commit[:message] - - message.strip.truncate(70) - end - - def commit_from_sha - if create? - nil - else - data[:before] - end - end - - def commit_to_sha - if remove? - nil - else - data[:after] - end - end - - def data - super || {} - end - - def commits - data[:commits] || [] - end - - def commit_count - data[:total_commits_count] || 0 - end - - def ref - data[:ref] - end - - def trimmed_ref_name - if ref_type == :tag - ref[10..-1] - else - ref[11..-1] - end - end - - def create? - data[:before] == BLANK_REF - end - - def remove? - data[:after] == BLANK_REF - end - - def push_action - if create? - :created - elsif remove? - :removed - else - :pushed - end - end - - def ref_type - if ref.start_with?(TAG_REF_PREFIX) - :tag - else - :branch - end - end - end - - class EventForMigration < ActiveRecord::Base - self.table_name = 'events_for_migration' - end - - class PushEventPayload < ActiveRecord::Base - self.table_name = 'push_event_payloads' - - enum action: { - created: 0, - removed: 1, - pushed: 2 - } - - enum ref_type: { - branch: 0, - tag: 1 - } - end - - # start_id - The start ID of the range of events to process - # end_id - The end ID of the range to process. - def perform(start_id, end_id) - return unless migrate? - - find_events(start_id, end_id).each { |event| process_event(event) } - end - - def process_event(event) - ActiveRecord::Base.transaction do - replicate_event(event) - create_push_event_payload(event) if event.push_event? - end - rescue ActiveRecord::InvalidForeignKey => e - # A foreign key error means the associated event was removed. In this - # case we'll just skip migrating the event. - Rails.logger.error("Unable to migrate event #{event.id}: #{e}") - end - - def replicate_event(event) - new_attributes = event.attributes - .with_indifferent_access.except(:title, :data) - - EventForMigration.create!(new_attributes) - end - - def create_push_event_payload(event) - commit_from = pack(event.commit_from_sha) - commit_to = pack(event.commit_to_sha) - - PushEventPayload.create!( - event_id: event.id, - commit_count: event.commit_count, - ref_type: event.ref_type, - action: event.push_action, - commit_from: commit_from, - commit_to: commit_to, - ref: event.trimmed_ref_name, - commit_title: event.commit_title - ) - end - - def find_events(start_id, end_id) - Event - .where('NOT EXISTS (SELECT true FROM events_for_migration WHERE events_for_migration.id = events.id)') - .where(id: start_id..end_id) - end - - def migrate? - Event.table_exists? && PushEventPayload.table_exists? && - EventForMigration.table_exists? - end - - def pack(value) - value ? [value].pack('H*') : nil - end - end - end -end diff --git a/lib/gitlab/background_migration/migrate_legacy_artifacts.rb b/lib/gitlab/background_migration/migrate_legacy_artifacts.rb index 5cd638083b0..4377ec2987c 100644 --- a/lib/gitlab/background_migration/migrate_legacy_artifacts.rb +++ b/lib/gitlab/background_migration/migrate_legacy_artifacts.rb @@ -39,10 +39,10 @@ module Gitlab SELECT project_id, id, - artifacts_expire_at, + artifacts_expire_at #{add_missing_db_timezone}, #{LEGACY_PATH_FILE_LOCATION}, - created_at, - created_at, + created_at #{add_missing_db_timezone}, + created_at #{add_missing_db_timezone}, artifacts_file, artifacts_size, COALESCE(artifacts_file_store, #{FILE_LOCAL_STORE}), @@ -81,10 +81,10 @@ module Gitlab SELECT project_id, id, - artifacts_expire_at, + artifacts_expire_at #{add_missing_db_timezone}, #{LEGACY_PATH_FILE_LOCATION}, - created_at, - created_at, + created_at #{add_missing_db_timezone}, + created_at #{add_missing_db_timezone}, artifacts_metadata, NULL, COALESCE(artifacts_metadata_store, #{FILE_LOCAL_STORE}), @@ -121,6 +121,12 @@ module Gitlab AND artifacts_file <> '' SQL end + + def add_missing_db_timezone + return '' unless Gitlab::Database.postgresql? + + 'at time zone \'UTC\'' + end end end end diff --git a/lib/gitlab/background_migration/migrate_null_private_profile_to_false.rb b/lib/gitlab/background_migration/migrate_null_private_profile_to_false.rb new file mode 100644 index 00000000000..32ed6a2756d --- /dev/null +++ b/lib/gitlab/background_migration/migrate_null_private_profile_to_false.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class is responsible for migrating a range of users with private_profile == NULL to false + class MigrateNullPrivateProfileToFalse + # Temporary AR class for users + class User < ActiveRecord::Base + self.table_name = 'users' + end + + def perform(start_id, stop_id) + User.where(private_profile: nil, id: start_id..stop_id).update_all(private_profile: false) + end + end + end +end diff --git a/lib/gitlab/background_migration/migrate_stage_index.rb b/lib/gitlab/background_migration/migrate_stage_index.rb index f921233460d..55608529cee 100644 --- a/lib/gitlab/background_migration/migrate_stage_index.rb +++ b/lib/gitlab/background_migration/migrate_stage_index.rb @@ -13,34 +13,22 @@ module Gitlab private def migrate_stage_index_sql(start_id, stop_id) - if Gitlab::Database.postgresql? - <<~SQL - WITH freqs AS ( - SELECT stage_id, stage_idx, COUNT(*) AS freq FROM ci_builds - WHERE stage_id BETWEEN #{start_id} AND #{stop_id} - AND stage_idx IS NOT NULL - GROUP BY stage_id, stage_idx - ), indexes AS ( - SELECT DISTINCT stage_id, first_value(stage_idx) - OVER (PARTITION BY stage_id ORDER BY freq DESC) AS index - FROM freqs - ) + <<~SQL + WITH freqs AS ( + SELECT stage_id, stage_idx, COUNT(*) AS freq FROM ci_builds + WHERE stage_id BETWEEN #{start_id} AND #{stop_id} + AND stage_idx IS NOT NULL + GROUP BY stage_id, stage_idx + ), indexes AS ( + SELECT DISTINCT stage_id, first_value(stage_idx) + OVER (PARTITION BY stage_id ORDER BY freq DESC) AS index + FROM freqs + ) - UPDATE ci_stages SET position = indexes.index - FROM indexes WHERE indexes.stage_id = ci_stages.id - AND ci_stages.position IS NULL; - SQL - else - <<~SQL - UPDATE ci_stages - SET position = - (SELECT stage_idx FROM ci_builds - WHERE ci_builds.stage_id = ci_stages.id - GROUP BY ci_builds.stage_idx ORDER BY COUNT(*) DESC LIMIT 1) - WHERE ci_stages.id BETWEEN #{start_id} AND #{stop_id} - AND ci_stages.position IS NULL - SQL - end + UPDATE ci_stages SET position = indexes.index + FROM indexes WHERE indexes.stage_id = ci_stages.id + AND ci_stages.position IS NULL; + SQL end end end diff --git a/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb b/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb deleted file mode 100644 index ef50fe4adb1..00000000000 --- a/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class MigrateSystemUploadsToNewFolder - include Gitlab::Database::MigrationHelpers - attr_reader :old_folder, :new_folder - - class Upload < ActiveRecord::Base - self.table_name = 'uploads' - include EachBatch - end - - def perform(old_folder, new_folder) - replace_sql = replace_sql(uploads[:path], old_folder, new_folder) - affected_uploads = Upload.where(uploads[:path].matches("#{old_folder}%")) - - affected_uploads.each_batch do |batch| - batch.update_all("path = #{replace_sql}") - end - end - - def uploads - Arel::Table.new('uploads') - end - end - end -end diff --git a/lib/gitlab/background_migration/move_personal_snippet_files.rb b/lib/gitlab/background_migration/move_personal_snippet_files.rb deleted file mode 100644 index 5b2b2af718a..00000000000 --- a/lib/gitlab/background_migration/move_personal_snippet_files.rb +++ /dev/null @@ -1,82 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class MovePersonalSnippetFiles - delegate :select_all, :execute, :quote_string, to: :connection - - def perform(relative_source, relative_destination) - @source_relative_location = relative_source - @destination_relative_location = relative_destination - - move_personal_snippet_files - end - - def move_personal_snippet_files - query = "SELECT uploads.path, uploads.model_id FROM uploads "\ - "INNER JOIN snippets ON snippets.id = uploads.model_id WHERE uploader = 'PersonalFileUploader'" - select_all(query).each do |upload| - secret = upload['path'].split('/')[0] - file_name = upload['path'].split('/')[1] - - move_file(upload['model_id'], secret, file_name) - update_markdown(upload['model_id'], secret, file_name) - end - end - - def move_file(snippet_id, secret, file_name) - source_dir = File.join(base_directory, @source_relative_location, snippet_id.to_s, secret) - destination_dir = File.join(base_directory, @destination_relative_location, snippet_id.to_s, secret) - - source_file_path = File.join(source_dir, file_name) - destination_file_path = File.join(destination_dir, file_name) - - unless File.exist?(source_file_path) - say "Source file `#{source_file_path}` doesn't exist. Skipping." - return - end - - say "Moving file #{source_file_path} -> #{destination_file_path}" - - FileUtils.mkdir_p(destination_dir) - FileUtils.move(source_file_path, destination_file_path) - end - - def update_markdown(snippet_id, secret, file_name) - source_markdown_path = File.join(@source_relative_location, snippet_id.to_s, secret, file_name) - destination_markdown_path = File.join(@destination_relative_location, snippet_id.to_s, secret, file_name) - - source_markdown = "](#{source_markdown_path})" - destination_markdown = "](#{destination_markdown_path})" - quoted_source = quote_string(source_markdown) - quoted_destination = quote_string(destination_markdown) - - execute("UPDATE snippets "\ - "SET description = replace(snippets.description, '#{quoted_source}', '#{quoted_destination}'), description_html = NULL "\ - "WHERE id = #{snippet_id}") - - query = "SELECT id, note FROM notes WHERE noteable_id = #{snippet_id} "\ - "AND noteable_type = 'Snippet' AND note IS NOT NULL" - select_all(query).each do |note| - text = note['note'].gsub(source_markdown, destination_markdown) - quoted_text = quote_string(text) - - execute("UPDATE notes SET note = '#{quoted_text}', note_html = NULL WHERE id = #{note['id']}") - end - end - - def base_directory - File.join(Rails.root, 'public') - end - - def connection - ActiveRecord::Base.connection - end - - def say(message) - Rails.logger.debug(message) - end - end - end -end diff --git a/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb b/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb deleted file mode 100644 index 48aa369705f..00000000000 --- a/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb +++ /dev/null @@ -1,319 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Metrics/MethodLength -# rubocop:disable Metrics/ClassLength -# rubocop:disable Metrics/BlockLength -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class NormalizeLdapExternUidsRange - class Identity < ActiveRecord::Base - self.table_name = 'identities' - end - - # Copied this class to make this migration resilient to future code changes. - # And if the normalize behavior is changed in the future, it must be - # accompanied by another migration. - module Gitlab - module Auth - module LDAP - class DN - FormatError = Class.new(StandardError) - MalformedError = Class.new(FormatError) - UnsupportedError = Class.new(FormatError) - - def self.normalize_value(given_value) - dummy_dn = "placeholder=#{given_value}" - normalized_dn = new(*dummy_dn).to_normalized_s - normalized_dn.sub(/\Aplaceholder=/, '') - end - - ## - # Initialize a DN, escaping as required. Pass in attributes in name/value - # pairs. If there is a left over argument, it will be appended to the dn - # without escaping (useful for a base string). - # - # Most uses of this class will be to escape a DN, rather than to parse it, - # so storing the dn as an escaped String and parsing parts as required - # with a state machine seems sensible. - def initialize(*args) - if args.length > 1 - initialize_array(args) - else - initialize_string(args[0]) - end - end - - ## - # Parse a DN into key value pairs using ASN from - # http://tools.ietf.org/html/rfc2253 section 3. - # rubocop:disable Metrics/AbcSize - # rubocop:disable Metrics/CyclomaticComplexity - # rubocop:disable Metrics/PerceivedComplexity - def each_pair - state = :key - key = StringIO.new - value = StringIO.new - hex_buffer = "" - - @dn.each_char.with_index do |char, dn_index| - case state - when :key then - case char - when 'a'..'z', 'A'..'Z' then - state = :key_normal - key << char - when '0'..'9' then - state = :key_oid - key << char - when ' ' then state = :key - else raise(MalformedError, "Unrecognized first character of an RDN attribute type name \"#{char}\"") - end - when :key_normal then - case char - when '=' then state = :value - when 'a'..'z', 'A'..'Z', '0'..'9', '-', ' ' then key << char - else raise(MalformedError, "Unrecognized RDN attribute type name character \"#{char}\"") - end - when :key_oid then - case char - when '=' then state = :value - when '0'..'9', '.', ' ' then key << char - else raise(MalformedError, "Unrecognized RDN OID attribute type name character \"#{char}\"") - end - when :value then - case char - when '\\' then state = :value_normal_escape - when '"' then state = :value_quoted - when ' ' then state = :value - when '#' then - state = :value_hexstring - value << char - when ',' then - state = :key - yield key.string.strip, rstrip_except_escaped(value.string, dn_index) - key = StringIO.new - value = StringIO.new - else - state = :value_normal - value << char - end - when :value_normal then - case char - when '\\' then state = :value_normal_escape - when ',' then - state = :key - yield key.string.strip, rstrip_except_escaped(value.string, dn_index) - key = StringIO.new - value = StringIO.new - when '+' then raise(UnsupportedError, "Multivalued RDNs are not supported") - else value << char - end - when :value_normal_escape then - case char - when '0'..'9', 'a'..'f', 'A'..'F' then - state = :value_normal_escape_hex - hex_buffer = char - else - state = :value_normal - value << char - end - when :value_normal_escape_hex then - case char - when '0'..'9', 'a'..'f', 'A'..'F' then - state = :value_normal - value << "#{hex_buffer}#{char}".to_i(16).chr - else raise(MalformedError, "Invalid escaped hex code \"\\#{hex_buffer}#{char}\"") - end - when :value_quoted then - case char - when '\\' then state = :value_quoted_escape - when '"' then state = :value_end - else value << char - end - when :value_quoted_escape then - case char - when '0'..'9', 'a'..'f', 'A'..'F' then - state = :value_quoted_escape_hex - hex_buffer = char - else - state = :value_quoted - value << char - end - when :value_quoted_escape_hex then - case char - when '0'..'9', 'a'..'f', 'A'..'F' then - state = :value_quoted - value << "#{hex_buffer}#{char}".to_i(16).chr - else raise(MalformedError, "Expected the second character of a hex pair inside a double quoted value, but got \"#{char}\"") - end - when :value_hexstring then - case char - when '0'..'9', 'a'..'f', 'A'..'F' then - state = :value_hexstring_hex - value << char - when ' ' then state = :value_end - when ',' then - state = :key - yield key.string.strip, rstrip_except_escaped(value.string, dn_index) - key = StringIO.new - value = StringIO.new - else raise(MalformedError, "Expected the first character of a hex pair, but got \"#{char}\"") - end - when :value_hexstring_hex then - case char - when '0'..'9', 'a'..'f', 'A'..'F' then - state = :value_hexstring - value << char - else raise(MalformedError, "Expected the second character of a hex pair, but got \"#{char}\"") - end - when :value_end then - case char - when ' ' then state = :value_end - when ',' then - state = :key - yield key.string.strip, rstrip_except_escaped(value.string, dn_index) - key = StringIO.new - value = StringIO.new - else raise(MalformedError, "Expected the end of an attribute value, but got \"#{char}\"") - end - else raise "Fell out of state machine" - end - end - - # Last pair - raise(MalformedError, 'DN string ended unexpectedly') unless - [:value, :value_normal, :value_hexstring, :value_end].include? state - - yield key.string.strip, rstrip_except_escaped(value.string, @dn.length) - end - - def rstrip_except_escaped(str, dn_index) - str_ends_with_whitespace = str.match(/\s\z/) - - if str_ends_with_whitespace - dn_part_ends_with_escaped_whitespace = @dn[0, dn_index].match(/\\(\s+)\z/) - - if dn_part_ends_with_escaped_whitespace - dn_part_rwhitespace = dn_part_ends_with_escaped_whitespace[1] - num_chars_to_remove = dn_part_rwhitespace.length - 1 - str = str[0, str.length - num_chars_to_remove] - else - str.rstrip! - end - end - - str - end - - ## - # Returns the DN as an array in the form expected by the constructor. - def to_a - a = [] - self.each_pair { |key, value| a << key << value } unless @dn.empty? - a - end - - ## - # Return the DN as an escaped string. - def to_s - @dn - end - - ## - # Return the DN as an escaped and normalized string. - def to_normalized_s - self.class.new(*to_a).to_s.downcase - end - - # https://tools.ietf.org/html/rfc4514 section 2.4 lists these exceptions - # for DN values. All of the following must be escaped in any normal string - # using a single backslash ('\') as escape. The space character is left - # out here because in a "normalized" string, spaces should only be escaped - # if necessary (i.e. leading or trailing space). - NORMAL_ESCAPES = [',', '+', '"', '\\', '<', '>', ';', '='].freeze - - # The following must be represented as escaped hex - HEX_ESCAPES = { - "\n" => '\0a', - "\r" => '\0d' - }.freeze - - # Compiled character class regexp using the keys from the above hash, and - # checking for a space or # at the start, or space at the end, of the - # string. - ESCAPE_RE = Regexp.new("(^ |^#| $|[" + - NORMAL_ESCAPES.map { |e| Regexp.escape(e) }.join + - "])") - - HEX_ESCAPE_RE = Regexp.new("([" + - HEX_ESCAPES.keys.map { |e| Regexp.escape(e) }.join + - "])") - - ## - # Escape a string for use in a DN value - def self.escape(string) - escaped = string.gsub(ESCAPE_RE) { |char| "\\" + char } - escaped.gsub(HEX_ESCAPE_RE) { |char| HEX_ESCAPES[char] } - end - - private - - def initialize_array(args) - buffer = StringIO.new - - args.each_with_index do |arg, index| - if index.even? # key - buffer << "," if index > 0 - buffer << arg - else # value - buffer << "=" - buffer << self.class.escape(arg) - end - end - - @dn = buffer.string - end - - def initialize_string(arg) - @dn = arg.to_s - end - - ## - # Proxy all other requests to the string object, because a DN is mainly - # used within the library as a string - # rubocop:disable GitlabSecurity/PublicSend - def method_missing(method, *args, &block) - @dn.send(method, *args, &block) - end - - ## - # Redefined to be consistent with redefined `method_missing` behavior - def respond_to?(sym, include_private = false) - @dn.respond_to?(sym, include_private) - end - end - end - end - end - - def perform(start_id, end_id) - return unless migrate? - - ldap_identities = Identity.where("provider like 'ldap%'").where(id: start_id..end_id) - ldap_identities.each do |identity| - identity.extern_uid = Gitlab::Auth::LDAP::DN.new(identity.extern_uid).to_normalized_s - unless identity.save - Rails.logger.info "Unable to normalize \"#{identity.extern_uid}\". Skipping." - end - rescue Gitlab::Auth::LDAP::DN::FormatError => e - Rails.logger.info "Unable to normalize \"#{identity.extern_uid}\" due to \"#{e.message}\". Skipping." - end - end - - def migrate? - Identity.table_exists? - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_external_pipeline_source.rb b/lib/gitlab/background_migration/populate_external_pipeline_source.rb deleted file mode 100644 index 036fe641757..00000000000 --- a/lib/gitlab/background_migration/populate_external_pipeline_source.rb +++ /dev/null @@ -1,50 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class PopulateExternalPipelineSource - module Migratable - class Pipeline < ActiveRecord::Base - self.table_name = 'ci_pipelines' - - def self.sources - { - unknown: nil, - push: 1, - web: 2, - trigger: 3, - schedule: 4, - api: 5, - external: 6 - } - end - end - - class CommitStatus < ActiveRecord::Base - self.table_name = 'ci_builds' - self.inheritance_column = :_type_disabled - - scope :has_pipeline, -> { where('ci_builds.commit_id=ci_pipelines.id') } - scope :of_type, -> (type) { where('type=?', type) } - end - end - - def perform(start_id, stop_id) - external_pipelines(start_id, stop_id) - .update_all(source: Migratable::Pipeline.sources[:external]) - end - - private - - def external_pipelines(start_id, stop_id) - Migratable::Pipeline.where(id: (start_id..stop_id)) - .where( - 'EXISTS (?) AND NOT EXISTS (?)', - Migratable::CommitStatus.of_type('GenericCommitStatus').has_pipeline.select(1), - Migratable::CommitStatus.of_type('Ci::Build').has_pipeline.select(1) - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_fork_networks_range.rb b/lib/gitlab/background_migration/populate_fork_networks_range.rb deleted file mode 100644 index aa4f130538c..00000000000 --- a/lib/gitlab/background_migration/populate_fork_networks_range.rb +++ /dev/null @@ -1,128 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This background migration is going to create all `fork_networks` and - # the `fork_network_members` for the roots of fork networks based on the - # existing `forked_project_links`. - # - # When the source of a fork is deleted, we will create the fork with the - # target project as the root. This way, when there are forks of the target - # project, they will be joined into the same fork network. - # - # When the `fork_networks` and memberships for the root projects are created - # the `CreateForkNetworkMembershipsRange` migration is scheduled. This - # migration will create the memberships for all remaining forks-of-forks - class PopulateForkNetworksRange - def perform(start_id, end_id) - create_fork_networks_for_existing_projects(start_id, end_id) - create_fork_networks_for_missing_projects(start_id, end_id) - create_fork_networks_memberships_for_root_projects(start_id, end_id) - - delay = BackgroundMigration::CreateForkNetworkMembershipsRange::RESCHEDULE_DELAY - BackgroundMigrationWorker.perform_in( - delay, "CreateForkNetworkMembershipsRange", [start_id, end_id] - ) - end - - def create_fork_networks_for_existing_projects(start_id, end_id) - log("Creating fork networks: #{start_id} - #{end_id}") - ActiveRecord::Base.connection.execute <<~INSERT_NETWORKS - INSERT INTO fork_networks (root_project_id) - SELECT DISTINCT forked_project_links.forked_from_project_id - - FROM forked_project_links - - -- Exclude the forks that are not the first level fork of a project - WHERE NOT EXISTS ( - SELECT true - FROM forked_project_links inner_links - WHERE inner_links.forked_to_project_id = forked_project_links.forked_from_project_id - ) - - /* Exclude the ones that are already created, in case the fork network - was already created for another fork of the project. - */ - AND NOT EXISTS ( - SELECT true - FROM fork_networks - WHERE forked_project_links.forked_from_project_id = fork_networks.root_project_id - ) - - -- Only create a fork network for a root project that still exists - AND EXISTS ( - SELECT true - FROM projects - WHERE projects.id = forked_project_links.forked_from_project_id - ) - AND forked_project_links.id BETWEEN #{start_id} AND #{end_id} - INSERT_NETWORKS - end - - def create_fork_networks_for_missing_projects(start_id, end_id) - log("Creating fork networks with missing root: #{start_id} - #{end_id}") - ActiveRecord::Base.connection.execute <<~INSERT_NETWORKS - INSERT INTO fork_networks (root_project_id) - SELECT DISTINCT forked_project_links.forked_to_project_id - - FROM forked_project_links - - -- Exclude forks that are not the root forks - WHERE NOT EXISTS ( - SELECT true - FROM forked_project_links inner_links - WHERE inner_links.forked_to_project_id = forked_project_links.forked_from_project_id - ) - - /* Exclude the ones that are already created, in case this migration is - re-run - */ - AND NOT EXISTS ( - SELECT true - FROM fork_networks - WHERE forked_project_links.forked_to_project_id = fork_networks.root_project_id - ) - - /* Exclude projects for which the project still exists, those are - Processed in the previous step of this migration - */ - AND NOT EXISTS ( - SELECT true - FROM projects - WHERE projects.id = forked_project_links.forked_from_project_id - ) - AND forked_project_links.id BETWEEN #{start_id} AND #{end_id} - INSERT_NETWORKS - end - - def create_fork_networks_memberships_for_root_projects(start_id, end_id) - log("Creating memberships for root projects: #{start_id} - #{end_id}") - - ActiveRecord::Base.connection.execute <<~INSERT_ROOT - INSERT INTO fork_network_members (fork_network_id, project_id) - SELECT DISTINCT fork_networks.id, fork_networks.root_project_id - - FROM fork_networks - - /* Joining both on forked_from- and forked_to- so we could create the - memberships for forks for which the source was deleted - */ - INNER JOIN forked_project_links - ON forked_project_links.forked_from_project_id = fork_networks.root_project_id - OR forked_project_links.forked_to_project_id = fork_networks.root_project_id - - WHERE NOT EXISTS ( - SELECT true - FROM fork_network_members - WHERE fork_network_members.project_id = fork_networks.root_project_id - ) - AND forked_project_links.id BETWEEN #{start_id} AND #{end_id} - INSERT_ROOT - end - - def log(message) - Rails.logger.info("#{self.class.name} - #{message}") - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_import_state.rb b/lib/gitlab/background_migration/populate_import_state.rb deleted file mode 100644 index 695a2a713c5..00000000000 --- a/lib/gitlab/background_migration/populate_import_state.rb +++ /dev/null @@ -1,39 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This background migration creates all the records on the - # import state table for projects that are considered imports or forks - class PopulateImportState - def perform(start_id, end_id) - move_attributes_data_to_import_state(start_id, end_id) - rescue ActiveRecord::RecordNotUnique - retry - end - - def move_attributes_data_to_import_state(start_id, end_id) - Rails.logger.info("#{self.class.name} - Moving import attributes data to project mirror data table: #{start_id} - #{end_id}") - - ActiveRecord::Base.connection.execute <<~SQL - INSERT INTO project_mirror_data (project_id, status, jid, last_error) - SELECT id, import_status, import_jid, import_error - FROM projects - WHERE projects.import_status != 'none' - AND projects.id BETWEEN #{start_id} AND #{end_id} - AND NOT EXISTS ( - SELECT id - FROM project_mirror_data - WHERE project_id = projects.id - ) - SQL - - ActiveRecord::Base.connection.execute <<~SQL - UPDATE projects - SET import_status = 'none' - WHERE import_status != 'none' - AND id BETWEEN #{start_id} AND #{end_id} - SQL - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_merge_request_assignees_table.rb b/lib/gitlab/background_migration/populate_merge_request_assignees_table.rb index a4c6540c61b..eb4bc0aaf28 100644 --- a/lib/gitlab/background_migration/populate_merge_request_assignees_table.rb +++ b/lib/gitlab/background_migration/populate_merge_request_assignees_table.rb @@ -18,6 +18,14 @@ module Gitlab execute("INSERT INTO merge_request_assignees (merge_request_id, user_id) #{select_sql}") end + def perform_all_sync(batch_size:) + MergeRequest.each_batch(of: batch_size) do |batch| + range = batch.pluck('MIN(id)', 'MAX(id)').first + + perform(*range) + end + end + private def merge_request_assignees_not_exists_clause diff --git a/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data.rb b/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data.rb deleted file mode 100644 index d89ce358bb9..00000000000 --- a/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data.rb +++ /dev/null @@ -1,132 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class PopulateMergeRequestMetricsWithEventsData - def perform(min_merge_request_id, max_merge_request_id) - insert_metrics_for_range(min_merge_request_id, max_merge_request_id) - update_metrics_with_events_data(min_merge_request_id, max_merge_request_id) - end - - # Inserts merge_request_metrics records for merge_requests without it for - # a given merge request batch. - def insert_metrics_for_range(min, max) - metrics_not_exists_clause = - <<-SQL.strip_heredoc - NOT EXISTS (SELECT 1 FROM merge_request_metrics - WHERE merge_request_metrics.merge_request_id = merge_requests.id) - SQL - - MergeRequest.where(metrics_not_exists_clause).where(id: min..max).each_batch do |batch| - select_sql = batch.select(:id, :created_at, :updated_at).to_sql - - execute("INSERT INTO merge_request_metrics (merge_request_id, created_at, updated_at) #{select_sql}") - end - end - - def update_metrics_with_events_data(min, max) - if Gitlab::Database.postgresql? - # Uses WITH syntax in order to update merged and closed events with a single UPDATE. - # WITH is not supported by MySQL. - update_events_for_range(min, max) - else - update_merged_events_for_range(min, max) - update_closed_events_for_range(min, max) - end - end - - private - - # Updates merge_request_metrics latest_closed_at, latest_closed_by_id and merged_by_id - # based on the latest event records on events table for a given merge request batch. - def update_events_for_range(min, max) - sql = <<-SQL.strip_heredoc - WITH events_for_update AS ( - SELECT DISTINCT ON (target_id, action) target_id, action, author_id, updated_at - FROM events - WHERE target_id BETWEEN #{min} AND #{max} - AND target_type = 'MergeRequest' - AND action IN (#{Event::CLOSED},#{Event::MERGED}) - ORDER BY target_id, action, id DESC - ) - UPDATE merge_request_metrics met - SET latest_closed_at = latest_closed.updated_at, - latest_closed_by_id = latest_closed.author_id, - merged_by_id = latest_merged.author_id - FROM (SELECT * FROM events_for_update WHERE action = #{Event::CLOSED}) AS latest_closed - FULL OUTER JOIN - (SELECT * FROM events_for_update WHERE action = #{Event::MERGED}) AS latest_merged - USING (target_id) - WHERE target_id = merge_request_id; - SQL - - execute(sql) - end - - # Updates merge_request_metrics latest_closed_at, latest_closed_by_id based on the latest closed - # records on events table for a given merge request batch. - def update_closed_events_for_range(min, max) - sql = - <<-SQL.strip_heredoc - UPDATE merge_request_metrics metrics, - (#{select_events(min, max, Event::CLOSED)}) closed_events - SET metrics.latest_closed_by_id = closed_events.author_id, - metrics.latest_closed_at = closed_events.updated_at #{where_matches_closed_events}; - SQL - - execute(sql) - end - - # Updates merge_request_metrics merged_by_id based on the latest merged - # records on events table for a given merge request batch. - def update_merged_events_for_range(min, max) - sql = - <<-SQL.strip_heredoc - UPDATE merge_request_metrics metrics, - (#{select_events(min, max, Event::MERGED)}) merged_events - SET metrics.merged_by_id = merged_events.author_id #{where_matches_merged_events}; - SQL - - execute(sql) - end - - def execute(sql) - @connection ||= ActiveRecord::Base.connection - @connection.execute(sql) - end - - def select_events(min, max, action) - select_max_event_id = <<-SQL.strip_heredoc - SELECT max(id) - FROM events - WHERE action = #{action} - AND target_type = 'MergeRequest' - AND target_id BETWEEN #{min} AND #{max} - GROUP BY target_id - SQL - - <<-SQL.strip_heredoc - SELECT author_id, updated_at, target_id - FROM events - WHERE id IN(#{select_max_event_id}) - SQL - end - - def where_matches_closed_events - <<-SQL.strip_heredoc - WHERE metrics.merge_request_id = closed_events.target_id - AND metrics.latest_closed_at IS NULL - AND metrics.latest_closed_by_id IS NULL - SQL - end - - def where_matches_merged_events - <<-SQL.strip_heredoc - WHERE metrics.merge_request_id = merged_events.target_id - AND metrics.merged_by_id IS NULL - SQL - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data_improved.rb b/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data_improved.rb deleted file mode 100644 index 37592d67dd9..00000000000 --- a/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data_improved.rb +++ /dev/null @@ -1,99 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class PopulateMergeRequestMetricsWithEventsDataImproved - CLOSED_EVENT_ACTION = 3 - MERGED_EVENT_ACTION = 7 - - def perform(min_merge_request_id, max_merge_request_id) - insert_metrics_for_range(min_merge_request_id, max_merge_request_id) - update_metrics_with_events_data(min_merge_request_id, max_merge_request_id) - end - - # Inserts merge_request_metrics records for merge_requests without it for - # a given merge request batch. - def insert_metrics_for_range(min, max) - metrics_not_exists_clause = - <<-SQL.strip_heredoc - NOT EXISTS (SELECT 1 FROM merge_request_metrics - WHERE merge_request_metrics.merge_request_id = merge_requests.id) - SQL - - MergeRequest.where(metrics_not_exists_clause).where(id: min..max).each_batch do |batch| - select_sql = batch.select(:id, :created_at, :updated_at).to_sql - - execute("INSERT INTO merge_request_metrics (merge_request_id, created_at, updated_at) #{select_sql}") - end - end - - def update_metrics_with_events_data(min, max) - if Gitlab::Database.postgresql? - psql_update_metrics_with_events_data(min, max) - else - mysql_update_metrics_with_events_data(min, max) - end - end - - def psql_update_metrics_with_events_data(min, max) - update_sql = <<-SQL.strip_heredoc - UPDATE merge_request_metrics - SET (latest_closed_at, - latest_closed_by_id) = - ( SELECT updated_at, - author_id - FROM events - WHERE target_id = merge_request_id - AND target_type = 'MergeRequest' - AND action = #{CLOSED_EVENT_ACTION} - ORDER BY id DESC - LIMIT 1 ), - merged_by_id = - ( SELECT author_id - FROM events - WHERE target_id = merge_request_id - AND target_type = 'MergeRequest' - AND action = #{MERGED_EVENT_ACTION} - ORDER BY id DESC - LIMIT 1 ) - WHERE merge_request_id BETWEEN #{min} AND #{max} - SQL - - execute(update_sql) - end - - def mysql_update_metrics_with_events_data(min, max) - closed_updated_at_subquery = mysql_events_select(:updated_at, CLOSED_EVENT_ACTION) - closed_author_id_subquery = mysql_events_select(:author_id, CLOSED_EVENT_ACTION) - merged_author_id_subquery = mysql_events_select(:author_id, MERGED_EVENT_ACTION) - - update_sql = <<-SQL.strip_heredoc - UPDATE merge_request_metrics - SET latest_closed_at = (#{closed_updated_at_subquery}), - latest_closed_by_id = (#{closed_author_id_subquery}), - merged_by_id = (#{merged_author_id_subquery}) - WHERE merge_request_id BETWEEN #{min} AND #{max} - SQL - - execute(update_sql) - end - - def mysql_events_select(column, action) - <<-SQL.strip_heredoc - SELECT #{column} FROM events - WHERE target_id = merge_request_id - AND target_type = 'MergeRequest' - AND action = #{action} - ORDER BY id DESC - LIMIT 1 - SQL - end - - def execute(sql) - @connection ||= ActiveRecord::Base.connection - @connection.execute(sql) - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_merge_requests_latest_merge_request_diff_id.rb b/lib/gitlab/background_migration/populate_merge_requests_latest_merge_request_diff_id.rb deleted file mode 100644 index dcac355e1b0..00000000000 --- a/lib/gitlab/background_migration/populate_merge_requests_latest_merge_request_diff_id.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class PopulateMergeRequestsLatestMergeRequestDiffId - BATCH_SIZE = 1_000 - - class MergeRequest < ActiveRecord::Base - self.table_name = 'merge_requests' - - include ::EachBatch - end - - def perform(start_id, stop_id) - update = ' - latest_merge_request_diff_id = ( - SELECT MAX(id) - FROM merge_request_diffs - WHERE merge_requests.id = merge_request_diffs.merge_request_id - )'.squish - - MergeRequest - .where(id: start_id..stop_id) - .where(latest_merge_request_diff_id: nil) - .each_batch(of: BATCH_SIZE) do |relation| - - relation.update_all(update) - end - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 755b5ee725a..d2924d10225 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -42,7 +42,7 @@ module Gitlab #{e.message} #{e.backtrace.join("\n ")} MSG - Rails.logger.error(msg) + Rails.logger.error(msg) # rubocop:disable Gitlab/RailsLogger false end end diff --git a/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb b/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb index 1924f2ffee2..f5fb33f1660 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb @@ -176,23 +176,12 @@ module Gitlab self.table_name = 'projects' def self.find_by_full_path(path) - binary = Gitlab::Database.mysql? ? 'BINARY' : '' - order_sql = "(CASE WHEN #{binary} routes.path = #{connection.quote(path)} THEN 0 ELSE 1 END)" + order_sql = "(CASE WHEN routes.path = #{connection.quote(path)} THEN 0 ELSE 1 END)" where_full_path_in(path).reorder(order_sql).take end def self.where_full_path_in(path) - cast_lower = Gitlab::Database.postgresql? - - path = connection.quote(path) - - where = - if cast_lower - "(LOWER(routes.path) = LOWER(#{path}))" - else - "(routes.path = #{path})" - end - + where = "(LOWER(routes.path) = LOWER(#{connection.quote(path)}))" joins("INNER JOIN routes ON routes.source_id = projects.id AND routes.source_type = 'Project'").where(where) end end diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 1ee44a3a5a9..2ac51dd7b55 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -55,7 +55,7 @@ module Gitlab def ensure_temporary_tracking_table_exists table_name = :untracked_files_for_uploads - unless ActiveRecord::Base.connection.data_source_exists?(table_name) + unless ActiveRecord::Base.connection.table_exists?(table_name) UntrackedFile.connection.create_table table_name do |t| t.string :path, limit: 600, null: false t.index :path, unique: true @@ -111,7 +111,7 @@ module Gitlab cmd = %W[#{ionice} -c Idle] + cmd if ionice log_msg = "PrepareUntrackedUploads find command: \"#{cmd.join(' ')}\"" - Rails.logger.info log_msg + Rails.logger.info log_msg # rubocop:disable Gitlab/RailsLogger cmd end @@ -133,12 +133,9 @@ module Gitlab def insert_sql(file_paths) if postgresql_pre_9_5? "INSERT INTO #{table_columns_and_values_for_insert(file_paths)};" - elsif postgresql? + else "INSERT INTO #{table_columns_and_values_for_insert(file_paths)}"\ " ON CONFLICT DO NOTHING;" - else # MySQL - "INSERT IGNORE INTO"\ - " #{table_columns_and_values_for_insert(file_paths)};" end end @@ -150,19 +147,13 @@ module Gitlab "#{UntrackedFile.table_name} (path) VALUES #{values}" end - def postgresql? - strong_memoize(:postgresql) do - Gitlab::Database.postgresql? - end - end - def can_bulk_insert_and_ignore_duplicates? !postgresql_pre_9_5? end def postgresql_pre_9_5? strong_memoize(:postgresql_pre_9_5) do - postgresql? && Gitlab::Database.version.to_f < 9.5 + Gitlab::Database.version.to_f < 9.5 end end diff --git a/lib/gitlab/background_migration/redact_links.rb b/lib/gitlab/background_migration/redact_links.rb deleted file mode 100644 index 92256e59a6c..00000000000 --- a/lib/gitlab/background_migration/redact_links.rb +++ /dev/null @@ -1,51 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -require_relative 'redact_links/redactable' - -module Gitlab - module BackgroundMigration - class RedactLinks - class Note < ActiveRecord::Base - include EachBatch - include ::Gitlab::BackgroundMigration::RedactLinks::Redactable - - self.table_name = 'notes' - self.inheritance_column = :_type_disabled - end - - class Issue < ActiveRecord::Base - include EachBatch - include ::Gitlab::BackgroundMigration::RedactLinks::Redactable - - self.table_name = 'issues' - self.inheritance_column = :_type_disabled - end - - class MergeRequest < ActiveRecord::Base - include EachBatch - include ::Gitlab::BackgroundMigration::RedactLinks::Redactable - - self.table_name = 'merge_requests' - self.inheritance_column = :_type_disabled - end - - class Snippet < ActiveRecord::Base - include EachBatch - include ::Gitlab::BackgroundMigration::RedactLinks::Redactable - - self.table_name = 'snippets' - self.inheritance_column = :_type_disabled - end - - def perform(model_name, field, start_id, stop_id) - link_pattern = "%/sent_notifications/" + ("_" * 32) + "/unsubscribe%" - model = "Gitlab::BackgroundMigration::RedactLinks::#{model_name}".constantize - - model.where("#{field} like ?", link_pattern).where(id: start_id..stop_id).each do |resource| - resource.redact_field!(field) - end - end - end - end -end diff --git a/lib/gitlab/background_migration/redact_links/redactable.rb b/lib/gitlab/background_migration/redact_links/redactable.rb deleted file mode 100644 index baab34221f1..00000000000 --- a/lib/gitlab/background_migration/redact_links/redactable.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class RedactLinks - module Redactable - extend ActiveSupport::Concern - - def redact_field!(field) - self[field].gsub!(%r{/sent_notifications/\h{32}/unsubscribe}, '/sent_notifications/REDACTED/unsubscribe') - - if self.changed? - self.update_columns(field => self[field], - "#{field}_html" => nil) - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/remove_restricted_todos.rb b/lib/gitlab/background_migration/remove_restricted_todos.rb index 47579d46c1b..9ef6d8654ae 100644 --- a/lib/gitlab/background_migration/remove_restricted_todos.rb +++ b/lib/gitlab/background_migration/remove_restricted_todos.rb @@ -50,14 +50,7 @@ module Gitlab private def remove_non_members_todos(project_id) - if Gitlab::Database.postgresql? - batch_remove_todos_cte(project_id) - else - unauthorized_project_todos(project_id) - .each_batch(of: 5000) do |batch| - batch.delete_all - end - end + batch_remove_todos_cte(project_id) end def remove_confidential_issue_todos(project_id) @@ -90,13 +83,7 @@ module Gitlab next if target_types.empty? - if Gitlab::Database.postgresql? - batch_remove_todos_cte(project_id, target_types) - else - unauthorized_project_todos(project_id) - .where(target_type: target_types) - .delete_all - end + batch_remove_todos_cte(project_id, target_types) end end diff --git a/lib/gitlab/background_migration/rollback_import_state_data.rb b/lib/gitlab/background_migration/rollback_import_state_data.rb deleted file mode 100644 index a7c986747d8..00000000000 --- a/lib/gitlab/background_migration/rollback_import_state_data.rb +++ /dev/null @@ -1,40 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This background migration migrates all the data of import_state - # back to the projects table for projects that are considered imports or forks - class RollbackImportStateData - def perform(start_id, end_id) - move_attributes_data_to_project(start_id, end_id) - end - - def move_attributes_data_to_project(start_id, end_id) - Rails.logger.info("#{self.class.name} - Moving import attributes data to projects table: #{start_id} - #{end_id}") - - if Gitlab::Database.mysql? - ActiveRecord::Base.connection.execute <<~SQL - UPDATE projects, project_mirror_data - SET - projects.import_status = project_mirror_data.status, - projects.import_jid = project_mirror_data.jid, - projects.import_error = project_mirror_data.last_error - WHERE project_mirror_data.project_id = projects.id - AND project_mirror_data.id BETWEEN #{start_id} AND #{end_id} - SQL - else - ActiveRecord::Base.connection.execute <<~SQL - UPDATE projects - SET - import_status = project_mirror_data.status, - import_jid = project_mirror_data.jid, - import_error = project_mirror_data.last_error - FROM project_mirror_data - WHERE project_mirror_data.project_id = projects.id - AND project_mirror_data.id BETWEEN #{start_id} AND #{end_id} - SQL - end - end - end - end -end diff --git a/lib/gitlab/background_migration/schedule_diff_files_deletion.rb b/lib/gitlab/background_migration/schedule_diff_files_deletion.rb deleted file mode 100644 index 609cf19187c..00000000000 --- a/lib/gitlab/background_migration/schedule_diff_files_deletion.rb +++ /dev/null @@ -1,44 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class ScheduleDiffFilesDeletion - class MergeRequestDiff < ActiveRecord::Base - self.table_name = 'merge_request_diffs' - - belongs_to :merge_request - - include EachBatch - end - - DIFF_BATCH_SIZE = 5_000 - INTERVAL = 5.minutes - MIGRATION = 'DeleteDiffFiles' - - def perform - diffs = MergeRequestDiff - .from("(#{diffs_collection.to_sql}) merge_request_diffs") - .where('merge_request_diffs.id != merge_request_diffs.latest_merge_request_diff_id') - .select(:id) - - diffs.each_batch(of: DIFF_BATCH_SIZE) do |relation, index| - ids = relation.pluck(:id) - - BackgroundMigrationWorker.perform_in(index * INTERVAL, MIGRATION, [ids]) - end - end - - private - - def diffs_collection - MergeRequestDiff - .joins(:merge_request) - .where("merge_requests.state = 'merged'") - .where('merge_requests.latest_merge_request_diff_id IS NOT NULL') - .where("merge_request_diffs.state NOT IN ('without_files', 'empty')") - .select('merge_requests.latest_merge_request_diff_id, merge_request_diffs.id') - end - end - end -end diff --git a/lib/gitlab/badge/coverage/report.rb b/lib/gitlab/badge/coverage/report.rb index 7f7cc62c8ef..15cccc6f287 100644 --- a/lib/gitlab/badge/coverage/report.rb +++ b/lib/gitlab/badge/coverage/report.rb @@ -14,7 +14,7 @@ module Gitlab @ref = ref @job = job - @pipeline = @project.ci_pipelines.latest_successful_for(@ref) + @pipeline = @project.ci_pipelines.latest_successful_for_ref(@ref) end def entity diff --git a/lib/gitlab/batch_pop_queueing.rb b/lib/gitlab/batch_pop_queueing.rb new file mode 100644 index 00000000000..61011abddf5 --- /dev/null +++ b/lib/gitlab/batch_pop_queueing.rb @@ -0,0 +1,112 @@ +# frozen_string_literal: true + +module Gitlab + ## + # This class is a queuing system for processing expensive tasks in an atomic manner + # with batch poping to let you optimize the total processing time. + # + # In usual queuing system, the first item started being processed immediately + # and the following items wait until the next items have been popped from the queue. + # On the other hand, this queueing system, the former part is same, however, + # it pops the enqueued items as batch. This is especially useful when you want to + # drop redandant items from the queue in order to process important items only, + # thus it's more efficient than the traditional queueing system. + # + # Caveats: + # - The order of the items are not guaranteed because of `sadd` (Redis Sets). + # + # Example: + # ``` + # class TheWorker + # def perform + # result = Gitlab::BatchPopQueueing.new('feature', 'queue').safe_execute([item]) do |items_in_queue| + # item = extract_the_most_important_item_from(items_in_queue) + # expensive_process(item) + # end + # + # if result[:status] == :finished && result[:new_items].present? + # item = extract_the_most_important_item_from(items_in_queue) + # TheWorker.perform_async(item.id) + # end + # end + # end + # ``` + # + class BatchPopQueueing + attr_reader :namespace, :queue_id + + EXTRA_QUEUE_EXPIRE_WINDOW = 1.hour + MAX_COUNTS_OF_POP_ALL = 1000 + + # Initialize queue + # + # @param [String] namespace The namespace of the exclusive lock and queue key. Typically, it's a feature name. + # @param [String] queue_id The identifier of the queue. + # @return [Boolean] + def initialize(namespace, queue_id) + raise ArgumentError if namespace.empty? || queue_id.empty? + + @namespace, @queue_id = namespace, queue_id + end + + ## + # Execute the given block in an exclusive lock. + # If there is the other thread has already working on the block, + # it enqueues the items without processing the block. + # + # @param [Array<String>] new_items New items to be added to the queue. + # @param [Time] lock_timeout The timeout of the exclusive lock. Generally, this value should be longer than the maximum prosess timing of the given block. + # @return [Hash] + # - status => One of the `:enqueued` or `:finished`. + # - new_items => Newly enqueued items during the given block had been processed. + # + # NOTE: If an exception is raised in the block, the poppped items will not be recovered. + # We should NOT re-enqueue the items in this case because it could end up in an infinite loop. + def safe_execute(new_items, lock_timeout: 10.minutes, &block) + enqueue(new_items, lock_timeout + EXTRA_QUEUE_EXPIRE_WINDOW) + + lease = Gitlab::ExclusiveLease.new(lock_key, timeout: lock_timeout) + + return { status: :enqueued } unless uuid = lease.try_obtain + + begin + all_args = pop_all + + yield all_args if block_given? + + { status: :finished, new_items: peek_all } + ensure + Gitlab::ExclusiveLease.cancel(lock_key, uuid) + end + end + + private + + def lock_key + @lock_key ||= "batch_pop_queueing:lock:#{namespace}:#{queue_id}" + end + + def queue_key + @queue_key ||= "batch_pop_queueing:queue:#{namespace}:#{queue_id}" + end + + def enqueue(items, expire_time) + Gitlab::Redis::Queues.with do |redis| + redis.sadd(queue_key, items) + redis.expire(queue_key, expire_time.to_i) + end + end + + def pop_all + Gitlab::Redis::Queues.with do |redis| + redis.spop(queue_key, MAX_COUNTS_OF_POP_ALL) + end + end + + def peek_all + Gitlab::Redis::Queues.with do |redis| + redis.smembers(queue_key) + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index c9f0ed66a54..24bc73e0de5 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -11,6 +11,7 @@ module Gitlab { title: 'task', color: '#7F8C8D' }].freeze attr_reader :project, :client, :errors, :users + attr_accessor :logger def initialize(project) @project = project @@ -19,6 +20,7 @@ module Gitlab @labels = {} @errors = [] @users = {} + @logger = Gitlab::Import::Logger.build end def execute @@ -41,6 +43,18 @@ module Gitlab }.to_json) end + def store_pull_request_error(pull_request, ex) + backtrace = Gitlab::Profiler.clean_backtrace(ex.backtrace) + error = { type: :pull_request, iid: pull_request.iid, errors: ex.message, trace: backtrace, raw_response: pull_request.raw } + + log_error(error) + # Omit the details from the database to avoid blowing up usage in the error column + error.delete(:trace) + error.delete(:raw_response) + + errors << error + end + def gitlab_user_id(project, username) find_user_id(username) || project.creator_id end @@ -176,7 +190,7 @@ module Gitlab import_pull_request_comments(pull_request, merge_request) if merge_request.persisted? rescue StandardError => e - errors << { type: :pull_request, iid: pull_request.iid, errors: e.message, trace: e.backtrace.join("\n"), raw_response: pull_request.raw } + store_pull_request_error(pull_request, e) end end @@ -248,12 +262,30 @@ module Gitlab def pull_request_comment_attributes(comment) { project: project, - note: comment.note, author_id: gitlab_user_id(project, comment.author), + note: comment_note(comment), created_at: comment.created_at, updated_at: comment.updated_at } end + + def comment_note(comment) + author = @formatter.author_line(comment.author) unless find_user_id(comment.author) + + author.to_s + comment.note.to_s + end + + def log_error(details) + logger.error(log_base_data.merge(details)) + end + + def log_base_data + { + class: self.class.name, + project_id: project.id, + project_path: project.full_path + } + end end end end diff --git a/lib/gitlab/blob_helper.rb b/lib/gitlab/blob_helper.rb index d3e15a79a8b..fc579ad8d2a 100644 --- a/lib/gitlab/blob_helper.rb +++ b/lib/gitlab/blob_helper.rb @@ -45,7 +45,7 @@ module Gitlab end def image? - ['.png', '.jpg', '.jpeg', '.gif'].include?(extname.downcase) + ['.png', '.jpg', '.jpeg', '.gif', '.svg'].include?(extname.downcase) end # Internal: Lookup mime type for extension. diff --git a/lib/gitlab/chaos.rb b/lib/gitlab/chaos.rb new file mode 100644 index 00000000000..4f47cdef971 --- /dev/null +++ b/lib/gitlab/chaos.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Gitlab + # Chaos methods for GitLab. + # See https://docs.gitlab.com/ee/development/chaos_endpoints.html for more details. + class Chaos + # leak_mem will retain the specified amount of memory and sleep. + # On return, the memory will be released. + def self.leak_mem(memory_mb, duration_s) + start_time = Time.now + + retainer = [] + # Add `n` 1mb chunks of memory to the retainer array + memory_mb.times { retainer << "x" * 1.megabyte } + + duration_left = [start_time + duration_s - Time.now, 0].max + Kernel.sleep(duration_left) + end + + # cpu_spin will consume all CPU on a single core for the specified duration + def self.cpu_spin(duration_s) + expected_end_time = Time.now + duration_s + + rand while Time.now < expected_end_time + end + + # db_spin will query the database in a tight loop for the specified duration + def self.db_spin(duration_s, interval_s) + expected_end_time = Time.now + duration_s + + while Time.now < expected_end_time + ActiveRecord::Base.connection.execute("SELECT 1") + + end_interval_time = Time.now + [duration_s, interval_s].min + rand while Time.now < end_interval_time + end + end + + # sleep will sleep for the specified duration + def self.sleep(duration_s) + Kernel.sleep(duration_s) + end + + # Kill will send a SIGKILL signal to the current process + def self.kill + Process.kill("KILL", Process.pid) + end + end +end diff --git a/lib/gitlab/checks/tag_check.rb b/lib/gitlab/checks/tag_check.rb index 2a75c8059bd..ced0612a7a3 100644 --- a/lib/gitlab/checks/tag_check.rb +++ b/lib/gitlab/checks/tag_check.rb @@ -19,7 +19,7 @@ module Gitlab return unless tag_name logger.log_timed(LOG_MESSAGES[:tag_checks]) do - if tag_exists? && user_access.cannot_do_action?(:admin_project) + if tag_exists? && user_access.cannot_do_action?(:admin_tag) raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:change_existing_tags] end end diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb index fba0de20ced..b7886114e9c 100644 --- a/lib/gitlab/ci/ansi2html.rb +++ b/lib/gitlab/ci/ansi2html.rb @@ -131,9 +131,9 @@ module Gitlab def on_109(_) set_bg_color(9, 'l') end - attr_accessor :offset, :n_open_tags, :fg_color, :bg_color, :style_mask + attr_accessor :offset, :n_open_tags, :fg_color, :bg_color, :style_mask, :sections, :lineno_in_section - STATE_PARAMS = [:offset, :n_open_tags, :fg_color, :bg_color, :style_mask].freeze + STATE_PARAMS = [:offset, :n_open_tags, :fg_color, :bg_color, :style_mask, :sections, :lineno_in_section].freeze def convert(stream, new_state) reset_state @@ -153,10 +153,9 @@ module Gitlab start_offset = @offset - open_new_tag - stream.each_line do |line| s = StringScanner.new(line) + until s.eos? if s.scan(Gitlab::Regex.build_trace_section_regex) @@ -166,11 +165,11 @@ module Gitlab elsif s.scan(/\e(([@-_])(.*?)?)?$/) break elsif s.scan(/</) - @out << '<' + write_in_tag '<' elsif s.scan(/\r?\n/) - @out << '<br>' + handle_new_line else - @out << s.scan(/./m) + write_in_tag s.scan(/./m) end @offset += s.matched_size @@ -190,13 +189,53 @@ module Gitlab ) end + def section_to_class_name(section) + section.to_s.downcase.gsub(/[^a-z0-9]/, '-') + end + + def handle_new_line + write_in_tag %{<br/>} + + close_open_tags if @sections.any? && @lineno_in_section == 0 + @lineno_in_section += 1 + end + def handle_section(scanner) action = scanner[1] timestamp = scanner[2] section = scanner[3] - line = scanner.matched[0...-5] # strips \r\033[0K - @out << %{<div class="hidden" data-action="#{action}" data-timestamp="#{timestamp}" data-section="#{section}">#{line}</div>} + normalized_section = section_to_class_name(section) + + if action == "start" + handle_section_start(normalized_section, timestamp) + elsif action == "end" + handle_section_end(normalized_section, timestamp) + end + end + + def handle_section_start(section, timestamp) + return if @sections.include?(section) + + @sections << section + write_raw %{<div class="js-section-start section-start fa fa-caret-down pr-2 cursor-pointer" data-timestamp="#{timestamp}" data-section="#{data_section_names}" role="button"></div>} + @lineno_in_section = 0 + end + + def handle_section_end(section, timestamp) + return unless @sections.include?(section) + + # close all sections up to section + until @sections.empty? + write_raw %{<div class="section-end" data-section="#{data_section_names}"></div>} + + last_section = @sections.pop + break if section == last_section + end + end + + def data_section_names + @sections.join(" ") end def handle_sequence(scanner) @@ -217,8 +256,6 @@ module Gitlab end evaluate_command_stack(commands) - - open_new_tag end def evaluate_command_stack(stack) @@ -231,6 +268,20 @@ module Gitlab evaluate_command_stack(stack) end + def write_in_tag(data) + ensure_open_new_tag + @out << data + end + + def write_raw(data) + close_open_tags + @out << data + end + + def ensure_open_new_tag + open_new_tag if @n_open_tags == 0 + end + def open_new_tag css_classes = [] @@ -251,9 +302,26 @@ module Gitlab css_classes << "term-#{css_class}" if @style_mask & flag != 0 end - return if css_classes.empty? + if @sections.any? + css_classes << "section" + + css_classes << if @lineno_in_section == 0 + "js-section-header section-header cursor-pointer" + else + "line" + end + + css_classes += sections.map { |section| "js-s-#{section}" } + end + + close_open_tags + + @out << if css_classes.any? + %{<span class="#{css_classes.join(' ')}">} + else + %{<span>} + end - @out << %{<span class="#{css_classes.join(' ')}">} @n_open_tags += 1 end @@ -268,6 +336,8 @@ module Gitlab @offset = 0 @n_open_tags = 0 @out = +'' + @sections = [] + @lineno_in_section = 0 reset end diff --git a/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb b/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb index dbdc59505ac..6ab4fca3854 100644 --- a/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb +++ b/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb @@ -8,32 +8,51 @@ module Gitlab def unmet? deployment_cluster.present? && deployment_cluster.managed? && - !deployment_cluster.project_type? && - kubernetes_namespace.new_record? + missing_namespace? end def complete! return unless unmet? - create_or_update_namespace + create_namespace end private + def missing_namespace? + kubernetes_namespace.nil? || kubernetes_namespace.service_account_token.blank? + end + def deployment_cluster build.deployment&.cluster end + def environment + build.deployment.environment + end + def kubernetes_namespace strong_memoize(:kubernetes_namespace) do - deployment_cluster.find_or_initialize_kubernetes_namespace_for_project(build.project) + Clusters::KubernetesNamespaceFinder.new( + deployment_cluster, + project: environment.project, + environment_slug: environment.slug, + allow_blank_token: true + ).execute end end - def create_or_update_namespace + def create_namespace Clusters::Gcp::Kubernetes::CreateOrUpdateNamespaceService.new( cluster: deployment_cluster, - kubernetes_namespace: kubernetes_namespace + kubernetes_namespace: kubernetes_namespace || build_namespace_record + ).execute + end + + def build_namespace_record + Clusters::BuildKubernetesNamespaceService.new( + deployment_cluster, + environment: environment ).execute end end diff --git a/lib/gitlab/ci/charts.rb b/lib/gitlab/ci/charts.rb index 7cabaadb122..3fbfdffe277 100644 --- a/lib/gitlab/ci/charts.rb +++ b/lib/gitlab/ci/charts.rb @@ -21,16 +21,10 @@ module Gitlab module MonthlyInterval # rubocop: disable CodeReuse/ActiveRecord def grouped_count(query) - if Gitlab::Database.postgresql? - query - .group("to_char(#{::Ci::Pipeline.table_name}.created_at, '01 Month YYYY')") - .count(:created_at) - .transform_keys(&:squish) - else - query - .group("DATE_FORMAT(#{::Ci::Pipeline.table_name}.created_at, '01 %M %Y')") - .count(:created_at) - end + query + .group("to_char(#{::Ci::Pipeline.table_name}.created_at, '01 Month YYYY')") + .count(:created_at) + .transform_keys(&:squish) end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index f187e456993..cde042c5e0a 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -8,25 +8,36 @@ module Gitlab class Config ConfigError = Class.new(StandardError) + RESCUE_ERRORS = [ + Gitlab::Config::Loader::FormatError, + Extendable::ExtensionError, + External::Processor::IncludeError + ].freeze + + attr_reader :root + def initialize(config, project: nil, sha: nil, user: nil) @config = Config::Extendable .new(build_config(config, project: project, sha: sha, user: user)) .to_hash - @global = Entry::Global.new(@config) - @global.compose! - rescue Gitlab::Config::Loader::FormatError, - Extendable::ExtensionError, - External::Processor::IncludeError => e + @root = Entry::Root.new(@config) + @root.compose! + + rescue Gitlab::Config::Loader::Yaml::DataTooLargeError => e + Gitlab::Sentry.track_exception(e, extra: { user: user.inspect, project: project.inspect }) + raise Config::ConfigError, e.message + + rescue *rescue_errors => e raise Config::ConfigError, e.message end def valid? - @global.valid? + @root.valid? end def errors - @global.errors + @root.errors end def to_hash @@ -36,36 +47,16 @@ module Gitlab ## # Temporary method that should be removed after refactoring # - def before_script - @global.before_script_value - end - - def image - @global.image_value - end - - def services - @global.services_value - end - - def after_script - @global.after_script_value - end - def variables - @global.variables_value + root.variables_value end def stages - @global.stages_value - end - - def cache - @global.cache_value + root.stages_value end def jobs - @global.jobs_value + root.jobs_value end private @@ -83,6 +74,11 @@ module Gitlab user: user, expandset: Set.new).perform end + + # Overriden in EE + def rescue_errors + RESCUE_ERRORS + end end end end diff --git a/lib/gitlab/ci/config/entry/default.rb b/lib/gitlab/ci/config/entry/default.rb new file mode 100644 index 00000000000..6200d7c7f87 --- /dev/null +++ b/lib/gitlab/ci/config/entry/default.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # This class represents a default entry + # Entry containing default values for all jobs + # defined in configuration file. + # + class Default < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable + + DuplicateError = Class.new(Gitlab::Config::Loader::FormatError) + + ALLOWED_KEYS = %i[before_script image services + after_script cache].freeze + + validations do + validates :config, allowed_keys: ALLOWED_KEYS + end + + entry :before_script, Entry::Script, + description: 'Script that will be executed before each job.', + inherit: true + + entry :image, Entry::Image, + description: 'Docker image that will be used to execute jobs.', + inherit: true + + entry :services, Entry::Services, + description: 'Docker images that will be linked to the container.', + inherit: true + + entry :after_script, Entry::Script, + description: 'Script that will be executed after each job.', + inherit: true + + entry :cache, Entry::Cache, + description: 'Configure caching between build jobs.', + inherit: true + + helpers :before_script, :image, :services, :after_script, :cache + + def compose!(deps = nil) + super(self) + + inherit!(deps) + end + + private + + def inherit!(deps) + return unless deps + + self.class.nodes.each do |key, factory| + next unless factory.inheritable? + + root_entry = deps[key] + next unless root_entry.specified? + + if self[key].specified? + raise DuplicateError, "#{key} is defined in top-level and `default:` entry" + end + + @entries[key] = root_entry + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/global.rb b/lib/gitlab/ci/config/entry/global.rb deleted file mode 100644 index 2b5a59c078e..00000000000 --- a/lib/gitlab/ci/config/entry/global.rb +++ /dev/null @@ -1,79 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - class Config - module Entry - ## - # This class represents a global entry - root Entry for entire - # GitLab CI Configuration file. - # - class Global < ::Gitlab::Config::Entry::Node - include ::Gitlab::Config::Entry::Configurable - - entry :before_script, Entry::Script, - description: 'Script that will be executed before each job.' - - entry :image, Entry::Image, - description: 'Docker image that will be used to execute jobs.' - - entry :include, Entry::Includes, - description: 'List of external YAML files to include.' - - entry :services, Entry::Services, - description: 'Docker images that will be linked to the container.' - - entry :after_script, Entry::Script, - description: 'Script that will be executed after each job.' - - entry :variables, Entry::Variables, - description: 'Environment variables that will be used.' - - entry :stages, Entry::Stages, - description: 'Configuration of stages for this pipeline.' - - entry :types, Entry::Stages, - description: 'Deprecated: stages for this pipeline.' - - entry :cache, Entry::Cache, - description: 'Configure caching between build jobs.' - - helpers :before_script, :image, :services, :after_script, - :variables, :stages, :types, :cache, :jobs - - def compose!(_deps = nil) - super(self) do - compose_jobs! - compose_deprecated_entries! - end - end - - private - - # rubocop: disable CodeReuse/ActiveRecord - def compose_jobs! - factory = ::Gitlab::Config::Entry::Factory.new(Entry::Jobs) - .value(@config.except(*self.class.nodes.keys)) - .with(key: :jobs, parent: self, - description: 'Jobs definition for this pipeline') - - @entries[:jobs] = factory.create! - end - # rubocop: enable CodeReuse/ActiveRecord - - def compose_deprecated_entries! - ## - # Deprecated `:types` key workaround - if types are defined and - # stages are not defined we use types definition as stages. - # - if types_defined? && !stages_defined? - @entries[:stages] = @entries[:types] - end - - @entries.delete(:types) - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/entry/hidden.rb b/lib/gitlab/ci/config/entry/hidden.rb index 76e5d05639f..f9a9f036527 100644 --- a/lib/gitlab/ci/config/entry/hidden.rb +++ b/lib/gitlab/ci/config/entry/hidden.rb @@ -14,6 +14,14 @@ module Gitlab validates :config, presence: true end + def self.matching?(name, config) + name.to_s.start_with?('.') + end + + def self.visible? + false + end + def relevant? false end diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 762532f7007..29a52b9da17 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -13,11 +13,14 @@ module Gitlab ALLOWED_KEYS = %i[tags script only except type image services allow_failure type stage when start_in artifacts cache - dependencies before_script after_script variables + dependencies needs before_script after_script variables environment coverage retry parallel extends].freeze + REQUIRED_BY_NEEDS = %i[stage].freeze + validations do validates :config, allowed_keys: ALLOWED_KEYS + validates :config, required_keys: REQUIRED_BY_NEEDS, if: :has_needs? validates :config, presence: true validates :script, presence: true validates :name, presence: true @@ -34,15 +37,27 @@ module Gitlab message: 'should be on_success, on_failure, ' \ 'always, manual or delayed' } validates :dependencies, array_of_strings: true + validates :needs, array_of_strings: true validates :extends, array_of_strings_or_string: true end validates :start_in, duration: { limit: '1 day' }, if: :delayed? validates :start_in, absence: true, unless: :delayed? + + validate do + next unless dependencies.present? + next unless needs.present? + + missing_needs = dependencies - needs + if missing_needs.any? + errors.add(:dependencies, "the #{missing_needs.join(", ")} should be part of needs") + end + end end entry :before_script, Entry::Script, - description: 'Global before script overridden in this job.' + description: 'Global before script overridden in this job.', + inherit: true entry :script, Entry::Commands, description: 'Commands that will be executed in this job.' @@ -54,16 +69,20 @@ module Gitlab description: 'Deprecated: stage this job will be executed into.' entry :after_script, Entry::Script, - description: 'Commands that will be executed when finishing job.' + description: 'Commands that will be executed when finishing job.', + inherit: true entry :cache, Entry::Cache, - description: 'Cache definition for this job.' + description: 'Cache definition for this job.', + inherit: true entry :image, Entry::Image, - description: 'Image that will be used to execute this job.' + description: 'Image that will be used to execute this job.', + inherit: true entry :services, Entry::Services, - description: 'Services that will be used to execute this job.' + description: 'Services that will be used to execute this job.', + inherit: true entry :only, Entry::Policy, description: 'Refs policy this job will be executed for.', @@ -90,10 +109,19 @@ module Gitlab helpers :before_script, :script, :stage, :type, :after_script, :cache, :image, :services, :only, :except, :variables, :artifacts, :environment, :coverage, :retry, - :parallel + :parallel, :needs attributes :script, :tags, :allow_failure, :when, :dependencies, - :retry, :parallel, :extends, :start_in + :needs, :retry, :parallel, :extends, :start_in + + def self.matching?(name, config) + !name.to_s.start_with?('.') && + config.is_a?(Hash) && config.key?(:script) + end + + def self.visible? + true + end def compose!(deps = nil) super do @@ -129,15 +157,19 @@ module Gitlab private + # We inherit config entries from `default:` + # if the entry has the `inherit: true` flag set def inherit!(deps) return unless deps - self.class.nodes.each_key do |key| - global_entry = deps[key] + self.class.nodes.each do |key, factory| + next unless factory.inheritable? + + default_entry = deps.default[key] job_entry = self[key] - if global_entry.specified? && !job_entry.specified? - @entries[key] = global_entry + if default_entry.specified? && !job_entry.specified? + @entries[key] = default_entry end end end @@ -152,7 +184,7 @@ module Gitlab cache: cache_value, only: only_value, except: except_value, - variables: variables_defined? ? variables_value : nil, + variables: variables_defined? ? variables_value : {}, environment: environment_defined? ? environment_value : nil, environment_name: environment_defined? ? environment_value[:name] : nil, coverage: coverage_defined? ? coverage_value : nil, @@ -160,7 +192,8 @@ module Gitlab parallel: parallel_defined? ? parallel_value.to_i : nil, artifacts: artifacts_value, after_script: after_script_value, - ignore: ignored? } + ignore: ignored?, + needs: needs_defined? ? needs_value : nil } end end end diff --git a/lib/gitlab/ci/config/entry/jobs.rb b/lib/gitlab/ci/config/entry/jobs.rb index 9845c4af655..9d1a1ee8c4b 100644 --- a/lib/gitlab/ci/config/entry/jobs.rb +++ b/lib/gitlab/ci/config/entry/jobs.rb @@ -14,29 +14,48 @@ module Gitlab validates :config, type: Hash validate do + unless has_valid_jobs? + errors.add(:config, 'should contain valid jobs') + end + unless has_visible_job? errors.add(:config, 'should contain at least one visible job') end end + def has_valid_jobs? + config.all? do |name, value| + Jobs.find_type(name, value) + end + end + def has_visible_job? - config.any? { |name, _| !hidden?(name) } + config.any? do |name, value| + Jobs.find_type(name, value)&.visible? + end end end - def hidden?(name) - name.to_s.start_with?('.') + TYPES = [Entry::Hidden, Entry::Job].freeze + + private_constant :TYPES + + def self.all_types + TYPES end - def node_type(name) - hidden?(name) ? Entry::Hidden : Entry::Job + def self.find_type(name, config) + self.all_types.find do |type| + type.matching?(name, config) + end end # rubocop: disable CodeReuse/ActiveRecord def compose!(deps = nil) super do @config.each do |name, config| - node = node_type(name) + node = self.class.find_type(name, config) + next unless node factory = ::Gitlab::Config::Entry::Factory.new(node) .value(config || {}) diff --git a/lib/gitlab/ci/config/entry/root.rb b/lib/gitlab/ci/config/entry/root.rb new file mode 100644 index 00000000000..0589ad3edf9 --- /dev/null +++ b/lib/gitlab/ci/config/entry/root.rb @@ -0,0 +1,146 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # This class represents a global entry - root Entry for entire + # GitLab CI Configuration file. + # + class Root < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable + + ALLOWED_KEYS = %i[default include before_script image services + after_script variables stages types cache].freeze + + validations do + validates :config, allowed_keys: ALLOWED_KEYS + end + + # reserved: + # defines whether the node name is reserved + # the reserved name cannot be used a job name + # reserved should not be used as it will make + # breaking change to `.gitlab-ci.yml` + + entry :default, Entry::Default, + description: 'Default configuration for all jobs.', + default: {} + + entry :include, Entry::Includes, + description: 'List of external YAML files to include.', + reserved: true + + entry :before_script, Entry::Script, + description: 'Script that will be executed before each job.', + reserved: true + + entry :image, Entry::Image, + description: 'Docker image that will be used to execute jobs.', + reserved: true + + entry :services, Entry::Services, + description: 'Docker images that will be linked to the container.', + reserved: true + + entry :after_script, Entry::Script, + description: 'Script that will be executed after each job.', + reserved: true + + entry :variables, Entry::Variables, + description: 'Environment variables that will be used.', + reserved: true + + entry :stages, Entry::Stages, + description: 'Configuration of stages for this pipeline.', + reserved: true + + entry :types, Entry::Stages, + description: 'Deprecated: stages for this pipeline.', + reserved: true + + entry :cache, Entry::Cache, + description: 'Configure caching between build jobs.', + reserved: true + + helpers :default, :jobs, :stages, :types, :variables + + delegate :before_script_value, + :image_value, + :services_value, + :after_script_value, + :cache_value, to: :default + + attr_reader :jobs_config + + class << self + include ::Gitlab::Utils::StrongMemoize + + def reserved_nodes_names + strong_memoize(:reserved_nodes_names) do + self.nodes.select do |_, node| + node.reserved? + end.keys + end + end + end + + def initialize(config, **metadata) + super do + filter_jobs! + end + end + + def compose!(_deps = nil) + super(self) do + compose_deprecated_entries! + compose_jobs! + end + end + + def default + self[:default] + end + + private + + # rubocop: disable CodeReuse/ActiveRecord + def compose_jobs! + factory = ::Gitlab::Config::Entry::Factory.new(Entry::Jobs) + .value(jobs_config) + .with(key: :jobs, parent: self, + description: 'Jobs definition for this pipeline') + + @entries[:jobs] = factory.create! + end + # rubocop: enable CodeReuse/ActiveRecord + + def compose_deprecated_entries! + ## + # Deprecated `:types` key workaround - if types are defined and + # stages are not defined we use types definition as stages. + # + if types_defined? && !stages_defined? + @entries[:stages] = @entries[:types] + end + + @entries.delete(:types) + end + + def filter_jobs! + return unless @config.is_a?(Hash) + + @jobs_config = @config + .except(*self.class.reserved_nodes_names) # rubocop: disable CodeReuse/ActiveRecord + .select do |name, config| + Entry::Jobs.find_type(name, config).present? + end + + @config = @config.except(*@jobs_config.keys) # rubocop: disable CodeReuse/ActiveRecord + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/normalizer.rb b/lib/gitlab/ci/config/normalizer.rb index 191f5d09645..09f9bf5f69f 100644 --- a/lib/gitlab/ci/config/normalizer.rb +++ b/lib/gitlab/ci/config/normalizer.rb @@ -4,61 +4,63 @@ module Gitlab module Ci class Config class Normalizer + include Gitlab::Utils::StrongMemoize + def initialize(jobs_config) @jobs_config = jobs_config end def normalize_jobs - extract_parallelized_jobs! - return @jobs_config if @parallelized_jobs.empty? + return @jobs_config if parallelized_jobs.empty? + + expand_parallelize_jobs do |job_name, config| + if config[:dependencies] + config[:dependencies] = expand_names(config[:dependencies]) + end - parallelized_config = parallelize_jobs - parallelize_dependencies(parallelized_config) + if config[:needs] + config[:needs] = expand_names(config[:needs]) + end + + config + end end private - def extract_parallelized_jobs! - @parallelized_jobs = {} + def expand_names(job_names) + return unless job_names - @jobs_config.each do |job_name, config| - if config[:parallel] - @parallelized_jobs[job_name] = self.class.parallelize_job_names(job_name, config[:parallel]) - end + job_names.flat_map do |job_name| + parallelized_jobs[job_name.to_sym] || job_name end - - @parallelized_jobs end - def parallelize_jobs - @jobs_config.each_with_object({}) do |(job_name, config), hash| - if @parallelized_jobs.key?(job_name) - @parallelized_jobs[job_name].each { |name, index| hash[name.to_sym] = config.merge(name: name, instance: index) } - else - hash[job_name] = config - end + def parallelized_jobs + strong_memoize(:parallelized_jobs) do + @jobs_config.each_with_object({}) do |(job_name, config), hash| + next unless config[:parallel] - hash + hash[job_name] = self.class.parallelize_job_names(job_name, config[:parallel]) + end end end - def parallelize_dependencies(parallelized_config) - parallelized_job_names = @parallelized_jobs.keys.map(&:to_s) - parallelized_config.each_with_object({}) do |(job_name, config), hash| - if config[:dependencies] && (intersection = config[:dependencies] & parallelized_job_names).any? - parallelized_deps = intersection.map { |dep| @parallelized_jobs[dep.to_sym].map(&:first) }.flatten - deps = config[:dependencies] - intersection + parallelized_deps - hash[job_name] = config.merge(dependencies: deps) + def expand_parallelize_jobs + @jobs_config.each_with_object({}) do |(job_name, config), hash| + if parallelized_jobs.key?(job_name) + parallelized_jobs[job_name].each_with_index do |name, index| + hash[name.to_sym] = + yield(name, config.merge(name: name, instance: index + 1)) + end else - hash[job_name] = config + hash[job_name] = yield(job_name, config) end - - hash end end def self.parallelize_job_names(name, total) - Array.new(total) { |index| ["#{name} #{index + 1}/#{total}", index + 1] } + Array.new(total) { |index| "#{name} #{index + 1}/#{total}" } end end end diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index c911bfa7ff6..afad391e8e0 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -20,6 +20,12 @@ module Gitlab end end + def uses_unsupported_legacy_trigger? + trigger_request.present? && + trigger_request.trigger.legacy? && + !trigger_request.trigger.supports_legacy_tokens? + end + def branch_exists? strong_memoize(:is_branch) do project.repository.branch_exists?(ref) diff --git a/lib/gitlab/ci/pipeline/chain/populate.rb b/lib/gitlab/ci/pipeline/chain/populate.rb index 0405292a25b..65029f5ce7f 100644 --- a/lib/gitlab/ci/pipeline/chain/populate.rb +++ b/lib/gitlab/ci/pipeline/chain/populate.rb @@ -23,12 +23,17 @@ module Gitlab @command.seeds_block&.call(pipeline) ## - # Populate pipeline with all stages, and stages with builds. + # Gather all runtime build/stage errors # - pipeline.stage_seeds.each do |stage| - pipeline.stages << stage.to_resource + if seeds_errors = pipeline.stage_seeds.flat_map(&:errors).compact.presence + return error(seeds_errors.join("\n")) end + ## + # Populate pipeline with all stages, and stages with builds. + # + pipeline.stages = pipeline.stage_seeds.map(&:to_resource) + if pipeline.stages.none? return error('No stages / jobs for this pipeline.') end diff --git a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb index aaa3daddcc5..357a1d55b3b 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb @@ -14,6 +14,10 @@ module Gitlab return error('Pipelines are disabled!') end + if @command.uses_unsupported_legacy_trigger? + return error('Trigger token is invalid because is not owned by any user') + end + unless allowed_to_trigger_pipeline? if can?(current_user, :create_pipeline, project) return error("Insufficient permissions for protected ref '#{command.ref}'") diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb index d7f47c0e7e6..942e4e55323 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb @@ -13,16 +13,6 @@ module Gitlab regexp = @right.evaluate(variables) regexp.scan(text.to_s).any? - - if ci_variables_complex_expressions? - # return offset of first match, or nil if no matches - if match = regexp.scan(text.to_s).first - text.to_s.index(match) - end - else - # return true or false - regexp.scan(text.to_s).any? - end end def self.build(_value, behind, ahead) @@ -32,12 +22,6 @@ module Gitlab def self.precedence 10 # See: https://ruby-doc.org/core-2.5.0/doc/syntax/precedence_rdoc.html end - - private - - def ci_variables_complex_expressions? - Feature.enabled?(:ci_variables_complex_expressions) - end end end end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb index eb32dffc454..0212fa9d661 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb @@ -8,11 +8,10 @@ module Gitlab require_dependency 're2' class Pattern < Lexeme::Value - PATTERN = %r{^/.+/[ismU]*$}.freeze - NEW_PATTERN = %r{^\/([^\/]|\\/)+[^\\]\/[ismU]*}.freeze + PATTERN = %r{^\/([^\/]|\\/)+[^\\]\/[ismU]*}.freeze def initialize(regexp) - @value = self.class.eager_matching_with_escape_characters? ? regexp.gsub(/\\\//, '/') : regexp + @value = regexp.gsub(/\\\//, '/') unless Gitlab::UntrustedRegexp::RubySyntax.valid?(@value) raise Lexer::SyntaxError, 'Invalid regular expression!' @@ -26,16 +25,12 @@ module Gitlab end def self.pattern - eager_matching_with_escape_characters? ? NEW_PATTERN : PATTERN + PATTERN end def self.build(string) new(string) end - - def self.eager_matching_with_escape_characters? - Feature.enabled?(:ci_variables_complex_expressions) - end end end end diff --git a/lib/gitlab/ci/pipeline/expression/lexer.rb b/lib/gitlab/ci/pipeline/expression/lexer.rb index 23aa1ca4a36..7d7582612f9 100644 --- a/lib/gitlab/ci/pipeline/expression/lexer.rb +++ b/lib/gitlab/ci/pipeline/expression/lexer.rb @@ -17,17 +17,6 @@ module Gitlab Expression::Lexeme::Equals, Expression::Lexeme::Matches, Expression::Lexeme::NotEquals, - Expression::Lexeme::NotMatches - ].freeze - - NEW_LEXEMES = [ - Expression::Lexeme::Variable, - Expression::Lexeme::String, - Expression::Lexeme::Pattern, - Expression::Lexeme::Null, - Expression::Lexeme::Equals, - Expression::Lexeme::Matches, - Expression::Lexeme::NotEquals, Expression::Lexeme::NotMatches, Expression::Lexeme::And, Expression::Lexeme::Or @@ -58,7 +47,7 @@ module Gitlab return tokens if @scanner.eos? - lexeme = available_lexemes.find do |type| + lexeme = LEXEMES.find do |type| type.scan(@scanner).tap do |token| tokens.push(token) if token.present? end @@ -71,10 +60,6 @@ module Gitlab raise Lexer::SyntaxError, 'Too many tokens!' end - - def available_lexemes - Feature.enabled?(:ci_variables_complex_expressions) ? NEW_LEXEMES : LEXEMES - end end end end diff --git a/lib/gitlab/ci/pipeline/expression/parser.rb b/lib/gitlab/ci/pipeline/expression/parser.rb index 0679ed329e5..edb55edf356 100644 --- a/lib/gitlab/ci/pipeline/expression/parser.rb +++ b/lib/gitlab/ci/pipeline/expression/parser.rb @@ -13,39 +13,6 @@ module Gitlab end def tree - if Feature.enabled?(:ci_variables_complex_expressions) - rpn_parse_tree - else - reverse_descent_parse_tree - end - end - - def self.seed(statement) - new(Expression::Lexer.new(statement).tokens) - end - - private - - # This produces a reverse descent parse tree. - # It does not support precedence of operators. - def reverse_descent_parse_tree - while token = @tokens.next - case token.type - when :operator - token.build(@nodes.pop, tree).tap do |node| - @nodes.push(node) - end - when :value - token.build.tap do |leaf| - @nodes.push(leaf) - end - end - end - rescue StopIteration - @nodes.last || Lexeme::Null.new - end - - def rpn_parse_tree results = [] tokens_rpn.each do |token| @@ -70,6 +37,12 @@ module Gitlab results.pop end + def self.seed(statement) + new(Expression::Lexer.new(statement).tokens) + end + + private + # Parse the expression into Reverse Polish Notation # (See: Shunting-yard algorithm) def tokens_rpn diff --git a/lib/gitlab/ci/pipeline/seed/base.rb b/lib/gitlab/ci/pipeline/seed/base.rb index 1fd3a61017f..e9e22569ae0 100644 --- a/lib/gitlab/ci/pipeline/seed/base.rb +++ b/lib/gitlab/ci/pipeline/seed/base.rb @@ -13,6 +13,10 @@ module Gitlab raise NotImplementedError end + def errors + raise NotImplementedError + end + def to_resource raise NotImplementedError end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index d8296940a04..7ec03d132c0 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -9,9 +9,15 @@ module Gitlab delegate :dig, to: :@attributes - def initialize(pipeline, attributes) + # When the `ci_dag_limit_needs` is enabled it uses the lower limit + LOW_NEEDS_LIMIT = 5 + HARD_NEEDS_LIMIT = 50 + + def initialize(pipeline, attributes, previous_stages) @pipeline = pipeline @attributes = attributes + @previous_stages = previous_stages + @needs_attributes = dig(:needs_attributes) @only = Gitlab::Ci::Build::Policy .fabricate(attributes.delete(:only)) @@ -19,10 +25,22 @@ module Gitlab .fabricate(attributes.delete(:except)) end + def name + dig(:name) + end + def included? strong_memoize(:inclusion) do - @only.all? { |spec| spec.satisfied_by?(@pipeline, self) } && - @except.none? { |spec| spec.satisfied_by?(@pipeline, self) } + all_of_only? && + none_of_except? + end + end + + def errors + return unless included? + + strong_memoize(:errors) do + needs_errors end end @@ -39,7 +57,10 @@ module Gitlab end def bridge? - @attributes.to_h.dig(:options, :trigger).present? + attributes_hash = @attributes.to_h + attributes_hash.dig(:options, :trigger).present? || + (attributes_hash.dig(:options, :bridge_needs).instance_of?(Hash) && + attributes_hash.dig(:options, :bridge_needs, :pipeline).present?) end def to_resource @@ -51,6 +72,43 @@ module Gitlab end end end + + private + + def all_of_only? + @only.all? { |spec| spec.satisfied_by?(@pipeline, self) } + end + + def none_of_except? + @except.none? { |spec| spec.satisfied_by?(@pipeline, self) } + end + + def needs_errors + return if @needs_attributes.nil? + + if @needs_attributes.size > max_needs_allowed + return [ + "#{name}: one job can only need #{max_needs_allowed} others, but you have listed #{@needs_attributes.size}. " \ + "See needs keyword documentation for more details" + ] + end + + @needs_attributes.flat_map do |need| + result = @previous_stages.any? do |stage| + stage.seeds_names.include?(need[:name]) + end + + "#{name}: needs '#{need[:name]}'" unless result + end.compact + end + + def max_needs_allowed + if Feature.enabled?(:ci_dag_limit_needs, @project, default_enabled: true) + LOW_NEEDS_LIMIT + else + HARD_NEEDS_LIMIT + end + end end end end diff --git a/lib/gitlab/ci/pipeline/seed/stage.rb b/lib/gitlab/ci/pipeline/seed/stage.rb index 9c15064756a..b600df2f656 100644 --- a/lib/gitlab/ci/pipeline/seed/stage.rb +++ b/lib/gitlab/ci/pipeline/seed/stage.rb @@ -10,12 +10,13 @@ module Gitlab delegate :size, to: :seeds delegate :dig, to: :seeds - def initialize(pipeline, attributes) + def initialize(pipeline, attributes, previous_stages) @pipeline = pipeline @attributes = attributes + @previous_stages = previous_stages @builds = attributes.fetch(:builds).map do |attributes| - Seed::Build.new(@pipeline, attributes) + Seed::Build.new(@pipeline, attributes, previous_stages) end end @@ -32,6 +33,18 @@ module Gitlab end end + def errors + strong_memoize(:errors) do + seeds.flat_map(&:errors).compact + end + end + + def seeds_names + strong_memoize(:seeds_names) do + seeds.map(&:name).to_set + end + end + def included? seeds.any? end @@ -39,13 +52,7 @@ module Gitlab def to_resource strong_memoize(:stage) do ::Ci::Stage.new(attributes).tap do |stage| - seeds.each do |seed| - if seed.bridge? - stage.bridges << seed.to_resource - else - stage.builds << seed.to_resource - end - end + stage.statuses = seeds.map(&:to_resource) end end end diff --git a/lib/gitlab/ci/status/build/manual.rb b/lib/gitlab/ci/status/build/manual.rb index d01b09f1398..df572188194 100644 --- a/lib/gitlab/ci/status/build/manual.rb +++ b/lib/gitlab/ci/status/build/manual.rb @@ -10,7 +10,7 @@ module Gitlab image: 'illustrations/manual_action.svg', size: 'svg-394', title: _('This job requires a manual action'), - content: _('This job depends on a user to trigger its process. Often they are used to deploy code to production environments') + content: _('This job requires manual intervention to start. Before starting this job, you can add variables below for last-minute configuration changes.') } end diff --git a/lib/gitlab/ci/status/factory.rb b/lib/gitlab/ci/status/factory.rb index 3446644eff8..2a0bf060c9b 100644 --- a/lib/gitlab/ci/status/factory.rb +++ b/lib/gitlab/ci/status/factory.rb @@ -34,11 +34,9 @@ module Gitlab def extended_statuses return @extended_statuses if defined?(@extended_statuses) - groups = self.class.extended_statuses.map do |group| + @extended_statuses = self.class.extended_statuses.flat_map do |group| Array(group).find { |status| status.matches?(@subject, @user) } - end - - @extended_statuses = groups.flatten.compact + end.compact end def self.extended_statuses diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index 65a6630365d..5c1c0c142e5 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -50,13 +50,12 @@ variables: POSTGRES_DB: $CI_ENVIRONMENT_SLUG POSTGRES_VERSION: 9.6.2 - KUBERNETES_VERSION: 1.11.10 - HELM_VERSION: 2.14.0 - DOCKER_DRIVER: overlay2 ROLLOUT_RESOURCE_TYPE: deployment + DOCKER_TLS_CERTDIR: "" # https://gitlab.com/gitlab-org/gitlab-runner/issues/4501 + stages: - build - test @@ -74,16 +73,16 @@ stages: - cleanup include: - - template: Jobs/Build.gitlab-ci.yml - - template: Jobs/Test.gitlab-ci.yml - - template: Jobs/Code-Quality.gitlab-ci.yml - - template: Jobs/Deploy.gitlab-ci.yml - - template: Jobs/Browser-Performance-Testing.gitlab-ci.yml - - template: Security/DAST.gitlab-ci.yml - - template: Security/Container-Scanning.gitlab-ci.yml - - template: Security/Dependency-Scanning.gitlab-ci.yml - - template: Security/License-Management.gitlab-ci.yml - - template: Security/SAST.gitlab-ci.yml + - template: Jobs/Build.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml + - template: Jobs/Test.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml + - template: Jobs/Code-Quality.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml + - template: Jobs/Deploy.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml + - template: Jobs/Browser-Performance-Testing.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml + - template: Security/DAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml + - template: Security/Container-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml + - template: Security/Dependency-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml + - template: Security/License-Management.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Security/License-Management.gitlab-ci.yml + - template: Security/SAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml # Override DAST job to exclude master branch dast: diff --git a/lib/gitlab/ci/templates/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Code-Quality.gitlab-ci.yml index 10b25af904f..b4ccf96b859 100644 --- a/lib/gitlab/ci/templates/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Code-Quality.gitlab-ci.yml @@ -1,17 +1,2 @@ -code_quality: - image: docker:stable - allow_failure: true - services: - - docker:stable-dind - variables: - DOCKER_DRIVER: overlay2 - script: - - docker run - --env SOURCE_CODE="$PWD" - --volume "$PWD":/code - --volume /var/run/docker.sock:/var/run/docker.sock - "registry.gitlab.com/gitlab-org/security-products/codequality:11-8-stable" /code - artifacts: - reports: - codequality: gl-code-quality-report.json - expire_in: 1 week +include: + template: Jobs/Code-Quality.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml index a09217e8cf0..b0a79950667 100644 --- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml @@ -2,6 +2,8 @@ performance: stage: performance image: docker:stable allow_failure: true + variables: + DOCKER_TLS_CERTDIR: "" services: - docker:stable-dind script: diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index 18f7290e1d9..8061da968ed 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,6 +1,8 @@ build: stage: build image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image/master:stable" + variables: + DOCKER_TLS_CERTDIR: "" services: - docker:stable-dind script: diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index b09a24d8e22..3adc6a72874 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -4,21 +4,26 @@ code_quality: allow_failure: true services: - docker:stable-dind + variables: + DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" script: - - export CQ_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/') - | if ! docker info &>/dev/null; then if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then export DOCKER_HOST='tcp://localhost:2375' fi fi - - | - docker run --env SOURCE_CODE="$PWD" \ - --volume "$PWD":/code \ - --volume /var/run/docker.sock:/var/run/docker.sock \ - "registry.gitlab.com/gitlab-org/security-products/codequality:$CQ_VERSION" /code + - docker run + --env SOURCE_CODE="$PWD" + --volume "$PWD":/code + --volume /var/run/docker.sock:/var/run/docker.sock + "registry.gitlab.com/gitlab-org/security-products/codequality:12-0-stable" /code artifacts: - paths: [gl-code-quality-report.json] + reports: + codequality: gl-code-quality-report.json + expire_in: 1 week + dependencies: [] only: - branches - tags diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index dcf8254ef94..a8ec2d4781d 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,14 +1,17 @@ +.auto-deploy: + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.1.0" + review: + extends: .auto-deploy stage: review script: - - check_kube_domain - - install_dependencies - - download_chart - - ensure_namespace - - initialize_tiller - - create_secret - - deploy - - persist_environment_url + - auto-deploy check_kube_domain + - auto-deploy download_chart + - auto-deploy ensure_namespace + - auto-deploy initialize_tiller + - auto-deploy create_secret + - auto-deploy deploy + - auto-deploy persist_environment_url environment: name: review/$CI_COMMIT_REF_NAME url: http://$CI_PROJECT_ID-$CI_ENVIRONMENT_SLUG.$KUBE_INGRESS_BASE_DOMAIN @@ -27,13 +30,13 @@ review: - $REVIEW_DISABLED stop_review: + extends: .auto-deploy stage: cleanup variables: GIT_STRATEGY: none script: - - install_dependencies - - initialize_tiller - - delete + - auto-deploy initialize_tiller + - auto-deploy delete environment: name: review/$CI_COMMIT_REF_NAME action: stop @@ -57,15 +60,15 @@ stop_review: # STAGING_ENABLED. staging: + extends: .auto-deploy stage: staging script: - - check_kube_domain - - install_dependencies - - download_chart - - ensure_namespace - - initialize_tiller - - create_secret - - deploy + - auto-deploy check_kube_domain + - auto-deploy download_chart + - auto-deploy ensure_namespace + - auto-deploy initialize_tiller + - auto-deploy create_secret + - auto-deploy deploy environment: name: staging url: http://$CI_PROJECT_PATH_SLUG-staging.$KUBE_INGRESS_BASE_DOMAIN @@ -81,15 +84,15 @@ staging: # CANARY_ENABLED. canary: + extends: .auto-deploy stage: canary script: - - check_kube_domain - - install_dependencies - - download_chart - - ensure_namespace - - initialize_tiller - - create_secret - - deploy canary + - auto-deploy check_kube_domain + - auto-deploy download_chart + - auto-deploy ensure_namespace + - auto-deploy initialize_tiller + - auto-deploy create_secret + - auto-deploy deploy canary environment: name: production url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN @@ -102,18 +105,18 @@ canary: - $CANARY_ENABLED .production: &production_template + extends: .auto-deploy stage: production script: - - check_kube_domain - - install_dependencies - - download_chart - - ensure_namespace - - initialize_tiller - - create_secret - - deploy - - delete canary - - delete rollout - - persist_environment_url + - auto-deploy check_kube_domain + - auto-deploy download_chart + - auto-deploy ensure_namespace + - auto-deploy initialize_tiller + - auto-deploy create_secret + - auto-deploy deploy + - auto-deploy delete canary + - auto-deploy delete rollout + - auto-deploy persist_environment_url environment: name: production url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN @@ -152,17 +155,17 @@ production_manual: # This job implements incremental rollout on for every push to `master`. .rollout: &rollout_template + extends: .auto-deploy script: - - check_kube_domain - - install_dependencies - - download_chart - - ensure_namespace - - initialize_tiller - - create_secret - - deploy rollout $ROLLOUT_PERCENTAGE - - scale stable $((100-ROLLOUT_PERCENTAGE)) - - delete canary - - persist_environment_url + - auto-deploy check_kube_domain + - auto-deploy download_chart + - auto-deploy ensure_namespace + - auto-deploy initialize_tiller + - auto-deploy create_secret + - auto-deploy deploy rollout $ROLLOUT_PERCENTAGE + - auto-deploy scale stable $((100-ROLLOUT_PERCENTAGE)) + - auto-deploy delete canary + - auto-deploy persist_environment_url environment: name: production url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN @@ -240,331 +243,3 @@ rollout 100%: <<: *manual_rollout_template <<: *production_template allow_failure: false - -.deploy_helpers: &deploy_helpers | - [[ "$TRACE" ]] && set -x - auto_database_url=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${CI_ENVIRONMENT_SLUG}-postgres:5432/${POSTGRES_DB} - export DATABASE_URL=${DATABASE_URL-$auto_database_url} - export TILLER_NAMESPACE=$KUBE_NAMESPACE - # Extract "MAJOR.MINOR" from CI_SERVER_VERSION and generate "MAJOR-MINOR-stable" for Security Products - - function get_replicas() { - track="${1:-stable}" - percentage="${2:-100}" - - env_track=$( echo $track | tr -s '[:lower:]' '[:upper:]' ) - env_slug=$( echo ${CI_ENVIRONMENT_SLUG//-/_} | tr -s '[:lower:]' '[:upper:]' ) - - if [[ "$track" == "stable" ]] || [[ "$track" == "rollout" ]]; then - # for stable track get number of replicas from `PRODUCTION_REPLICAS` - eval new_replicas=\$${env_slug}_REPLICAS - if [[ -z "$new_replicas" ]]; then - new_replicas=$REPLICAS - fi - else - # for all tracks get number of replicas from `CANARY_PRODUCTION_REPLICAS` - eval new_replicas=\$${env_track}_${env_slug}_REPLICAS - if [[ -z "$new_replicas" ]]; then - eval new_replicas=\${env_track}_REPLICAS - fi - fi - - replicas="${new_replicas:-1}" - replicas="$(($replicas * $percentage / 100))" - - # always return at least one replicas - if [[ $replicas -gt 0 ]]; then - echo "$replicas" - else - echo 1 - fi - } - - # Extracts variables prefixed with K8S_SECRET_ - # and creates a Kubernetes secret. - # - # e.g. If we have the following environment variables: - # K8S_SECRET_A=value1 - # K8S_SECRET_B=multi\ word\ value - # - # Then we will create a secret with the following key-value pairs: - # data: - # A: dmFsdWUxCg== - # B: bXVsdGkgd29yZCB2YWx1ZQo= - function create_application_secret() { - track="${1-stable}" - export APPLICATION_SECRET_NAME=$(application_secret_name "$track") - - env | sed -n "s/^K8S_SECRET_\(.*\)$/\1/p" > k8s_prefixed_variables - - kubectl create secret \ - -n "$KUBE_NAMESPACE" generic "$APPLICATION_SECRET_NAME" \ - --from-env-file k8s_prefixed_variables -o yaml --dry-run | - kubectl replace -n "$KUBE_NAMESPACE" --force -f - - - export APPLICATION_SECRET_CHECKSUM=$(cat k8s_prefixed_variables | sha256sum | cut -d ' ' -f 1) - - rm k8s_prefixed_variables - } - - function deploy_name() { - name="$CI_ENVIRONMENT_SLUG" - track="${1-stable}" - - if [[ "$track" != "stable" ]]; then - name="$name-$track" - fi - - echo $name - } - - function application_secret_name() { - track="${1-stable}" - name=$(deploy_name "$track") - - echo "${name}-secret" - } - - function deploy() { - track="${1-stable}" - percentage="${2:-100}" - name=$(deploy_name "$track") - - if [[ -z "$CI_COMMIT_TAG" ]]; then - image_repository=${CI_APPLICATION_REPOSITORY:-$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG} - image_tag=${CI_APPLICATION_TAG:-$CI_COMMIT_SHA} - else - image_repository=${CI_APPLICATION_REPOSITORY:-$CI_REGISTRY_IMAGE} - image_tag=${CI_APPLICATION_TAG:-$CI_COMMIT_TAG} - fi - - service_enabled="true" - postgres_enabled="$POSTGRES_ENABLED" - - # if track is different than stable, - # re-use all attached resources - if [[ "$track" != "stable" ]]; then - service_enabled="false" - postgres_enabled="false" - fi - - replicas=$(get_replicas "$track" "$percentage") - - if [[ "$CI_PROJECT_VISIBILITY" != "public" ]]; then - secret_name='gitlab-registry' - else - secret_name='' - fi - - create_application_secret "$track" - - env_slug=$(echo ${CI_ENVIRONMENT_SLUG//-/_} | tr -s '[:lower:]' '[:upper:]') - eval env_ADDITIONAL_HOSTS=\$${env_slug}_ADDITIONAL_HOSTS - if [ -n "$env_ADDITIONAL_HOSTS" ]; then - additional_hosts="{$env_ADDITIONAL_HOSTS}" - elif [ -n "$ADDITIONAL_HOSTS" ]; then - additional_hosts="{$ADDITIONAL_HOSTS}" - fi - - if [[ -n "$DB_INITIALIZE" && -z "$(helm ls -q "^$name$")" ]]; then - echo "Deploying first release with database initialization..." - helm upgrade --install \ - --wait \ - --set service.enabled="$service_enabled" \ - --set gitlab.app="$CI_PROJECT_PATH_SLUG" \ - --set gitlab.env="$CI_ENVIRONMENT_SLUG" \ - --set releaseOverride="$CI_ENVIRONMENT_SLUG" \ - --set image.repository="$image_repository" \ - --set image.tag="$image_tag" \ - --set image.pullPolicy=IfNotPresent \ - --set image.secrets[0].name="$secret_name" \ - --set application.track="$track" \ - --set application.database_url="$DATABASE_URL" \ - --set application.secretName="$APPLICATION_SECRET_NAME" \ - --set application.secretChecksum="$APPLICATION_SECRET_CHECKSUM" \ - --set service.commonName="le-$CI_PROJECT_ID.$KUBE_INGRESS_BASE_DOMAIN" \ - --set service.url="$CI_ENVIRONMENT_URL" \ - --set service.additionalHosts="$additional_hosts" \ - --set replicaCount="$replicas" \ - --set postgresql.enabled="$postgres_enabled" \ - --set postgresql.nameOverride="postgres" \ - --set postgresql.postgresUser="$POSTGRES_USER" \ - --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \ - --set postgresql.postgresDatabase="$POSTGRES_DB" \ - --set postgresql.imageTag="$POSTGRES_VERSION" \ - --set application.initializeCommand="$DB_INITIALIZE" \ - $HELM_UPGRADE_EXTRA_ARGS \ - --namespace="$KUBE_NAMESPACE" \ - "$name" \ - chart/ - - echo "Deploying second release..." - helm upgrade --reuse-values \ - --wait \ - --set application.initializeCommand="" \ - --set application.migrateCommand="$DB_MIGRATE" \ - $HELM_UPGRADE_EXTRA_ARGS \ - --namespace="$KUBE_NAMESPACE" \ - "$name" \ - chart/ - else - echo "Deploying new release..." - helm upgrade --install \ - --wait \ - --set service.enabled="$service_enabled" \ - --set gitlab.app="$CI_PROJECT_PATH_SLUG" \ - --set gitlab.env="$CI_ENVIRONMENT_SLUG" \ - --set releaseOverride="$CI_ENVIRONMENT_SLUG" \ - --set image.repository="$image_repository" \ - --set image.tag="$image_tag" \ - --set image.pullPolicy=IfNotPresent \ - --set image.secrets[0].name="$secret_name" \ - --set application.track="$track" \ - --set application.database_url="$DATABASE_URL" \ - --set application.secretName="$APPLICATION_SECRET_NAME" \ - --set application.secretChecksum="$APPLICATION_SECRET_CHECKSUM" \ - --set service.commonName="le-$CI_PROJECT_ID.$KUBE_INGRESS_BASE_DOMAIN" \ - --set service.url="$CI_ENVIRONMENT_URL" \ - --set service.additionalHosts="$additional_hosts" \ - --set replicaCount="$replicas" \ - --set postgresql.enabled="$postgres_enabled" \ - --set postgresql.nameOverride="postgres" \ - --set postgresql.postgresUser="$POSTGRES_USER" \ - --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \ - --set postgresql.postgresDatabase="$POSTGRES_DB" \ - --set postgresql.imageTag="$POSTGRES_VERSION" \ - --set application.migrateCommand="$DB_MIGRATE" \ - $HELM_UPGRADE_EXTRA_ARGS \ - --namespace="$KUBE_NAMESPACE" \ - "$name" \ - chart/ - fi - - if [[ -z "$ROLLOUT_STATUS_DISABLED" ]]; then - kubectl rollout status -n "$KUBE_NAMESPACE" -w "$ROLLOUT_RESOURCE_TYPE/$name" - fi - } - - function scale() { - track="${1-stable}" - percentage="${2-100}" - name=$(deploy_name "$track") - - replicas=$(get_replicas "$track" "$percentage") - - if [[ -n "$(helm ls -q "^$name$")" ]]; then - helm upgrade --reuse-values \ - --wait \ - --set replicaCount="$replicas" \ - --namespace="$KUBE_NAMESPACE" \ - "$name" \ - chart/ - fi - } - - function install_dependencies() { - apk add -U openssl curl tar gzip bash ca-certificates git - curl -sSL -o /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub - curl -sSL -O https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.28-r0/glibc-2.28-r0.apk - apk add glibc-2.28-r0.apk - rm glibc-2.28-r0.apk - - curl -sS "https://kubernetes-helm.storage.googleapis.com/helm-v${HELM_VERSION}-linux-amd64.tar.gz" | tar zx - mv linux-amd64/helm /usr/bin/ - mv linux-amd64/tiller /usr/bin/ - helm version --client - tiller -version - - curl -sSL -o /usr/bin/kubectl "https://storage.googleapis.com/kubernetes-release/release/v${KUBERNETES_VERSION}/bin/linux/amd64/kubectl" - chmod +x /usr/bin/kubectl - kubectl version --client - } - - function download_chart() { - if [[ ! -d chart ]]; then - auto_chart=${AUTO_DEVOPS_CHART:-gitlab/auto-deploy-app} - auto_chart_name=$(basename $auto_chart) - auto_chart_name=${auto_chart_name%.tgz} - auto_chart_name=${auto_chart_name%.tar.gz} - else - auto_chart="chart" - auto_chart_name="chart" - fi - - helm init --client-only - helm repo add ${AUTO_DEVOPS_CHART_REPOSITORY_NAME:-gitlab} ${AUTO_DEVOPS_CHART_REPOSITORY:-https://charts.gitlab.io} ${AUTO_DEVOPS_CHART_REPOSITORY_USERNAME:+"--username" "$AUTO_DEVOPS_CHART_REPOSITORY_USERNAME"} ${AUTO_DEVOPS_CHART_REPOSITORY_PASSWORD:+"--password" "$AUTO_DEVOPS_CHART_REPOSITORY_PASSWORD"} - if [[ ! -d "$auto_chart" ]]; then - helm fetch ${auto_chart} --untar - fi - if [ "$auto_chart_name" != "chart" ]; then - mv ${auto_chart_name} chart - fi - - helm dependency update chart/ - helm dependency build chart/ - } - - function ensure_namespace() { - kubectl get namespace "$KUBE_NAMESPACE" || kubectl create namespace "$KUBE_NAMESPACE" - } - - function check_kube_domain() { - if [[ -z "$KUBE_INGRESS_BASE_DOMAIN" ]]; then - echo "In order to deploy or use Review Apps," - echo "KUBE_INGRESS_BASE_DOMAIN variables must be set" - echo "From 11.8, you can set KUBE_INGRESS_BASE_DOMAIN in cluster settings" - echo "or by defining a variable at group or project level." - echo "You can also manually add it in .gitlab-ci.yml" - false - else - true - fi - } - - function initialize_tiller() { - echo "Checking Tiller..." - - export HELM_HOST="localhost:44134" - tiller -listen ${HELM_HOST} -alsologtostderr > /dev/null 2>&1 & - echo "Tiller is listening on ${HELM_HOST}" - - if ! helm version --debug; then - echo "Failed to init Tiller." - return 1 - fi - echo "" - } - - function create_secret() { - echo "Create secret..." - if [[ "$CI_PROJECT_VISIBILITY" == "public" ]]; then - return - fi - - kubectl create secret -n "$KUBE_NAMESPACE" \ - docker-registry gitlab-registry \ - --docker-server="$CI_REGISTRY" \ - --docker-username="${CI_DEPLOY_USER:-$CI_REGISTRY_USER}" \ - --docker-password="${CI_DEPLOY_PASSWORD:-$CI_REGISTRY_PASSWORD}" \ - --docker-email="$GITLAB_USER_EMAIL" \ - -o yaml --dry-run | kubectl replace -n "$KUBE_NAMESPACE" --force -f - - } - - function persist_environment_url() { - echo $CI_ENVIRONMENT_URL > environment_url.txt - } - - function delete() { - track="${1-stable}" - name=$(deploy_name "$track") - - if [[ -n "$(helm ls -q "^$name$")" ]]; then - helm delete --purge "$name" - fi - - secret_name=$(application_secret_name "$track") - kubectl delete secret --ignore-not-found -n "$KUBE_NAMESPACE" "$secret_name" - } - -before_script: - - *deploy_helpers diff --git a/lib/gitlab/ci/templates/Maven.gitlab-ci.yml b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml index 13ab98d3a16..84bb0ff3b33 100644 --- a/lib/gitlab/ci/templates/Maven.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml @@ -1,5 +1,3 @@ -# This file is a template, and might need editing before it works on your project. - # Build JAVA applications using Apache Maven (http://maven.apache.org) # For docker image tags see https://hub.docker.com/_/maven/ # diff --git a/lib/gitlab/ci/templates/PHP.gitlab-ci.yml b/lib/gitlab/ci/templates/PHP.gitlab-ci.yml index b9fee2d5731..25ea20e454f 100644 --- a/lib/gitlab/ci/templates/PHP.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/PHP.gitlab-ci.yml @@ -1,5 +1,5 @@ # Select image from https://hub.docker.com/_/php/ -image: php:7.1.1 +image: php:latest # Select what we should cache between builds cache: diff --git a/lib/gitlab/ci/templates/Packer.gitlab-ci.yml b/lib/gitlab/ci/templates/Packer.gitlab-ci.yml index 83e179f37c3..0a3cf3dcf77 100644 --- a/lib/gitlab/ci/templates/Packer.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Packer.gitlab-ci.yml @@ -1,5 +1,5 @@ image: - name: hashicorp/packer:1.0.4 + name: hashicorp/packer:latest entrypoint: - '/usr/bin/env' - 'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' diff --git a/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml index 0d742aa282d..e7dacd3a1fc 100644 --- a/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml @@ -4,6 +4,7 @@ image: ruby:2.3 variables: JEKYLL_ENV: production + LC_ALL: C.UTF-8 before_script: - bundle install diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index 5372ec6cceb..2afc99d0bf8 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -5,6 +5,7 @@ container_scanning: image: docker:stable variables: DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" # Defining two new variables based on GitLab's CI/CD predefined variables # https://docs.gitlab.com/ee/ci/variables/#predefined-environment-variables CI_APPLICATION_REPOSITORY: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG @@ -22,7 +23,9 @@ container_scanning: DOCKER_SERVICE: docker DOCKER_HOST: tcp://${DOCKER_SERVICE}:2375/ # https://hub.docker.com/r/arminc/clair-local-scan/tags - CLAIR_LOCAL_SCAN_VERSION: v2.0.8_fe9b059d930314b54c78f75afe265955faf4fdc1 + CLAIR_LOCAL_SCAN_VERSION: v2.0.8_0ed98e9ead65a51ba53f7cc53fa5e80c92169207 + CLAIR_EXECUTABLE_VERSION: v12 + CLAIR_EXECUTABLE_SHA: 44f2a3fdd7b0d102c98510e7586f6956edc89ab72c6943980f92f4979f7f4081 ## Disable the proxy for clair-local-scan, otherwise Container Scanning will ## fail when a proxy is used. NO_PROXY: ${DOCKER_SERVICE},localhost @@ -30,7 +33,7 @@ container_scanning: services: - docker:stable-dind script: - - if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then { export DOCKER_SERVICE="localhost" ; export DOCKER_HOST="tcp://${DOCKER_SERVICE}:2375" ; } fi + - if [[ -n "$KUBERNETES_PORT" ]]; then { export DOCKER_SERVICE="localhost" ; export DOCKER_HOST="tcp://${DOCKER_SERVICE}:2375" ; } fi - | if [[ -n "$CI_REGISTRY_USER" ]]; then echo "Logging to GitLab Container Registry with CI credentials..." @@ -41,7 +44,8 @@ container_scanning: - docker run -p 6060:6060 --link db:postgres -d --name clair --restart on-failure arminc/clair-local-scan:${CLAIR_LOCAL_SCAN_VERSION} - apk add -U wget ca-certificates - docker pull ${CI_APPLICATION_REPOSITORY}:${CI_APPLICATION_TAG} - - wget https://github.com/arminc/clair-scanner/releases/download/v8/clair-scanner_linux_amd64 + - wget https://github.com/arminc/clair-scanner/releases/download/${CLAIR_EXECUTABLE_VERSION}/clair-scanner_linux_amd64 + - echo "${CLAIR_EXECUTABLE_SHA} clair-scanner_linux_amd64" | sha256sum -c - mv clair-scanner_linux_amd64 clair-scanner - chmod +x clair-scanner - touch clair-whitelist.yml diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index 8dd9775c583..15b84f1540d 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -9,6 +9,7 @@ dependency_scanning: image: docker:stable variables: DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" allow_failure: true services: - docker:stable-dind @@ -40,6 +41,10 @@ dependency_scanning: DS_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \ DS_PULL_ANALYZER_IMAGE_TIMEOUT \ DS_RUN_ANALYZER_TIMEOUT \ + DS_PYTHON_VERSION \ + DS_PIP_DEPENDENCY_PATH \ + PIP_INDEX_URL \ + PIP_EXTRA_INDEX_URL \ ) \ --volume "$PWD:/code" \ --volume /var/run/docker.sock:/var/run/docker.sock \ diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index 8713b833011..90278122361 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -9,6 +9,7 @@ sast: image: docker:stable variables: DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" allow_failure: true services: - docker:stable-dind @@ -45,15 +46,19 @@ sast: SAST_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \ SAST_PULL_ANALYZER_IMAGE_TIMEOUT \ SAST_RUN_ANALYZER_TIMEOUT \ + SAST_JAVA_VERSION \ ANT_HOME \ ANT_PATH \ GRADLE_PATH \ JAVA_OPTS \ JAVA_PATH \ + JAVA_8_VERSION \ + JAVA_11_VERSION \ MAVEN_CLI_OPTS \ MAVEN_PATH \ MAVEN_REPO_PATH \ SBT_PATH \ + FAIL_NEVER \ ) \ --volume "$PWD:/code" \ --volume /var/run/docker.sock:/var/run/docker.sock \ diff --git a/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml b/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml index a3db2705bf6..280e75d46f5 100644 --- a/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml @@ -8,26 +8,23 @@ stages: - deploy .serverless:build:image: - stage: build image: registry.gitlab.com/gitlab-org/gitlabktl:latest + stage: build script: /usr/bin/gitlabktl app build .serverless:deploy:image: + image: registry.gitlab.com/gitlab-org/gitlabktl:latest stage: deploy - image: gcr.io/triggermesh/tm@sha256:3cfdd470a66b741004fb02354319d79f1598c70117ce79978d2e07e192bfb336 # v0.0.11 environment: development - script: - - echo "$CI_REGISTRY_IMAGE" - - tm -n "$KUBE_NAMESPACE" --config "$KUBECONFIG" deploy service "$CI_PROJECT_NAME" --from-image "$CI_REGISTRY_IMAGE" --wait + script: /usr/bin/gitlabktl app deploy .serverless:build:functions: - stage: build - environment: development image: registry.gitlab.com/gitlab-org/gitlabktl:latest + stage: build script: /usr/bin/gitlabktl serverless build .serverless:deploy:functions: + image: registry.gitlab.com/gitlab-org/gitlabktl:latest stage: deploy environment: development - image: registry.gitlab.com/gitlab-org/gitlabktl:latest script: /usr/bin/gitlabktl serverless deploy diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index dfae260239e..9550bc6d39c 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -5,7 +5,7 @@ module Gitlab class Trace include ::Gitlab::ExclusiveLeaseHelpers - LOCK_TTL = 1.minute + LOCK_TTL = 10.minutes LOCK_RETRIES = 2 LOCK_SLEEP = 0.001.seconds @@ -63,7 +63,15 @@ module Gitlab end def exist? - trace_artifact&.exists? || job.trace_chunks.any? || current_path.present? || old_trace.present? + archived_trace_exist? || live_trace_exist? + end + + def archived_trace_exist? + trace_artifact&.exists? + end + + def live_trace_exist? + job.trace_chunks.any? || current_path.present? || old_trace.present? end def read @@ -118,7 +126,7 @@ module Gitlab raise AlreadyArchivedError, 'Could not write to the archived trace' elsif current_path File.open(current_path, mode) - elsif Feature.enabled?('ci_enable_live_trace') + elsif Feature.enabled?('ci_enable_live_trace', job.project) Gitlab::Ci::Trace::ChunkedIO.new(job) else File.open(ensure_path, mode) @@ -167,7 +175,7 @@ module Gitlab def clone_file!(src_stream, temp_dir) FileUtils.mkdir_p(temp_dir) - Dir.mktmpdir('tmp-trace', temp_dir) do |dir_path| + Dir.mktmpdir("tmp-trace-#{job.id}", temp_dir) do |dir_path| temp_path = File.join(dir_path, "job.log") FileUtils.touch(temp_path) size = IO.copy_stream(src_stream, temp_path) diff --git a/lib/gitlab/ci/trace/chunked_io.rb b/lib/gitlab/ci/trace/chunked_io.rb index 8c6fd56493f..e99889f4a25 100644 --- a/lib/gitlab/ci/trace/chunked_io.rb +++ b/lib/gitlab/ci/trace/chunked_io.rb @@ -166,6 +166,13 @@ module Gitlab end def destroy! + # TODO: Remove this logging once we confirmed new live trace architecture is functional. + # See https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/4667. + unless build.has_archived_trace? + Sidekiq.logger.warn(message: 'The job does not have archived trace but going to be destroyed.', + job_id: build.id) + end + trace_chunks.fast_destroy_all @tell = @size = 0 ensure diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 07ba6f83d47..2e1eab270ff 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -7,7 +7,7 @@ module Gitlab include Gitlab::Config::Entry::LegacyValidationHelpers - attr_reader :cache, :stages, :jobs + attr_reader :stages, :jobs def initialize(config, opts = {}) @ci_config = Gitlab::Ci::Config.new(config, **opts) @@ -40,6 +40,7 @@ module Gitlab environment: job[:environment_name], coverage_regex: job[:coverage], yaml_variables: yaml_variables(name), + needs_attributes: job[:needs]&.map { |need| { name: need } }, options: { image: job[:image], services: job[:services], @@ -54,7 +55,8 @@ module Gitlab parallel: job[:parallel], instance: job[:instance], start_in: job[:start_in], - trigger: job[:trigger] + trigger: job[:trigger], + bridge_needs: job[:needs] }.compact }.compact end @@ -95,13 +97,8 @@ module Gitlab ## # Global config # - @before_script = @ci_config.before_script - @image = @ci_config.image - @after_script = @ci_config.after_script - @services = @ci_config.services @variables = @ci_config.variables @stages = @ci_config.stages - @cache = @ci_config.cache ## # Jobs @@ -113,6 +110,7 @@ module Gitlab validate_job_stage!(name, job) validate_job_dependencies!(name, job) + validate_job_needs!(name, job) validate_job_environment!(name, job) end end @@ -149,12 +147,30 @@ module Gitlab job[:dependencies].each do |dependency| raise ValidationError, "#{name} job: undefined dependency: #{dependency}" unless @jobs[dependency.to_sym] - unless @stages.index(@jobs[dependency.to_sym][:stage]) < stage_index + dependency_stage_index = @stages.index(@jobs[dependency.to_sym][:stage]) + + unless dependency_stage_index.present? && dependency_stage_index < stage_index raise ValidationError, "#{name} job: dependency #{dependency} is not defined in prior stages" end end end + def validate_job_needs!(name, job) + return unless job[:needs] + + stage_index = @stages.index(job[:stage]) + + job[:needs].each do |need| + raise ValidationError, "#{name} job: undefined need: #{need}" unless @jobs[need.to_sym] + + needs_stage_index = @stages.index(@jobs[need.to_sym][:stage]) + + unless needs_stage_index.present? && needs_stage_index < stage_index + raise ValidationError, "#{name} job: need #{need} is not defined in prior stages" + end + end + end + def validate_job_environment!(name, job) return unless job[:environment] return unless job[:environment].is_a?(Hash) diff --git a/lib/gitlab/cleanup/orphan_job_artifact_files.rb b/lib/gitlab/cleanup/orphan_job_artifact_files.rb new file mode 100644 index 00000000000..808814c39e0 --- /dev/null +++ b/lib/gitlab/cleanup/orphan_job_artifact_files.rb @@ -0,0 +1,132 @@ +# frozen_string_literal: true + +module Gitlab + module Cleanup + class OrphanJobArtifactFiles + include Gitlab::Utils::StrongMemoize + + ABSOLUTE_ARTIFACT_DIR = ::JobArtifactUploader.root.freeze + LOST_AND_FOUND = File.join(ABSOLUTE_ARTIFACT_DIR, '-', 'lost+found').freeze + BATCH_SIZE = 500 + DEFAULT_NICENESS = 'Best-effort' + + attr_accessor :batch, :total_found, :total_cleaned + attr_reader :limit, :dry_run, :niceness, :logger + + def initialize(limit: nil, dry_run: true, niceness: nil, logger: nil) + @limit = limit + @dry_run = dry_run + @niceness = niceness || DEFAULT_NICENESS + @logger = logger || Rails.logger # rubocop:disable Gitlab/RailsLogger + @total_found = @total_cleaned = 0 + + new_batch! + end + + def run! + log_info('Looking for orphan job artifacts to clean up') + + find_artifacts do |artifact_file| + batch << artifact_file + + clean_batch! if batch.full? + break if limit_reached? + end + + clean_batch! + + log_info("Processed #{total_found} job artifacts to find and clean #{total_cleaned} orphans.") + end + + private + + def new_batch! + self.batch = ::Gitlab::Cleanup::OrphanJobArtifactFilesBatch + .new(batch_size: batch_size, logger: logger, dry_run: dry_run) + end + + def clean_batch! + batch.clean! + + update_stats!(batch) + + new_batch! + end + + def update_stats!(batch) + self.total_found += batch.artifact_files.count + self.total_cleaned += batch.lost_and_found.count + end + + def limit_reached? + return false unless limit + + total_cleaned >= limit + end + + def batch_size + return BATCH_SIZE unless limit + return if limit_reached? + + todo = limit - total_cleaned + [BATCH_SIZE, todo].min + end + + def find_artifacts + Open3.popen3(*find_command) do |stdin, stdout, stderr, status_thread| + stdout.each_line do |line| + yield line + end + + log_error(stderr.read.color(:red)) unless status_thread.value.success? + end + end + + def find_command + strong_memoize(:find_command) do + cmd = %W[find -L #{absolute_artifact_dir}] + + # Search for Job Artifact IDs, they are found 6 directory + # levels deep. For example: + # shared/artifacts/2c/62/2c...a3/2019_02_27/836/628/job.log + # 1 2 3 4 5 6 + # | | | ^- date | ^- Job Artifact ID + # | | | ^- Job ID + # ^--+--+- components of hashed storage project path + cmd += %w[-mindepth 6 -maxdepth 6] + + # Artifact directories are named on their ID + cmd += %w[-type d] + + if ionice + raise ArgumentError, 'Invalid niceness' unless niceness.match?(/^\w[\w\-]*$/) + + cmd.unshift(*%W[#{ionice} --class #{niceness}]) + end + + log_info("find command: '#{cmd.join(' ')}'") + + cmd + end + end + + def absolute_artifact_dir + File.absolute_path(ABSOLUTE_ARTIFACT_DIR) + end + + def ionice + strong_memoize(:ionice) do + Gitlab::Utils.which('ionice') + end + end + + def log_info(msg, params = {}) + logger.info("#{'[DRY RUN]' if dry_run} #{msg}") + end + + def log_error(msg, params = {}) + logger.error(msg) + end + end + end +end diff --git a/lib/gitlab/cleanup/orphan_job_artifact_files_batch.rb b/lib/gitlab/cleanup/orphan_job_artifact_files_batch.rb new file mode 100644 index 00000000000..53e0c83046e --- /dev/null +++ b/lib/gitlab/cleanup/orphan_job_artifact_files_batch.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +module Gitlab + module Cleanup + class OrphanJobArtifactFilesBatch + BatchFull = Class.new(StandardError) + + class ArtifactFile + attr_accessor :path + + def initialize(path) + @path = path + end + + def artifact_id + path.split('/').last.to_i + end + end + + include Gitlab::Utils::StrongMemoize + + attr_reader :batch_size, :dry_run + attr_accessor :artifact_files + + def initialize(batch_size:, dry_run: true, logger: Rails.logger) # rubocop:disable Gitlab/RailsLogger + @batch_size = batch_size + @dry_run = dry_run + @logger = logger + @artifact_files = [] + end + + def clean! + return if artifact_files.empty? + + lost_and_found.each do |artifact| + clean_one!(artifact) + end + end + + def full? + artifact_files.count >= batch_size + end + + def <<(artifact_path) + raise BatchFull, "Batch full! Already contains #{artifact_files.count} artifacts" if full? + + artifact_files << ArtifactFile.new(artifact_path) + end + + def lost_and_found + strong_memoize(:lost_and_found) do + artifact_file_ids = artifact_files.map(&:artifact_id) + existing_artifact_ids = ::Ci::JobArtifact.id_in(artifact_file_ids).pluck_primary_key + + artifact_files.reject { |artifact| existing_artifact_ids.include?(artifact.artifact_id) } + end + end + + private + + def clean_one!(artifact_file) + log_debug("Found orphan job artifact file @ #{artifact_file.path}") + + remove_file!(artifact_file) unless dry_run + end + + def remove_file!(artifact_file) + FileUtils.rm_rf(artifact_file.path) + end + + def log_info(msg, params = {}) + @logger.info("#{'[DRY RUN]' if dry_run} #{msg}") + end + + def log_debug(msg, params = {}) + @logger.debug(msg) + end + end + end +end diff --git a/lib/gitlab/cleanup/project_upload_file_finder.rb b/lib/gitlab/cleanup/project_upload_file_finder.rb index 2ee8b60e76a..5aace564c2d 100644 --- a/lib/gitlab/cleanup/project_upload_file_finder.rb +++ b/lib/gitlab/cleanup/project_upload_file_finder.rb @@ -49,7 +49,7 @@ module Gitlab cmd = %W[#{ionice} -c Idle] + cmd if ionice log_msg = "find command: \"#{cmd.join(' ')}\"" - Rails.logger.info log_msg + Rails.logger.info log_msg # rubocop:disable Gitlab/RailsLogger cmd end diff --git a/lib/gitlab/cleanup/project_uploads.rb b/lib/gitlab/cleanup/project_uploads.rb index 82a405362c2..056e075cb21 100644 --- a/lib/gitlab/cleanup/project_uploads.rb +++ b/lib/gitlab/cleanup/project_uploads.rb @@ -8,7 +8,7 @@ module Gitlab attr_reader :logger def initialize(logger: nil) - @logger = logger || Rails.logger + @logger = logger || Rails.logger # rubocop:disable Gitlab/RailsLogger end def run!(dry_run: true) diff --git a/lib/gitlab/cleanup/remote_uploads.rb b/lib/gitlab/cleanup/remote_uploads.rb index 03298d960a4..42c93b7aecb 100644 --- a/lib/gitlab/cleanup/remote_uploads.rb +++ b/lib/gitlab/cleanup/remote_uploads.rb @@ -7,7 +7,7 @@ module Gitlab BATCH_SIZE = 100 def initialize(logger: nil) - @logger = logger || Rails.logger + @logger = logger || Rails.logger # rubocop:disable Gitlab/RailsLogger end def run!(dry_run: false) diff --git a/lib/gitlab/cluster/lifecycle_events.rb b/lib/gitlab/cluster/lifecycle_events.rb index e0f9eb59924..8f796748199 100644 --- a/lib/gitlab/cluster/lifecycle_events.rb +++ b/lib/gitlab/cluster/lifecycle_events.rb @@ -11,6 +11,9 @@ module Gitlab # We have three lifecycle events. # # - before_fork (only in forking processes) + # In forking processes (Unicorn and Puma in multiprocess mode) this + # will be called exactly once, on startup, before the workers are + # forked. This will be called in the parent process. # - worker_start # - before_master_restart (only in forking processes) # diff --git a/lib/gitlab/cluster/puma_worker_killer_observer.rb b/lib/gitlab/cluster/puma_worker_killer_observer.rb index 3b4ebc3fbae..f53051c32ff 100644 --- a/lib/gitlab/cluster/puma_worker_killer_observer.rb +++ b/lib/gitlab/cluster/puma_worker_killer_observer.rb @@ -15,9 +15,7 @@ module Gitlab private def log_termination(worker) - labels = { worker: "worker_#{worker.index}" } - - @counter.increment(labels) + @counter.increment end end end diff --git a/lib/gitlab/config/entry/attributable.rb b/lib/gitlab/config/entry/attributable.rb index 560fe63df0e..87bd257f69a 100644 --- a/lib/gitlab/config/entry/attributable.rb +++ b/lib/gitlab/config/entry/attributable.rb @@ -18,6 +18,10 @@ module Gitlab config[attribute] end + + define_method("has_#{attribute}?") do + config.is_a?(Hash) && config.key?(attribute) + end end end end diff --git a/lib/gitlab/config/entry/configurable.rb b/lib/gitlab/config/entry/configurable.rb index 6667a5d3d33..b7ec4b7c4f8 100644 --- a/lib/gitlab/config/entry/configurable.rb +++ b/lib/gitlab/config/entry/configurable.rb @@ -58,13 +58,21 @@ module Gitlab Hash[(@nodes || {}).map { |key, factory| [key, factory.dup] }] end + def reserved_node_names + self.nodes.select do |_, node| + node.reserved? + end.keys + end + private # rubocop: disable CodeReuse/ActiveRecord - def entry(key, entry, metadata) + def entry(key, entry, description: nil, default: nil, inherit: nil, reserved: nil) factory = ::Gitlab::Config::Entry::Factory.new(entry) - .with(description: metadata[:description]) - .with(default: metadata[:default]) + .with(description: description) + .with(default: default) + .with(inherit: inherit) + .with(reserved: reserved) (@nodes ||= {}).merge!(key.to_sym => factory) end diff --git a/lib/gitlab/config/entry/factory.rb b/lib/gitlab/config/entry/factory.rb index 3c06b1e0d24..8f1f4a81bb5 100644 --- a/lib/gitlab/config/entry/factory.rb +++ b/lib/gitlab/config/entry/factory.rb @@ -30,6 +30,18 @@ module Gitlab self end + def description + @attributes[:description] + end + + def inheritable? + @attributes[:inherit] + end + + def reserved? + @attributes[:reserved] + end + def create! raise InvalidFactory unless defined?(@value) diff --git a/lib/gitlab/config/entry/validators.rb b/lib/gitlab/config/entry/validators.rb index 6796fcce75f..0289e675c6b 100644 --- a/lib/gitlab/config/entry/validators.rb +++ b/lib/gitlab/config/entry/validators.rb @@ -26,6 +26,17 @@ module Gitlab end end + class RequiredKeysValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + present_keys = options[:in] - value.try(:keys).to_a + + if present_keys.any? + record.errors.add(attribute, "missing required keys: " + + present_keys.join(', ')) + end + end + end + class AllowedValuesValidator < ActiveModel::EachValidator def validate_each(record, attribute, value) unless options[:in].include?(value.to_s) diff --git a/lib/gitlab/config/loader/yaml.rb b/lib/gitlab/config/loader/yaml.rb index 8159f8b8026..4cedab1545c 100644 --- a/lib/gitlab/config/loader/yaml.rb +++ b/lib/gitlab/config/loader/yaml.rb @@ -4,6 +4,13 @@ module Gitlab module Config module Loader class Yaml + DataTooLargeError = Class.new(Loader::FormatError) + + include Gitlab::Utils::StrongMemoize + + MAX_YAML_SIZE = 1.megabyte + MAX_YAML_DEPTH = 100 + def initialize(config) @config = YAML.safe_load(config, [Symbol], [], true) rescue Psych::Exception => e @@ -11,16 +18,35 @@ module Gitlab end def valid? - @config.is_a?(Hash) + hash? && !too_big? end def load! - unless valid? - raise Loader::FormatError, 'Invalid configuration format' - end + raise DataTooLargeError, 'The parsed YAML is too big' if too_big? + raise Loader::FormatError, 'Invalid configuration format' unless hash? @config.deep_symbolize_keys end + + private + + def hash? + @config.is_a?(Hash) + end + + def too_big? + return false unless Feature.enabled?(:ci_yaml_limit_size, default_enabled: true) + + !deep_size.valid? + end + + def deep_size + strong_memoize(:deep_size) do + Gitlab::Utils::DeepSize.new(@config, + max_size: MAX_YAML_SIZE, + max_depth: MAX_YAML_DEPTH) + end + end end end end diff --git a/lib/gitlab/content_security_policy/config_loader.rb b/lib/gitlab/content_security_policy/config_loader.rb new file mode 100644 index 00000000000..ff844645b11 --- /dev/null +++ b/lib/gitlab/content_security_policy/config_loader.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module ContentSecurityPolicy + class ConfigLoader + DIRECTIVES = %w(base_uri child_src connect_src default_src font_src + form_action frame_ancestors frame_src img_src manifest_src + media_src object_src report_uri script_src style_src worker_src).freeze + + def self.default_settings_hash + { + 'enabled' => false, + 'report_only' => false, + 'directives' => DIRECTIVES.each_with_object({}) { |directive, hash| hash[directive] = nil } + } + end + + def initialize(csp_directives) + @csp_directives = HashWithIndifferentAccess.new(csp_directives) + end + + def load(policy) + DIRECTIVES.each do |directive| + arguments = arguments_for(directive) + + next unless arguments.present? + + policy.public_send(directive, *arguments) # rubocop:disable GitlabSecurity/PublicSend + end + end + + private + + def arguments_for(directive) + arguments = @csp_directives[directive.to_s] + + return unless arguments.present? && arguments.is_a?(String) + + arguments.strip.split(' ').map(&:strip) + end + end + end +end diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index f7d046600e8..5b0b91de5da 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -84,11 +84,7 @@ module Gitlab .and(t[:created_at].lteq(Date.current.end_of_day)) .and(t[:author_id].eq(contributor.id)) - date_interval = if Gitlab::Database.postgresql? - "INTERVAL '#{Time.zone.now.utc_offset} seconds'" - else - "INTERVAL #{Time.zone.now.utc_offset} SECOND" - end + date_interval = "INTERVAL '#{Time.zone.now.utc_offset} seconds'" Event.reorder(nil) .select(t[:project_id], t[:target_type], t[:action], "date(created_at + #{date_interval}) AS date", 'count(id) as total_amount') diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 469a7fd9f7b..6ce47650562 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -7,6 +7,11 @@ module Gitlab Gitlab::SafeRequestStore.fetch(:current_application_settings) { ensure_application_settings! } end + def expire_current_application_settings + ::ApplicationSetting.expire + Gitlab::SafeRequestStore.delete(:current_application_settings) + end + def clear_in_memory_application_settings! @in_memory_application_settings = nil end @@ -45,7 +50,7 @@ module Gitlab # need to be added to the application settings. To prevent Rake tasks # and other callers from failing, use any loaded settings and return # defaults for missing columns. - if ActiveRecord::Migrator.needs_migration? + if ActiveRecord::Base.connection.migration_context.needs_migration? db_attributes = current_settings&.attributes || {} fake_application_settings(db_attributes) elsif current_settings.present? diff --git a/lib/gitlab/cycle_analytics/base_event_fetcher.rb b/lib/gitlab/cycle_analytics/base_event_fetcher.rb index 304d60996a6..07ae430c45e 100644 --- a/lib/gitlab/cycle_analytics/base_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/base_event_fetcher.rb @@ -4,13 +4,13 @@ module Gitlab module CycleAnalytics class BaseEventFetcher include BaseQuery + include GroupProjectsProvider - attr_reader :projections, :query, :stage, :order + attr_reader :projections, :query, :stage, :order, :options MAX_EVENTS = 50 - def initialize(project:, stage:, options:) - @project = project + def initialize(stage:, options:) @stage = stage @options = options end @@ -40,13 +40,13 @@ module Gitlab end def events_query - diff_fn = subtract_datetimes_diff(base_query, @options[:start_time_attrs], @options[:end_time_attrs]) + diff_fn = subtract_datetimes_diff(base_query, options[:start_time_attrs], options[:end_time_attrs]) base_query.project(extract_diff_epoch(diff_fn).as('total_time'), *projections).order(order.desc).take(MAX_EVENTS) end def default_order - [@options[:start_time_attrs]].flatten.first + [options[:start_time_attrs]].flatten.first end def serialize(_event) @@ -59,13 +59,21 @@ module Gitlab def allowed_ids @allowed_ids ||= allowed_ids_finder_class - .new(@options[:current_user], project_id: @project.id) + .new(options[:current_user], allowed_ids_source) .execute.where(id: event_result_ids).pluck(:id) end def event_result_ids event_result.map { |event| event['id'] } end + + def allowed_ids_source + group ? { group_id: group.id, include_subgroups: true } : { project_id: project.id } + end + + def serialization_context + {} + end end end end diff --git a/lib/gitlab/cycle_analytics/base_query.rb b/lib/gitlab/cycle_analytics/base_query.rb index 36231b187cd..459bb5177b5 100644 --- a/lib/gitlab/cycle_analytics/base_query.rb +++ b/lib/gitlab/cycle_analytics/base_query.rb @@ -10,23 +10,38 @@ module Gitlab private def base_query - @base_query ||= stage_query(@project.id) # rubocop:disable Gitlab/ModuleWithInstanceVariables + @base_query ||= stage_query(projects.map(&:id)) end def stage_query(project_ids) query = mr_closing_issues_table.join(issue_table).on(issue_table[:id].eq(mr_closing_issues_table[:issue_id])) .join(issue_metrics_table).on(issue_table[:id].eq(issue_metrics_table[:issue_id])) + .join(projects_table).on(issue_table[:project_id].eq(projects_table[:id])) + .join(routes_table).on(projects_table[:namespace_id].eq(routes_table[:source_id])) .project(issue_table[:project_id].as("project_id")) - .where(issue_table[:project_id].in(project_ids)) - .where(issue_table[:created_at].gteq(@options[:from])) # rubocop:disable Gitlab/ModuleWithInstanceVariables + .project(projects_table[:path].as("project_path")) + .project(routes_table[:path].as("namespace_path")) + + query = limit_query(query, project_ids) # Load merge_requests - query = query.join(mr_table, Arel::Nodes::OuterJoin) + + query = load_merge_requests(query) + + query + end + + def limit_query(query, project_ids) + query.where(issue_table[:project_id].in(project_ids)) + .where(routes_table[:source_type].eq('Namespace')) + .where(issue_table[:created_at].gteq(options[:from])) + end + + def load_merge_requests(query) + query.join(mr_table, Arel::Nodes::OuterJoin) .on(mr_table[:id].eq(mr_closing_issues_table[:merge_request_id])) .join(mr_metrics_table) .on(mr_table[:id].eq(mr_metrics_table[:merge_request_id])) - - query end end end diff --git a/lib/gitlab/cycle_analytics/base_stage.rb b/lib/gitlab/cycle_analytics/base_stage.rb index e2d6a301734..1cd54238bb4 100644 --- a/lib/gitlab/cycle_analytics/base_stage.rb +++ b/lib/gitlab/cycle_analytics/base_stage.rb @@ -4,9 +4,11 @@ module Gitlab module CycleAnalytics class BaseStage include BaseQuery + include GroupProjectsProvider - def initialize(project:, options:) - @project = project + attr_reader :options + + def initialize(options:) @options = options end @@ -14,30 +16,23 @@ module Gitlab event_fetcher.fetch end - def as_json - AnalyticsStageSerializer.new.represent(self) + def as_json(serializer: AnalyticsStageSerializer) + serializer.new.represent(self) end def title raise NotImplementedError.new("Expected #{self.name} to implement title") end - def median - BatchLoader.for(@project.id).batch(key: name) do |project_ids, loader| - cte_table = Arel::Table.new("cte_table_for_#{name}") - - # Build a `SELECT` query. We find the first of the `end_time_attrs` that isn't `NULL` (call this end_time). - # Next, we find the first of the start_time_attrs that isn't `NULL` (call this start_time). - # We compute the (end_time - start_time) interval, and give it an alias based on the current - # cycle analytics stage. - interval_query = Arel::Nodes::As.new(cte_table, - subtract_datetimes(stage_query(project_ids), start_time_attrs, end_time_attrs, name.to_s)) + def project_median + return if project.nil? + BatchLoader.for(project.id).batch(key: name) do |project_ids, loader| if project_ids.one? - loader.call(@project.id, median_datetime(cte_table, interval_query, name)) + loader.call(project.id, median_query(project_ids)) else begin - median_datetimes(cte_table, interval_query, name, :project_id)&.each do |project_id, median| + median_datetimes(cte_table, interval_query(project_ids), name, :project_id)&.each do |project_id, median| loader.call(project_id, median) end rescue NotSupportedError @@ -47,20 +42,41 @@ module Gitlab end end + def group_median + median_query(projects.map(&:id)) + end + + def median_query(project_ids) + # Build a `SELECT` query. We find the first of the `end_time_attrs` that isn't `NULL` (call this end_time). + # Next, we find the first of the start_time_attrs that isn't `NULL` (call this start_time). + # We compute the (end_time - start_time) interval, and give it an alias based on the current + # cycle analytics stage. + + median_datetime(cte_table, interval_query(project_ids), name) + end + def name raise NotImplementedError.new("Expected #{self.name} to implement name") end + def cte_table + Arel::Table.new("cte_table_for_#{name}") + end + + def interval_query(project_ids) + Arel::Nodes::As.new(cte_table, + subtract_datetimes(stage_query(project_ids), start_time_attrs, end_time_attrs, name.to_s)) + end + private def event_fetcher - @event_fetcher ||= Gitlab::CycleAnalytics::EventFetcher[name].new(project: @project, - stage: name, + @event_fetcher ||= Gitlab::CycleAnalytics::EventFetcher[name].new(stage: name, options: event_options) end def event_options - @options.merge(start_time_attrs: start_time_attrs, end_time_attrs: end_time_attrs) + options.merge(start_time_attrs: start_time_attrs, end_time_attrs: end_time_attrs) end end end diff --git a/lib/gitlab/cycle_analytics/builds_event_helper.rb b/lib/gitlab/cycle_analytics/builds_event_helper.rb new file mode 100644 index 00000000000..0d6f32fdc6f --- /dev/null +++ b/lib/gitlab/cycle_analytics/builds_event_helper.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module CycleAnalytics + module BuildsEventHelper + def initialize(*args) + @projections = [build_table[:id]] + @order = build_table[:created_at] + + super(*args) + end + + def fetch + Updater.update!(event_result, from: 'id', to: 'build', klass: ::Ci::Build) + + super + end + + def events_query + base_query.join(build_table).on(mr_metrics_table[:pipeline_id].eq(build_table[:commit_id])) + + super + end + + private + + def allowed_ids + nil + end + + def serialize(event) + AnalyticsBuildSerializer.new.represent(event['build']) + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/code_event_fetcher.rb b/lib/gitlab/cycle_analytics/code_event_fetcher.rb index 591db3c35e6..fcc282bf7a6 100644 --- a/lib/gitlab/cycle_analytics/code_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/code_event_fetcher.rb @@ -3,6 +3,8 @@ module Gitlab module CycleAnalytics class CodeEventFetcher < BaseEventFetcher + include CodeHelper + def initialize(*args) @projections = [mr_table[:title], mr_table[:iid], @@ -18,7 +20,7 @@ module Gitlab private def serialize(event) - AnalyticsMergeRequestSerializer.new(project: @project).represent(event) + AnalyticsMergeRequestSerializer.new(serialization_context).represent(event) end def allowed_ids_finder_class diff --git a/lib/gitlab/cycle_analytics/code_helper.rb b/lib/gitlab/cycle_analytics/code_helper.rb new file mode 100644 index 00000000000..8f28bdd2502 --- /dev/null +++ b/lib/gitlab/cycle_analytics/code_helper.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Gitlab + module CycleAnalytics + module CodeHelper + def stage_query(project_ids) + super(project_ids).where(mr_table[:created_at].gteq(issue_metrics_table[:first_mentioned_in_commit_at])) + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/code_stage.rb b/lib/gitlab/cycle_analytics/code_stage.rb index 2e5f9ef5a40..89a6430221c 100644 --- a/lib/gitlab/cycle_analytics/code_stage.rb +++ b/lib/gitlab/cycle_analytics/code_stage.rb @@ -3,6 +3,8 @@ module Gitlab module CycleAnalytics class CodeStage < BaseStage + include CodeHelper + def start_time_attrs @start_time_attrs ||= issue_metrics_table[:first_mentioned_in_commit_at] end diff --git a/lib/gitlab/cycle_analytics/group_projects_provider.rb b/lib/gitlab/cycle_analytics/group_projects_provider.rb new file mode 100644 index 00000000000..1287a48daaa --- /dev/null +++ b/lib/gitlab/cycle_analytics/group_projects_provider.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module CycleAnalytics + module GroupProjectsProvider + def projects + group ? projects_for_group : [project] + end + + def group + @group ||= options.fetch(:group, nil) + end + + def project + @project ||= options.fetch(:project, nil) + end + + private + + def projects_for_group + projects = Project.inside_path(group.full_path) + projects = projects.where(id: options[:projects]) if options[:projects] + projects + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/group_stage_summary.rb b/lib/gitlab/cycle_analytics/group_stage_summary.rb new file mode 100644 index 00000000000..a1fc941495d --- /dev/null +++ b/lib/gitlab/cycle_analytics/group_stage_summary.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module CycleAnalytics + class GroupStageSummary + attr_reader :group, :from, :current_user, :options + + def initialize(group, options:) + @group = group + @from = options[:from] + @current_user = options[:current_user] + @options = options + end + + def data + [serialize(Summary::Group::Issue.new(group: group, from: from, current_user: current_user, options: options)), + serialize(Summary::Group::Deploy.new(group: group, from: from, options: options))] + end + + private + + def serialize(summary_object) + AnalyticsSummarySerializer.new.represent(summary_object) + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/issue_event_fetcher.rb b/lib/gitlab/cycle_analytics/issue_event_fetcher.rb index 30c6ead8968..6914cf24c19 100644 --- a/lib/gitlab/cycle_analytics/issue_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/issue_event_fetcher.rb @@ -3,6 +3,8 @@ module Gitlab module CycleAnalytics class IssueEventFetcher < BaseEventFetcher + include IssueHelper + def initialize(*args) @projections = [issue_table[:title], issue_table[:iid], @@ -16,7 +18,7 @@ module Gitlab private def serialize(event) - AnalyticsIssueSerializer.new(project: @project).represent(event) + AnalyticsIssueSerializer.new(serialization_context).represent(event) end def allowed_ids_finder_class diff --git a/lib/gitlab/cycle_analytics/issue_helper.rb b/lib/gitlab/cycle_analytics/issue_helper.rb new file mode 100644 index 00000000000..295eca5edca --- /dev/null +++ b/lib/gitlab/cycle_analytics/issue_helper.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module CycleAnalytics + module IssueHelper + def stage_query(project_ids) + query = issue_table.join(issue_metrics_table).on(issue_table[:id].eq(issue_metrics_table[:issue_id])) + .join(projects_table).on(issue_table[:project_id].eq(projects_table[:id])) + .join(routes_table).on(projects_table[:namespace_id].eq(routes_table[:source_id])) + .project(issue_table[:project_id].as("project_id")) + .project(projects_table[:path].as("project_path")) + .project(routes_table[:path].as("namespace_path")) + + query = limit_query(query, project_ids) + + query + end + + def limit_query(query, project_ids) + query.where(issue_table[:project_id].in(project_ids)) + .where(routes_table[:source_type].eq('Namespace')) + .where(issue_table[:created_at].gteq(options[:from])) + .where(issue_metrics_table[:first_added_to_board_at].not_eq(nil).or(issue_metrics_table[:first_associated_with_milestone_at].not_eq(nil))) + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/issue_stage.rb b/lib/gitlab/cycle_analytics/issue_stage.rb index 4eae2da512c..738cb3eba03 100644 --- a/lib/gitlab/cycle_analytics/issue_stage.rb +++ b/lib/gitlab/cycle_analytics/issue_stage.rb @@ -3,6 +3,8 @@ module Gitlab module CycleAnalytics class IssueStage < BaseStage + include IssueHelper + def start_time_attrs @start_time_attrs ||= issue_table[:created_at] end diff --git a/lib/gitlab/cycle_analytics/metrics_tables.rb b/lib/gitlab/cycle_analytics/metrics_tables.rb index 3e0302d308d..015f7bfde24 100644 --- a/lib/gitlab/cycle_analytics/metrics_tables.rb +++ b/lib/gitlab/cycle_analytics/metrics_tables.rb @@ -35,6 +35,14 @@ module Gitlab User.arel_table end + def projects_table + Project.arel_table + end + + def routes_table + Route.arel_table + end + def build_table ::CommitStatus.arel_table end diff --git a/lib/gitlab/cycle_analytics/permissions.rb b/lib/gitlab/cycle_analytics/permissions.rb index afefd09b614..55214e6b896 100644 --- a/lib/gitlab/cycle_analytics/permissions.rb +++ b/lib/gitlab/cycle_analytics/permissions.rb @@ -23,7 +23,7 @@ module Gitlab end def get - ::CycleAnalytics::STAGES.each do |stage| + ::CycleAnalytics::LevelBase::STAGES.each do |stage| @stage_permission_hash[stage] = authorized_stage?(stage) end diff --git a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb index aeca9d00156..bad02e00a13 100644 --- a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb @@ -3,60 +3,26 @@ module Gitlab module CycleAnalytics class PlanEventFetcher < BaseEventFetcher + include PlanHelper + def initialize(*args) - @projections = [mr_diff_table[:id], - issue_metrics_table[:first_mentioned_in_commit_at]] + @projections = [issue_table[:title], + issue_table[:iid], + issue_table[:id], + issue_table[:created_at], + issue_table[:author_id]] super(*args) end - def events_query - base_query - .join(mr_diff_table) - .on(mr_diff_table[:merge_request_id].eq(mr_table[:id])) - - super - end - private - def allowed_ids - nil - end - - def merge_request_diff_commits - @merge_request_diff_commits ||= - MergeRequestDiffCommit - .where(merge_request_diff_id: event_result.map { |event| event['id'] }) - .group_by(&:merge_request_diff_id) - end - def serialize(event) - commit = first_time_reference_commit(event) - - return unless commit - - serialize_commit(event, commit, query) - end - - def first_time_reference_commit(event) - return unless event && merge_request_diff_commits - - commits = merge_request_diff_commits[event['id'].to_i] - - return if commits.blank? - - commits.find do |commit| - next unless commit[:committed_date] && event['first_mentioned_in_commit_at'] - - commit[:committed_date].to_i == DateTime.parse(event['first_mentioned_in_commit_at'].to_s).to_i - end + AnalyticsIssueSerializer.new(serialization_context).represent(event) end - def serialize_commit(event, commit, query) - commit = Commit.from_hash(commit.to_hash, @project) - - AnalyticsCommitSerializer.new(project: @project, total_time: event['total_time']).represent(commit) + def allowed_ids_finder_class + IssuesFinder end end end diff --git a/lib/gitlab/cycle_analytics/plan_helper.rb b/lib/gitlab/cycle_analytics/plan_helper.rb new file mode 100644 index 00000000000..a63ae58ad21 --- /dev/null +++ b/lib/gitlab/cycle_analytics/plan_helper.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module CycleAnalytics + module PlanHelper + def stage_query(project_ids) + query = issue_table.join(issue_metrics_table).on(issue_table[:id].eq(issue_metrics_table[:issue_id])) + .join(projects_table).on(issue_table[:project_id].eq(projects_table[:id])) + .join(routes_table).on(projects_table[:namespace_id].eq(routes_table[:source_id])) + .project(issue_table[:project_id].as("project_id")) + .project(projects_table[:path].as("project_path")) + .project(routes_table[:path].as("namespace_path")) + .where(issue_table[:project_id].in(project_ids)) + .where(routes_table[:source_type].eq('Namespace')) + query = limit_query(query) + + query + end + + def limit_query(query) + query.where(issue_table[:created_at].gteq(options[:from])) + .where(issue_metrics_table[:first_added_to_board_at].not_eq(nil).or(issue_metrics_table[:first_associated_with_milestone_at].not_eq(nil))) + .where(issue_metrics_table[:first_mentioned_in_commit_at].not_eq(nil)) + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/plan_stage.rb b/lib/gitlab/cycle_analytics/plan_stage.rb index 513e4575be0..0b27d114f52 100644 --- a/lib/gitlab/cycle_analytics/plan_stage.rb +++ b/lib/gitlab/cycle_analytics/plan_stage.rb @@ -3,6 +3,8 @@ module Gitlab module CycleAnalytics class PlanStage < BaseStage + include PlanHelper + def start_time_attrs @start_time_attrs ||= [issue_metrics_table[:first_associated_with_milestone_at], issue_metrics_table[:first_added_to_board_at]] @@ -21,7 +23,7 @@ module Gitlab end def legend - _("Related Commits") + _("Related Issues") end def description diff --git a/lib/gitlab/cycle_analytics/production_event_fetcher.rb b/lib/gitlab/cycle_analytics/production_event_fetcher.rb index 6681cb42c90..8843ab2bcb9 100644 --- a/lib/gitlab/cycle_analytics/production_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/production_event_fetcher.rb @@ -2,7 +2,29 @@ module Gitlab module CycleAnalytics - class ProductionEventFetcher < IssueEventFetcher + class ProductionEventFetcher < BaseEventFetcher + include ProductionHelper + + def initialize(*args) + @projections = [issue_table[:title], + issue_table[:iid], + issue_table[:id], + issue_table[:created_at], + issue_table[:author_id], + routes_table[:path]] + + super(*args) + end + + private + + def serialize(event) + AnalyticsIssueSerializer.new(serialization_context).represent(event) + end + + def allowed_ids_finder_class + IssuesFinder + end end end end diff --git a/lib/gitlab/cycle_analytics/production_helper.rb b/lib/gitlab/cycle_analytics/production_helper.rb index aff65b150fb..778757a9ede 100644 --- a/lib/gitlab/cycle_analytics/production_helper.rb +++ b/lib/gitlab/cycle_analytics/production_helper.rb @@ -6,7 +6,7 @@ module Gitlab def stage_query(project_ids) super(project_ids) .where(mr_metrics_table[:first_deployed_to_production_at] - .gteq(@options[:from])) # rubocop:disable Gitlab/ModuleWithInstanceVariables + .gteq(options[:from])) end end end diff --git a/lib/gitlab/cycle_analytics/review_event_fetcher.rb b/lib/gitlab/cycle_analytics/review_event_fetcher.rb index de100295281..4b5d79097b7 100644 --- a/lib/gitlab/cycle_analytics/review_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/review_event_fetcher.rb @@ -3,6 +3,8 @@ module Gitlab module CycleAnalytics class ReviewEventFetcher < BaseEventFetcher + include ReviewHelper + def initialize(*args) @projections = [mr_table[:title], mr_table[:iid], @@ -17,7 +19,7 @@ module Gitlab private def serialize(event) - AnalyticsMergeRequestSerializer.new(project: @project).represent(event) + AnalyticsMergeRequestSerializer.new(serialization_context).represent(event) end def allowed_ids_finder_class diff --git a/lib/gitlab/cycle_analytics/review_helper.rb b/lib/gitlab/cycle_analytics/review_helper.rb new file mode 100644 index 00000000000..c53249652b5 --- /dev/null +++ b/lib/gitlab/cycle_analytics/review_helper.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Gitlab + module CycleAnalytics + module ReviewHelper + def stage_query(project_ids) + super(project_ids).where(mr_metrics_table[:merged_at].not_eq(nil)) + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/review_stage.rb b/lib/gitlab/cycle_analytics/review_stage.rb index 294b656bc55..e9df8cd5a05 100644 --- a/lib/gitlab/cycle_analytics/review_stage.rb +++ b/lib/gitlab/cycle_analytics/review_stage.rb @@ -3,6 +3,8 @@ module Gitlab module CycleAnalytics class ReviewStage < BaseStage + include ReviewHelper + def start_time_attrs @start_time_attrs ||= mr_table[:created_at] end diff --git a/lib/gitlab/cycle_analytics/staging_event_fetcher.rb b/lib/gitlab/cycle_analytics/staging_event_fetcher.rb index 70ce82383b3..1454a1a33eb 100644 --- a/lib/gitlab/cycle_analytics/staging_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/staging_event_fetcher.rb @@ -3,34 +3,8 @@ module Gitlab module CycleAnalytics class StagingEventFetcher < BaseEventFetcher - def initialize(*args) - @projections = [build_table[:id]] - @order = build_table[:created_at] - - super(*args) - end - - def fetch - Updater.update!(event_result, from: 'id', to: 'build', klass: ::Ci::Build) - - super - end - - def events_query - base_query.join(build_table).on(mr_metrics_table[:pipeline_id].eq(build_table[:commit_id])) - - super - end - - private - - def allowed_ids - nil - end - - def serialize(event) - AnalyticsBuildSerializer.new.represent(event['build']) - end + include ProductionHelper + include BuildsEventHelper end end end diff --git a/lib/gitlab/cycle_analytics/staging_stage.rb b/lib/gitlab/cycle_analytics/staging_stage.rb index dbc2414ff66..e03627c6cd1 100644 --- a/lib/gitlab/cycle_analytics/staging_stage.rb +++ b/lib/gitlab/cycle_analytics/staging_stage.rb @@ -4,6 +4,7 @@ module Gitlab module CycleAnalytics class StagingStage < BaseStage include ProductionHelper + def start_time_attrs @start_time_attrs ||= mr_metrics_table[:merged_at] end diff --git a/lib/gitlab/cycle_analytics/summary/group/base.rb b/lib/gitlab/cycle_analytics/summary/group/base.rb new file mode 100644 index 00000000000..48d8164bde1 --- /dev/null +++ b/lib/gitlab/cycle_analytics/summary/group/base.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module CycleAnalytics + module Summary + module Group + class Base + attr_reader :group, :from, :options + + def initialize(group:, from:, options:) + @group = group + @from = from + @options = options + end + + def title + raise NotImplementedError.new("Expected #{self.name} to implement title") + end + + def value + raise NotImplementedError.new("Expected #{self.name} to implement value") + end + end + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/summary/group/deploy.rb b/lib/gitlab/cycle_analytics/summary/group/deploy.rb new file mode 100644 index 00000000000..78d677cf558 --- /dev/null +++ b/lib/gitlab/cycle_analytics/summary/group/deploy.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module CycleAnalytics + module Summary + module Group + class Deploy < Group::Base + include GroupProjectsProvider + + def title + n_('Deploy', 'Deploys', value) + end + + def value + @value ||= find_deployments + end + + private + + def find_deployments + deployments = Deployment.joins(:project).merge(Project.inside_path(group.full_path)) + deployments = deployments.where(projects: { id: options[:projects] }) if options[:projects] + deployments = deployments.where("deployments.created_at > ?", from) + deployments.success.count + end + end + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/summary/group/issue.rb b/lib/gitlab/cycle_analytics/summary/group/issue.rb new file mode 100644 index 00000000000..9daae8531d8 --- /dev/null +++ b/lib/gitlab/cycle_analytics/summary/group/issue.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module CycleAnalytics + module Summary + module Group + class Issue < Group::Base + attr_reader :group, :from, :current_user, :options + + def initialize(group:, from:, current_user:, options:) + @group = group + @from = from + @current_user = current_user + @options = options + end + + def title + n_('New Issue', 'New Issues', value) + end + + def value + @value ||= find_issues + end + + private + + def find_issues + issues = IssuesFinder.new(current_user, group_id: group.id, include_subgroups: true, created_after: from).execute + issues = issues.where(projects: { id: options[:projects] }) if options[:projects] + issues.count + end + end + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/test_event_fetcher.rb b/lib/gitlab/cycle_analytics/test_event_fetcher.rb index 4d5ea5b7c34..2fa44b1b364 100644 --- a/lib/gitlab/cycle_analytics/test_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/test_event_fetcher.rb @@ -2,7 +2,9 @@ module Gitlab module CycleAnalytics - class TestEventFetcher < StagingEventFetcher + class TestEventFetcher < BaseEventFetcher + include TestHelper + include BuildsEventHelper end end end diff --git a/lib/gitlab/cycle_analytics/test_helper.rb b/lib/gitlab/cycle_analytics/test_helper.rb new file mode 100644 index 00000000000..d9124d62c7c --- /dev/null +++ b/lib/gitlab/cycle_analytics/test_helper.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module CycleAnalytics + module TestHelper + def stage_query(project_ids) + if branch + super(project_ids).where(build_table[:ref].eq(branch)) + else + super(project_ids) + end + end + + private + + def branch + @branch ||= options[:branch] + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/test_stage.rb b/lib/gitlab/cycle_analytics/test_stage.rb index c31b664148b..4787a906c07 100644 --- a/lib/gitlab/cycle_analytics/test_stage.rb +++ b/lib/gitlab/cycle_analytics/test_stage.rb @@ -3,6 +3,8 @@ module Gitlab module CycleAnalytics class TestStage < BaseStage + include TestHelper + def start_time_attrs @start_time_attrs ||= mr_metrics_table[:latest_build_started_at] end @@ -26,14 +28,6 @@ module Gitlab def description _("Total test time for all commits/merges") end - - def stage_query(project_ids) - if @options[:branch] - super(project_ids).where(build_table[:ref].eq(@options[:branch])) - else - super(project_ids) - end - end end end end diff --git a/lib/gitlab/cycle_analytics/usage_data.rb b/lib/gitlab/cycle_analytics/usage_data.rb index 913ee373f54..644300caead 100644 --- a/lib/gitlab/cycle_analytics/usage_data.rb +++ b/lib/gitlab/cycle_analytics/usage_data.rb @@ -32,7 +32,7 @@ module Gitlab def medians_per_stage projects.each_with_object({}) do |project, hsh| - ::CycleAnalytics.new(project, options).all_medians_per_stage.each do |stage_name, median| + ::CycleAnalytics::ProjectLevel.new(project, options: options).all_medians_by_stage.each do |stage_name, median| hsh[stage_name] ||= [] hsh[stage_name] << median end diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb index 1ecf4a12db7..332ca8bf9b8 100644 --- a/lib/gitlab/danger/helper.rb +++ b/lib/gitlab/danger/helper.rb @@ -46,6 +46,16 @@ module Gitlab ee? ? 'gitlab-ee' : 'gitlab-ce' end + def markdown_list(items) + list = items.map { |item| "* `#{item}`" }.join("\n") + + if items.size > 10 + "\n<details>\n\n#{list}\n\n</details>\n" + else + list + end + end + # @return [Hash<String,Array<String>>] def changes_by_category all_changed_files.each_with_object(Hash.new { |h, k| h[k] = [] }) do |file, hash| @@ -103,6 +113,11 @@ module Gitlab yarn\.lock )\z}x => :frontend, + %r{\A(ee/)?db/(?!fixtures)[^/]+} => :database, + %r{\A(ee/)?lib/gitlab/(database|background_migration|sql|github_import)(/|\.rb)} => :database, + %r{\A(app/models/project_authorization|app/services/users/refresh_authorized_projects_service)(/|\.rb)} => :database, + %r{\Arubocop/cop/migration(/|\.rb)} => :database, + %r{\A(ee/)?app/(?!assets|views)[^/]+} => :backend, %r{\A(ee/)?(bin|config|danger|generator_templates|lib|rubocop|scripts)/} => :backend, %r{\A(ee/)?spec/features/} => :test, @@ -112,7 +127,6 @@ module Gitlab %r{\A(Dangerfile|Gemfile|Gemfile.lock|Procfile|Rakefile|\.gitlab-ci\.yml)\z} => :backend, %r{\A[A-Z_]+_VERSION\z} => :backend, - %r{\A(ee/)?db/} => :database, %r{\A(ee/)?qa/} => :qa, # Files that don't fit into any category are marked with :none @@ -128,6 +142,22 @@ module Gitlab def new_teammates(usernames) usernames.map { |u| Gitlab::Danger::Teammate.new('username' => u) } end + + def missing_database_labels(current_mr_labels) + labels = if has_database_scoped_labels?(current_mr_labels) + ['database'] + else + ['database', 'database::review pending'] + end + + labels - current_mr_labels + end + + private + + def has_database_scoped_labels?(current_mr_labels) + current_mr_labels.any? { |label| label.start_with?('database::') } + end end end end diff --git a/lib/gitlab/danger/teammate.rb b/lib/gitlab/danger/teammate.rb index b44f134f2c1..74cbcc11255 100644 --- a/lib/gitlab/danger/teammate.rb +++ b/lib/gitlab/danger/teammate.rb @@ -39,7 +39,7 @@ module Gitlab def has_capability?(project, category, kind, labels) case category when :test - area = role[/Test Automation Engineer, (\w+)/, 1] + area = role[/Test Automation Engineer(?:.*?, (\w+))/, 1] area && labels.any?(area) if kind == :reviewer else diff --git a/lib/gitlab/data_builder/note.rb b/lib/gitlab/data_builder/note.rb index 16e62622ed4..2c4ef73a688 100644 --- a/lib/gitlab/data_builder/note.rb +++ b/lib/gitlab/data_builder/note.rb @@ -44,7 +44,7 @@ module Gitlab data[:commit] = build_data_for_commit(project, user, note) elsif note.for_issue? data[:issue] = note.noteable.hook_attrs - data[:issue][:labels] = note.noteable.labels(&:hook_attrs) + data[:issue][:labels] = note.noteable.labels_hook_attrs elsif note.for_merge_request? data[:merge_request] = note.noteable.hook_attrs elsif note.for_snippet? diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index 40bda3410e1..75d9a2d55b9 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -60,7 +60,8 @@ module Gitlab # rubocop:disable Metrics/ParameterLists def build( project:, user:, ref:, oldrev: nil, newrev: nil, - commits: [], commits_count: nil, message: nil, push_options: {}) + commits: [], commits_count: nil, message: nil, push_options: {}, + with_changed_files: true) commits = Array(commits) @@ -75,7 +76,7 @@ module Gitlab # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/38259 commit_attrs = Gitlab::GitalyClient.allow_n_plus_1_calls do commits_limited.map do |commit| - commit.hook_attrs(with_changed_files: true) + commit.hook_attrs(with_changed_files: with_changed_files) end end @@ -128,8 +129,6 @@ module Gitlab SAMPLE_DATA end - private - def checkout_sha(repository, newrev, ref) # Checkout sha is nil when we remove branch or tag return if Gitlab::Git.blank_ref?(newrev) diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 8da98cc3909..cbdff0ab060 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -2,15 +2,26 @@ module Gitlab module Database - # The max value of INTEGER type is the same between MySQL and PostgreSQL: + include Gitlab::Metrics::Methods + # https://www.postgresql.org/docs/9.2/static/datatype-numeric.html - # http://dev.mysql.com/doc/refman/5.7/en/integer-types.html MAX_INT_VALUE = 2147483647 + # The max value between MySQL's TIMESTAMP and PostgreSQL's timestampz: # https://www.postgresql.org/docs/9.1/static/datatype-datetime.html # https://dev.mysql.com/doc/refman/5.7/en/datetime.html + # FIXME: this should just be the max value of timestampz MAX_TIMESTAMP_VALUE = Time.at((1 << 31) - 1).freeze + # Minimum schema version from which migrations are supported + # Migrations before this version may have been removed + MIN_SCHEMA_VERSION = 20190506135400 + MIN_SCHEMA_GITLAB_VERSION = '11.11.0' + + define_histogram :gitlab_database_transaction_seconds do + docstring "Time spent in database transactions, in seconds" + end + def self.config ActiveRecord::Base.configurations[Rails.env] end @@ -28,13 +39,14 @@ module Gitlab end def self.human_adapter_name - postgresql? ? 'PostgreSQL' : 'MySQL' - end - - def self.mysql? - adapter_name.casecmp('mysql2').zero? + if postgresql? + 'PostgreSQL' + else + 'Unknown' + end end + # @deprecated def self.postgresql? adapter_name.casecmp('postgresql').zero? end @@ -49,15 +61,14 @@ module Gitlab # Check whether the underlying database is in read-only mode def self.db_read_only? - if postgresql? - pg_is_in_recovery = - ActiveRecord::Base.connection.execute('SELECT pg_is_in_recovery()') - .first.fetch('pg_is_in_recovery') + pg_is_in_recovery = + ActiveRecord::Base + .connection + .execute('SELECT pg_is_in_recovery()') + .first + .fetch('pg_is_in_recovery') - Gitlab::Utils.to_boolean(pg_is_in_recovery) - else - false - end + Gitlab::Utils.to_boolean(pg_is_in_recovery) end def self.db_read_write? @@ -69,19 +80,19 @@ module Gitlab end def self.postgresql_9_or_less? - postgresql? && version.to_f < 10 + version.to_f < 10 end def self.join_lateral_supported? - postgresql? && version.to_f >= 9.3 + version.to_f >= 9.3 end def self.replication_slots_supported? - postgresql? && version.to_f >= 9.4 + version.to_f >= 9.4 end def self.postgresql_minimum_supported_version? - postgresql? && version.to_f >= 9.6 + version.to_f >= 9.6 end # map some of the function names that changed between PostgreSQL 9 and 10 @@ -107,51 +118,23 @@ module Gitlab end def self.nulls_last_order(field, direction = 'ASC') - order = "#{field} #{direction}" - - if postgresql? - order = "#{order} NULLS LAST" - else - # `field IS NULL` will be `0` for non-NULL columns and `1` for NULL - # columns. In the (default) ascending order, `0` comes first. - order = "#{field} IS NULL, #{order}" if direction == 'ASC' - end - - order + Arel.sql("#{field} #{direction} NULLS LAST") end def self.nulls_first_order(field, direction = 'ASC') - order = "#{field} #{direction}" - - if postgresql? - order = "#{order} NULLS FIRST" - else - # `field IS NULL` will be `0` for non-NULL columns and `1` for NULL - # columns. In the (default) ascending order, `0` comes first. - order = "#{field} IS NULL, #{order}" if direction == 'DESC' - end - - order + Arel.sql("#{field} #{direction} NULLS FIRST") end def self.random - postgresql? ? "RANDOM()" : "RAND()" + "RANDOM()" end def self.true_value - if postgresql? - "'t'" - else - 1 - end + "'t'" end def self.false_value - if postgresql? - "'f'" - else - 0 - end + "'f'" end def self.with_connection_pool(pool_size) @@ -171,7 +154,7 @@ module Gitlab # rows - An Array of Hash instances, each mapping the columns to their # values. # return_ids - When set to true the return value will be an Array of IDs of - # the inserted rows, this only works on PostgreSQL. + # the inserted rows # disable_quote - A key or an Array of keys to exclude from quoting (You # become responsible for protection from SQL injection for # these keys!) @@ -180,7 +163,6 @@ module Gitlab keys = rows.first.keys columns = keys.map { |key| connection.quote_column_name(key) } - return_ids = false if mysql? disable_quote = Array(disable_quote).to_set tuples = rows.map do |row| @@ -234,6 +216,7 @@ module Gitlab def self.connection ActiveRecord::Base.connection end + private_class_method :connection def self.cached_column_exists?(table_name, column_name) connection.schema_cache.columns_hash(table_name).has_key?(column_name.to_s) @@ -243,16 +226,10 @@ module Gitlab connection.schema_cache.data_source_exists?(table_name) end - private_class_method :connection - def self.database_version row = connection.execute("SELECT VERSION()").first - if postgresql? - row['version'] - else - row.first - end + row['version'] end private_class_method :database_version @@ -272,5 +249,58 @@ module Gitlab end end end + + # inside_transaction? will return true if the caller is running within a transaction. Handles special cases + # when running inside a test environment, where tests may be wrapped in transactions + def self.inside_transaction? + if Rails.env.test? + ActiveRecord::Base.connection.open_transactions > open_transactions_baseline + else + ActiveRecord::Base.connection.open_transactions > 0 + end + end + + # These methods that access @open_transactions_baseline are not thread-safe. + # These are fine though because we only call these in RSpec's main thread. If we decide to run + # specs multi-threaded, we would need to use something like ThreadGroup to keep track of this value + def self.set_open_transactions_baseline + @open_transactions_baseline = ActiveRecord::Base.connection.open_transactions + end + + def self.reset_open_transactions_baseline + @open_transactions_baseline = 0 + end + + def self.open_transactions_baseline + @open_transactions_baseline ||= 0 + end + private_class_method :open_transactions_baseline + + # Monkeypatch rails with upgraded database observability + def self.install_monkey_patches + ActiveRecord::Base.prepend(ActiveRecordBaseTransactionMetrics) + end + + # observe_transaction_duration is called from ActiveRecordBaseTransactionMetrics.transaction and used to + # record transaction durations. + def self.observe_transaction_duration(duration_seconds) + labels = Gitlab::Metrics::Transaction.current&.labels || {} + gitlab_database_transaction_seconds.observe(labels, duration_seconds) + rescue Prometheus::Client::LabelSetValidator::LabelSetError => err + # Ensure that errors in recording these metrics don't affect the operation of the application + Rails.logger.error("Unable to observe database transaction duration: #{err}") # rubocop:disable Gitlab/RailsLogger + end + + # MonkeyPatch for ActiveRecord::Base for adding observability + module ActiveRecordBaseTransactionMetrics + # A monkeypatch over ActiveRecord::Base.transaction. + # It provides observability into transactional methods. + def transaction(options = {}, &block) + start_time = Gitlab::Metrics::System.monotonic_time + super(options, &block) + ensure + Gitlab::Database.observe_transaction_duration(Gitlab::Metrics::System.monotonic_time - start_time) + end + end end end diff --git a/lib/gitlab/database/count.rb b/lib/gitlab/database/count.rb index f3d37ccd72a..eac61254bdf 100644 --- a/lib/gitlab/database/count.rb +++ b/lib/gitlab/database/count.rb @@ -37,16 +37,14 @@ module Gitlab # @return [Hash] of Model -> count mapping def self.approximate_counts(models, strategies: [TablesampleCountStrategy, ReltuplesCountStrategy, ExactCountStrategy]) strategies.each_with_object({}) do |strategy, counts_by_model| - if strategy.enabled? - models_with_missing_counts = models - counts_by_model.keys + models_with_missing_counts = models - counts_by_model.keys - break counts_by_model if models_with_missing_counts.empty? + break counts_by_model if models_with_missing_counts.empty? - counts = strategy.new(models_with_missing_counts).count + counts = strategy.new(models_with_missing_counts).count - counts.each do |model, count| - counts_by_model[model] = count - end + counts.each do |model, count| + counts_by_model[model] = count end end end diff --git a/lib/gitlab/database/count/exact_count_strategy.rb b/lib/gitlab/database/count/exact_count_strategy.rb index fa6951eda22..0b8fe640bf8 100644 --- a/lib/gitlab/database/count/exact_count_strategy.rb +++ b/lib/gitlab/database/count/exact_count_strategy.rb @@ -23,10 +23,6 @@ module Gitlab rescue *CONNECTION_ERRORS {} end - - def self.enabled? - true - end end end end diff --git a/lib/gitlab/database/count/reltuples_count_strategy.rb b/lib/gitlab/database/count/reltuples_count_strategy.rb index 695f6fa766e..6cd90c01ab2 100644 --- a/lib/gitlab/database/count/reltuples_count_strategy.rb +++ b/lib/gitlab/database/count/reltuples_count_strategy.rb @@ -31,10 +31,6 @@ module Gitlab {} end - def self.enabled? - Gitlab::Database.postgresql? - end - private # Models using single-type inheritance (STI) don't work with diff --git a/lib/gitlab/database/count/tablesample_count_strategy.rb b/lib/gitlab/database/count/tablesample_count_strategy.rb index 7777f31f702..e9387a91a14 100644 --- a/lib/gitlab/database/count/tablesample_count_strategy.rb +++ b/lib/gitlab/database/count/tablesample_count_strategy.rb @@ -28,10 +28,6 @@ module Gitlab {} end - def self.enabled? - Gitlab::Database.postgresql? && Feature.enabled?(:tablesample_counts) - end - private def perform_count(model, estimate) diff --git a/lib/gitlab/database/date_time.rb b/lib/gitlab/database/date_time.rb index 79d2caff151..1392b397012 100644 --- a/lib/gitlab/database/date_time.rb +++ b/lib/gitlab/database/date_time.rb @@ -7,8 +7,7 @@ module Gitlab # the first of the `start_time_attrs` that isn't NULL. `SELECT` the resulting interval # along with an alias specified by the `as` parameter. # - # Note: For MySQL, the interval is returned in seconds. - # For PostgreSQL, the interval is returned as an INTERVAL type. + # Note: the interval is returned as an INTERVAL type. def subtract_datetimes(query_so_far, start_time_attrs, end_time_attrs, as) diff_fn = subtract_datetimes_diff(query_so_far, start_time_attrs, end_time_attrs) @@ -16,17 +15,10 @@ module Gitlab end def subtract_datetimes_diff(query_so_far, start_time_attrs, end_time_attrs) - if Gitlab::Database.postgresql? - Arel::Nodes::Subtraction.new( - Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(end_time_attrs)), - Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(start_time_attrs))) - elsif Gitlab::Database.mysql? - Arel::Nodes::NamedFunction.new( - "TIMESTAMPDIFF", - [Arel.sql('second'), - Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(start_time_attrs)), - Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(end_time_attrs))]) - end + Arel::Nodes::Subtraction.new( + Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(end_time_attrs)), + Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(start_time_attrs)) + ) end end end diff --git a/lib/gitlab/database/grant.rb b/lib/gitlab/database/grant.rb index 862ab96c887..1f47f320a29 100644 --- a/lib/gitlab/database/grant.rb +++ b/lib/gitlab/database/grant.rb @@ -6,47 +6,25 @@ module Gitlab class Grant < ActiveRecord::Base include FromUnion - self.table_name = - if Database.postgresql? - 'information_schema.role_table_grants' - else - 'information_schema.schema_privileges' - end + self.table_name = 'information_schema.role_table_grants' # Returns true if the current user can create and execute triggers on the # given table. def self.create_and_execute_trigger?(table) - if Database.postgresql? - # We _must not_ use quote_table_name as this will produce double - # quotes on PostgreSQL and for "has_table_privilege" we need single - # quotes. - quoted_table = connection.quote(table) - - begin - from(nil) - .pluck("has_table_privilege(#{quoted_table}, 'TRIGGER')") - .first - rescue ActiveRecord::StatementInvalid - # This error is raised when using a non-existing table name. In this - # case we just want to return false as a user technically can't - # create triggers for such a table. - false - end - else - queries = [ - Grant.select(1) - .from('information_schema.user_privileges') - .where("PRIVILEGE_TYPE = 'SUPER'") - .where("GRANTEE = CONCAT('\\'', REPLACE(CURRENT_USER(), '@', '\\'@\\''), '\\'')"), - - Grant.select(1) - .from('information_schema.schema_privileges') - .where("PRIVILEGE_TYPE = 'TRIGGER'") - .where('TABLE_SCHEMA = ?', Gitlab::Database.database_name) - .where("GRANTEE = CONCAT('\\'', REPLACE(CURRENT_USER(), '@', '\\'@\\''), '\\'')") - ] + # We _must not_ use quote_table_name as this will produce double + # quotes on PostgreSQL and for "has_table_privilege" we need single + # quotes. + quoted_table = connection.quote(table) - Grant.from_union(queries, alias_as: 'privs').any? + begin + from(nil) + .pluck(Arel.sql("has_table_privilege(#{quoted_table}, 'TRIGGER')")) + .first + rescue ActiveRecord::StatementInvalid + # This error is raised when using a non-existing table name. In this + # case we just want to return false as a user technically can't + # create triggers for such a table. + false end end end diff --git a/lib/gitlab/database/median.rb b/lib/gitlab/database/median.rb index 1455e410d4b..603b125d8b4 100644 --- a/lib/gitlab/database/median.rb +++ b/lib/gitlab/database/median.rb @@ -17,13 +17,9 @@ module Gitlab def extract_median(results) result = results.compact.first - if Gitlab::Database.postgresql? - result = result.first.presence + result = result.first.presence - result['median']&.to_f if result - elsif Gitlab::Database.mysql? - result.to_a.flatten.first - end + result['median']&.to_f if result end def extract_medians(results) @@ -34,31 +30,6 @@ module Gitlab end end - def mysql_median_datetime_sql(arel_table, query_so_far, column_sym) - query = arel_table.from - .from(arel_table.project(Arel.sql('*')).order(arel_table[column_sym]).as(arel_table.table_name)) - .project(average([arel_table[column_sym]], 'median')) - .where( - Arel::Nodes::Between.new( - Arel.sql("(select @row_id := @row_id + 1)"), - Arel::Nodes::And.new( - [Arel.sql('@ct/2.0'), - Arel.sql('@ct/2.0 + 1')] - ) - ) - ). - # Disallow negative values - where(arel_table[column_sym].gteq(0)) - - [ - Arel.sql("CREATE TEMPORARY TABLE IF NOT EXISTS #{query_so_far.to_sql}"), - Arel.sql("set @ct := (select count(1) from #{arel_table.table_name});"), - Arel.sql("set @row_id := 0;"), - query.to_sql, - Arel.sql("DROP TEMPORARY TABLE IF EXISTS #{arel_table.table_name};") - ] - end - def pg_median_datetime_sql(arel_table, query_so_far, column_sym, partition_column = nil) # Create a CTE with the column we're operating on, row number (after sorting by the column # we're operating on), and count of the table we're operating on (duplicated across) all rows @@ -113,18 +84,8 @@ module Gitlab private - def median_queries(arel_table, query_so_far, column_sym, partition_column = nil) - if Gitlab::Database.postgresql? - pg_median_datetime_sql(arel_table, query_so_far, column_sym, partition_column) - elsif Gitlab::Database.mysql? - raise NotSupportedError, "partition_column is not supported for MySQL" if partition_column - - mysql_median_datetime_sql(arel_table, query_so_far, column_sym) - end - end - def execute_queries(arel_table, query_so_far, column_sym, partition_column = nil) - queries = median_queries(arel_table, query_so_far, column_sym, partition_column) + queries = pg_median_datetime_sql(arel_table, query_so_far, column_sym, partition_column) Array.wrap(queries).map { |query| ActiveRecord::Base.connection.execute(query) } end @@ -158,7 +119,7 @@ module Gitlab Arel::Nodes::Window.new.order(arel_table[column_sym]) ).as('row_id') - count = arel_table.project("COUNT(1)").as('ct') + count = arel_table.where(arel_table[column_sym].gteq(zero_interval)).project("COUNT(1)").as('ct') [column_row, row_id, count] end @@ -176,8 +137,6 @@ module Gitlab end def extract_diff_epoch(diff) - return diff unless Gitlab::Database.postgresql? - Arel.sql(%Q{EXTRACT(EPOCH FROM (#{diff.to_sql}))}) end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index cc61bb7fa02..9bba4f6ce1e 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -6,31 +6,45 @@ module Gitlab BACKGROUND_MIGRATION_BATCH_SIZE = 1000 # Number of rows to process per job BACKGROUND_MIGRATION_JOB_BUFFER_SIZE = 1000 # Number of jobs to bulk queue at a time + PERMITTED_TIMESTAMP_COLUMNS = %i[created_at updated_at deleted_at].to_set.freeze + DEFAULT_TIMESTAMP_COLUMNS = %i[created_at updated_at].freeze + # Adds `created_at` and `updated_at` columns with timezone information. # # This method is an improved version of Rails' built-in method `add_timestamps`. # + # By default, adds `created_at` and `updated_at` columns, but these can be specified as: + # + # add_timestamps_with_timezone(:my_table, columns: [:created_at, :deleted_at]) + # + # This allows you to create just the timestamps you need, saving space. + # # Available options are: - # default - The default value for the column. - # null - When set to `true` the column will allow NULL values. + # :default - The default value for the column. + # :null - When set to `true` the column will allow NULL values. # The default is to not allow NULL values. + # :columns - the column names to create. Must be one + # of `Gitlab::Database::MigrationHelpers::PERMITTED_TIMESTAMP_COLUMNS`. + # Default value: `DEFAULT_TIMESTAMP_COLUMNS` + # + # All options are optional. def add_timestamps_with_timezone(table_name, options = {}) options[:null] = false if options[:null].nil? + columns = options.fetch(:columns, DEFAULT_TIMESTAMP_COLUMNS) + default_value = options[:default] - [:created_at, :updated_at].each do |column_name| - if options[:default] && transaction_open? - raise '`add_timestamps_with_timezone` with default value cannot be run inside a transaction. ' \ - 'You can disable transactions by calling `disable_ddl_transaction!` ' \ - 'in the body of your migration class' - end + validate_not_in_transaction!(:add_timestamps_with_timezone, 'with default value') if default_value + + columns.each do |column_name| + validate_timestamp_column_name!(column_name) # If default value is presented, use `add_column_with_default` method instead. - if options[:default] + if default_value add_column_with_default( table_name, column_name, :datetime_with_timezone, - default: options[:default], + default: default_value, allow_null: options[:null] ) else @@ -39,10 +53,22 @@ module Gitlab end end - # Creates a new index, concurrently when supported + # To be used in the `#down` method of migrations that + # use `#add_timestamps_with_timezone`. # - # On PostgreSQL this method creates an index concurrently, on MySQL this - # creates a regular index. + # Available options are: + # :columns - the column names to remove. Must be one + # Default value: `DEFAULT_TIMESTAMP_COLUMNS` + # + # All options are optional. + def remove_timestamps(table_name, options = {}) + columns = options.fetch(:columns, DEFAULT_TIMESTAMP_COLUMNS) + columns.each do |column_name| + remove_column(table_name, column_name) + end + end + + # Creates a new index, concurrently # # Example: # @@ -56,12 +82,10 @@ module Gitlab 'in the body of your migration class' end - if Database.postgresql? - options = options.merge({ algorithm: :concurrently }) - end + options = options.merge({ algorithm: :concurrently }) if index_exists?(table_name, column_name, options) - Rails.logger.warn "Index not created because it already exists (this may be due to an aborted migration or similar): table_name: #{table_name}, column_name: #{column_name}" + Rails.logger.warn "Index not created because it already exists (this may be due to an aborted migration or similar): table_name: #{table_name}, column_name: #{column_name}" # rubocop:disable Gitlab/RailsLogger return end @@ -70,9 +94,7 @@ module Gitlab end end - # Removes an existed index, concurrently when supported - # - # On PostgreSQL this method removes an index concurrently. + # Removes an existed index, concurrently # # Example: # @@ -91,7 +113,7 @@ module Gitlab end unless index_exists?(table_name, column_name, options) - Rails.logger.warn "Index not removed because it does not exist (this may be due to an aborted migration or similar): table_name: #{table_name}, column_name: #{column_name}" + Rails.logger.warn "Index not removed because it does not exist (this may be due to an aborted migration or similar): table_name: #{table_name}, column_name: #{column_name}" # rubocop:disable Gitlab/RailsLogger return end @@ -100,9 +122,7 @@ module Gitlab end end - # Removes an existing index, concurrently when supported - # - # On PostgreSQL this method removes an index concurrently. + # Removes an existing index, concurrently # # Example: # @@ -121,7 +141,7 @@ module Gitlab end unless index_exists_by_name?(table_name, index_name) - Rails.logger.warn "Index not removed because it does not exist (this may be due to an aborted migration or similar): table_name: #{table_name}, index_name: #{index_name}" + Rails.logger.warn "Index not removed because it does not exist (this may be due to an aborted migration or similar): table_name: #{table_name}, index_name: #{index_name}" # rubocop:disable Gitlab/RailsLogger return end @@ -132,8 +152,6 @@ module Gitlab # Only available on Postgresql >= 9.2 def supports_drop_index_concurrently? - return false unless Database.postgresql? - version = select_one("SELECT current_setting('server_version_num') AS v")['v'].to_i version >= 90200 @@ -141,40 +159,25 @@ module Gitlab # Adds a foreign key with only minimal locking on the tables involved. # - # This method only requires minimal locking when using PostgreSQL. When - # using MySQL this method will use Rails' default `add_foreign_key`. + # This method only requires minimal locking # # source - The source table containing the foreign key. # target - The target table the key points to. # column - The name of the column to create the foreign key on. # on_delete - The action to perform when associated data is removed, # defaults to "CASCADE". - def add_concurrent_foreign_key(source, target, column:, on_delete: :cascade) + # + # rubocop:disable Gitlab/RailsLogger + def add_concurrent_foreign_key(source, target, column:, on_delete: :cascade, name: nil) # Transactions would result in ALTER TABLE locks being held for the # duration of the transaction, defeating the purpose of this method. if transaction_open? raise 'add_concurrent_foreign_key can not be run inside a transaction' end - # While MySQL does allow disabling of foreign keys it has no equivalent - # of PostgreSQL's "VALIDATE CONSTRAINT". As a result we'll just fall - # back to the normal foreign key procedure. - if Database.mysql? - if foreign_key_exists?(source, target, column: column) - Rails.logger.warn "Foreign key not created because it exists already " \ - "(this may be due to an aborted migration or similar): " \ - "source: #{source}, target: #{target}, column: #{column}" - return - end - - return add_foreign_key(source, target, - column: column, - on_delete: on_delete) - else - on_delete = 'SET NULL' if on_delete == :nullify - end + on_delete = 'SET NULL' if on_delete == :nullify - key_name = concurrent_foreign_key_name(source, column) + key_name = name || concurrent_foreign_key_name(source, column) unless foreign_key_exists?(source, target, column: column) Rails.logger.warn "Foreign key not created because it exists already " \ @@ -204,6 +207,7 @@ module Gitlab execute("ALTER TABLE #{source} VALIDATE CONSTRAINT #{key_name};") end end + # rubocop:enable Gitlab/RailsLogger def foreign_key_exists?(source, target = nil, column: nil) foreign_keys(source).any? do |key| @@ -221,12 +225,15 @@ module Gitlab # here is based on Rails' foreign_key_name() method, which unfortunately # is private so we can't rely on it directly. def concurrent_foreign_key_name(table, column) - "fk_#{Digest::SHA256.hexdigest("#{table}_#{column}_fk").first(10)}" + identifier = "#{table}_#{column}_fk" + hashed_identifier = Digest::SHA256.hexdigest(identifier).first(10) + + "fk_#{hashed_identifier}" end # Long-running migrations may take more than the timeout allowed by # the database. Disable the session's statement timeout to ensure - # migrations don't get killed prematurely. (PostgreSQL only) + # migrations don't get killed prematurely. # # There are two possible ways to disable the statement timeout: # @@ -238,15 +245,6 @@ module Gitlab # otherwise the statement will still be disabled until connection is dropped # or `RESET ALL` is executed def disable_statement_timeout - # bypass disabled_statement logic when not using postgres, but still execute block when one is given - unless Database.postgresql? - if block_given? - yield - end - - return - end - if block_given? begin execute('SET statement_timeout TO 0') @@ -427,7 +425,8 @@ module Gitlab end begin - update_column_in_batches(table, column, default, &block) + default_after_type_cast = connection.type_cast(default, column_for(table, column)) + update_column_in_batches(table, column, default_after_type_cast, &block) change_column_null(table, column, false) unless allow_null # We want to rescue _all_ exceptions here, even those that don't inherit @@ -495,13 +494,12 @@ module Gitlab quoted_old = quote_column_name(old_column) quoted_new = quote_column_name(new_column) - if Database.postgresql? - install_rename_triggers_for_postgresql(trigger_name, quoted_table, - quoted_old, quoted_new) - else - install_rename_triggers_for_mysql(trigger_name, quoted_table, - quoted_old, quoted_new) - end + install_rename_triggers_for_postgresql( + trigger_name, + quoted_table, + quoted_old, + quoted_new + ) end # Changes the type of a column concurrently. @@ -544,11 +542,7 @@ module Gitlab check_trigger_permissions!(table) - if Database.postgresql? - remove_rename_triggers_for_postgresql(table, trigger_name) - else - remove_rename_triggers_for_mysql(trigger_name) - end + remove_rename_triggers_for_postgresql(table, trigger_name) remove_column(table, old) end @@ -761,38 +755,12 @@ module Gitlab EOF end - # Installs the triggers necessary to perform a concurrent column rename on - # MySQL. - def install_rename_triggers_for_mysql(trigger, table, old, new) - execute <<-EOF.strip_heredoc - CREATE TRIGGER #{trigger}_insert - BEFORE INSERT - ON #{table} - FOR EACH ROW - SET NEW.#{new} = NEW.#{old} - EOF - - execute <<-EOF.strip_heredoc - CREATE TRIGGER #{trigger}_update - BEFORE UPDATE - ON #{table} - FOR EACH ROW - SET NEW.#{new} = NEW.#{old} - EOF - end - # Removes the triggers used for renaming a PostgreSQL column concurrently. def remove_rename_triggers_for_postgresql(table, trigger) execute("DROP TRIGGER IF EXISTS #{trigger} ON #{table}") execute("DROP FUNCTION IF EXISTS #{trigger}()") end - # Removes the triggers used for renaming a MySQL column concurrently. - def remove_rename_triggers_for_mysql(trigger) - execute("DROP TRIGGER IF EXISTS #{trigger}_insert") - execute("DROP TRIGGER IF EXISTS #{trigger}_update") - end - # Returns the (base) name to use for triggers when renaming columns. def rename_trigger_name(table, old, new) 'trigger_' + Digest::SHA256.hexdigest("#{table}_#{old}_#{new}").first(12) @@ -842,8 +810,6 @@ module Gitlab order: index.orders } - # These options are not supported by MySQL, so we only add them if - # they were previously set. options[:using] = index.using if index.using options[:where] = index.where if index.where @@ -883,26 +849,16 @@ module Gitlab end # This will replace the first occurrence of a string in a column with - # the replacement - # On postgresql we can use `regexp_replace` for that. - # On mysql we find the location of the pattern, and overwrite it - # with the replacement + # the replacement using `regexp_replace` def replace_sql(column, pattern, replacement) quoted_pattern = Arel::Nodes::Quoted.new(pattern.to_s) quoted_replacement = Arel::Nodes::Quoted.new(replacement.to_s) - if Database.mysql? - locate = Arel::Nodes::NamedFunction - .new('locate', [quoted_pattern, column]) - insert_in_place = Arel::Nodes::NamedFunction - .new('insert', [column, locate, pattern.size, quoted_replacement]) + replace = Arel::Nodes::NamedFunction.new( + "regexp_replace", [column, quoted_pattern, quoted_replacement] + ) - Arel::Nodes::SqlLiteral.new(insert_in_place.to_sql) - else - replace = Arel::Nodes::NamedFunction - .new("regexp_replace", [column, quoted_pattern, quoted_replacement]) - Arel::Nodes::SqlLiteral.new(replace.to_sql) - end + Arel::Nodes::SqlLiteral.new(replace.to_sql) end def remove_foreign_key_if_exists(*args) @@ -944,11 +900,7 @@ database (#{dbname}) using a super user and running: ALTER #{user} WITH SUPERUSER -For MySQL you instead need to run: - - GRANT ALL PRIVILEGES ON #{dbname}.* TO #{user}@'%' - -Both queries will grant the user super user permissions, ensuring you don't run +This query will grant the user super user permissions, ensuring you don't run into similar problems in the future (e.g. when new tables are created). EOF end @@ -1051,10 +1003,6 @@ into similar problems in the future (e.g. when new tables are created). # This will include indexes using an expression on the column, for example: # `CREATE INDEX CONCURRENTLY index_name ON table (LOWER(column));` # - # For mysql, it falls back to the default ActiveRecord implementation that - # will not find custom indexes. But it will select by name without passing - # a column. - # # We can remove this when upgrading to Rails 5 with an updated `index_exists?`: # - https://github.com/rails/rails/commit/edc2b7718725016e988089b5fb6d6fb9d6e16882 # @@ -1065,10 +1013,8 @@ into similar problems in the future (e.g. when new tables are created). # does not find indexes without passing a column name. if indexes(table).map(&:name).include?(index.to_s) true - elsif Gitlab::Database.postgresql? - postgres_exists_by_name?(table, index) else - false + postgres_exists_by_name?(table, index) end end @@ -1084,8 +1030,26 @@ into similar problems in the future (e.g. when new tables are created). connection.select_value(index_sql).to_i > 0 end - def mysql_compatible_index_length - Gitlab::Database.mysql? ? 20 : nil + private + + def validate_timestamp_column_name!(column_name) + return if PERMITTED_TIMESTAMP_COLUMNS.member?(column_name) + + raise <<~MESSAGE + Illegal timestamp column name! Got #{column_name}. + Must be one of: #{PERMITTED_TIMESTAMP_COLUMNS.to_a} + MESSAGE + end + + def validate_not_in_transaction!(method_name, modifier = nil) + return unless transaction_open? + + raise <<~ERROR + #{["`#{method_name}`", modifier].compact.join(' ')} cannot be run inside a transaction. + + You can disable transactions by calling `disable_ddl_transaction!` in the body of + your migration class + ERROR end end end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb index 60afa4bcd52..565f34b78b7 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb @@ -51,14 +51,10 @@ module Gitlab quoted_old_full_path = quote_string(old_full_path) quoted_old_wildcard_path = quote_string("#{old_full_path}/%") - filter = if Database.mysql? - "lower(routes.path) = lower('#{quoted_old_full_path}') "\ - "OR routes.path LIKE '#{quoted_old_wildcard_path}'" - else - "routes.id IN "\ - "( SELECT routes.id FROM routes WHERE lower(routes.path) = lower('#{quoted_old_full_path}') "\ - "UNION SELECT routes.id FROM routes WHERE routes.path ILIKE '#{quoted_old_wildcard_path}' )" - end + filter = + "routes.id IN "\ + "( SELECT routes.id FROM routes WHERE lower(routes.path) = lower('#{quoted_old_full_path}') "\ + "UNION SELECT routes.id FROM routes WHERE routes.path ILIKE '#{quoted_old_wildcard_path}' )" replace_statement = replace_sql(Route.arel_table[:path], old_full_path, diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb index 6bbad707f0f..3e8a9b89998 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb @@ -70,7 +70,7 @@ module Gitlab unless gitlab_shell.mv_namespace(repository_storage, old_full_path, new_full_path) message = "Exception moving on shard #{repository_storage} from #{old_full_path} to #{new_full_path}" - Rails.logger.error message + Rails.logger.error message # rubocop:disable Gitlab/RailsLogger end end end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb index 580be9fe267..4dc7a62797a 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb @@ -56,7 +56,7 @@ module Gitlab unless gitlab_shell.mv_repository(project.repository_storage, old_path, new_path) - Rails.logger.error "Error moving #{old_path} to #{new_path}" + Rails.logger.error "Error moving #{old_path} to #{new_path}" # rubocop:disable Gitlab/RailsLogger end end diff --git a/lib/gitlab/database/sha_attribute.rb b/lib/gitlab/database/sha_attribute.rb index 109ae7893da..ddbabc9098e 100644 --- a/lib/gitlab/database/sha_attribute.rb +++ b/lib/gitlab/database/sha_attribute.rb @@ -2,14 +2,9 @@ module Gitlab module Database - BINARY_TYPE = - if Gitlab::Database.postgresql? - # PostgreSQL defines its own class with slightly different - # behaviour from the default Binary type. - ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Bytea - else - ActiveModel::Type::Binary - end + # PostgreSQL defines its own class with slightly different + # behaviour from the default Binary type. + BINARY_TYPE = ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Bytea # Class for casting binary data to hexadecimal SHA1 hashes (and vice-versa). # diff --git a/lib/gitlab/database/subquery.rb b/lib/gitlab/database/subquery.rb index 10971d2b274..2a6f39c6a27 100644 --- a/lib/gitlab/database/subquery.rb +++ b/lib/gitlab/database/subquery.rb @@ -6,11 +6,7 @@ module Gitlab class << self def self_join(relation) t = relation.arel_table - # Work around a bug in Rails 5, where LIMIT causes trouble - # See https://gitlab.com/gitlab-org/gitlab-ce/issues/51729 - r = relation.limit(nil).arel - r.take(relation.limit_value) if relation.limit_value - t2 = r.as('t2') + t2 = relation.arel.as('t2') relation.unscoped.joins(t.join(t2).on(t[:id].eq(t2[:id])).join_sources.first) end diff --git a/lib/gitlab/database_importers/common_metrics.rb b/lib/gitlab/database_importers/common_metrics.rb new file mode 100644 index 00000000000..f964ae8a275 --- /dev/null +++ b/lib/gitlab/database_importers/common_metrics.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +module Gitlab + module DatabaseImporters + module CommonMetrics + end + end +end diff --git a/lib/gitlab/database_importers/common_metrics/importer.rb b/lib/gitlab/database_importers/common_metrics/importer.rb new file mode 100644 index 00000000000..6c61e05674e --- /dev/null +++ b/lib/gitlab/database_importers/common_metrics/importer.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +module Gitlab + module DatabaseImporters + module CommonMetrics + class Importer + MissingQueryId = Class.new(StandardError) + + attr_reader :content + + def initialize(filename = 'common_metrics.yml') + @content = YAML.load_file(Rails.root.join('config', 'prometheus', filename)) + end + + def execute + CommonMetrics::PrometheusMetric.reset_column_information + + process_content do |id, attributes| + find_or_build_metric!(id) + .update!(**attributes) + end + end + + private + + def process_content(&blk) + content['panel_groups'].map do |group| + process_group(group, &blk) + end + end + + def process_group(group, &blk) + attributes = { + group: find_group_title_key(group['group']) + } + + group['panels'].map do |panel| + process_panel(panel, attributes, &blk) + end + end + + def process_panel(panel, attributes, &blk) + attributes = attributes.merge( + title: panel['title'], + y_label: panel['y_label']) + + panel['metrics'].map do |metric_details| + process_metric_details(metric_details, attributes, &blk) + end + end + + def process_metric_details(metric_details, attributes, &blk) + attributes = attributes.merge( + legend: metric_details['label'], + query: metric_details['query_range'], + unit: metric_details['unit']) + + yield(metric_details['id'], attributes) + end + + def find_or_build_metric!(id) + raise MissingQueryId unless id + + CommonMetrics::PrometheusMetric.common.find_by(identifier: id) || + CommonMetrics::PrometheusMetric.new(common: true, identifier: id) + end + + def find_group_title_key(title) + CommonMetrics::PrometheusMetricEnums.groups[find_group_title(title)] + end + + def find_group_title(title) + CommonMetrics::PrometheusMetricEnums.group_titles.invert[title] + end + end + end + end +end diff --git a/lib/gitlab/database_importers/common_metrics/prometheus_metric.rb b/lib/gitlab/database_importers/common_metrics/prometheus_metric.rb new file mode 100644 index 00000000000..b4a392cbea9 --- /dev/null +++ b/lib/gitlab/database_importers/common_metrics/prometheus_metric.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Gitlab + module DatabaseImporters + module CommonMetrics + class PrometheusMetric < ApplicationRecord + enum group: PrometheusMetricEnums.groups + scope :common, -> { where(common: true) } + end + end + end +end diff --git a/lib/gitlab/database_importers/common_metrics/prometheus_metric_enums.rb b/lib/gitlab/database_importers/common_metrics/prometheus_metric_enums.rb new file mode 100644 index 00000000000..c9e957ec7c0 --- /dev/null +++ b/lib/gitlab/database_importers/common_metrics/prometheus_metric_enums.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module DatabaseImporters + module CommonMetrics + module PrometheusMetricEnums + def self.groups + { + # built-in groups + nginx_ingress_vts: -1, + ha_proxy: -2, + aws_elb: -3, + nginx: -4, + kubernetes: -5, + nginx_ingress: -6, + + # custom groups + business: 0, + response: 1, + system: 2 + } + end + + def self.group_titles + { + business: _('Business metrics (Custom)'), + response: _('Response metrics (Custom)'), + system: _('System metrics (Custom)'), + nginx_ingress_vts: _('Response metrics (NGINX Ingress VTS)'), + nginx_ingress: _('Response metrics (NGINX Ingress)'), + ha_proxy: _('Response metrics (HA Proxy)'), + aws_elb: _('Response metrics (AWS ELB)'), + nginx: _('Response metrics (NGINX)'), + kubernetes: _('System metrics (Kubernetes)') + } + end + end + end + end +end diff --git a/lib/gitlab/diff/lines_unfolder.rb b/lib/gitlab/diff/lines_unfolder.rb index 6cf904b2b2a..0bd18fe9622 100644 --- a/lib/gitlab/diff/lines_unfolder.rb +++ b/lib/gitlab/diff/lines_unfolder.rb @@ -54,7 +54,7 @@ module Gitlab def unfold_required? strong_memoize(:unfold_required) do next false unless @diff_file.text? - next false unless @position.unchanged? + next false unless @position.on_text? && @position.unchanged? next false if @diff_file.new_file? || @diff_file.deleted_file? next false unless @position.old_line # Invalid position (MR import scenario) diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb index e8f98f52111..dfa80eb4a64 100644 --- a/lib/gitlab/diff/position.rb +++ b/lib/gitlab/diff/position.rb @@ -134,6 +134,18 @@ module Gitlab @line_code ||= diff_file(repository)&.line_code_for_position(self) end + def file_hash + @file_hash ||= Digest::SHA1.hexdigest(file_path) + end + + def on_image? + position_type == 'image' + end + + def on_text? + position_type == 'text' + end + private def find_diff_file(repository) diff --git a/lib/gitlab/diff/position_tracer.rb b/lib/gitlab/diff/position_tracer.rb index af3df820422..a1c82ce9afc 100644 --- a/lib/gitlab/diff/position_tracer.rb +++ b/lib/gitlab/diff/position_tracer.rb @@ -17,187 +17,13 @@ module Gitlab @paths = paths end - def trace(ab_position) + def trace(old_position) return unless old_diff_refs&.complete? && new_diff_refs&.complete? - return unless ab_position.diff_refs == old_diff_refs + return unless old_position.diff_refs == old_diff_refs - # Suppose we have an MR with source branch `feature` and target branch `master`. - # When the MR was created, the head of `master` was commit A, and the - # head of `feature` was commit B, resulting in the original diff A->B. - # Since creation, `master` was updated to C. - # Now `feature` is being updated to D, and the newly generated MR diff is C->D. - # It is possible that C and D are direct descendants of A and B respectively, - # but this isn't necessarily the case as rebases and merges come into play. - # - # Suppose we have a diff note on the original diff A->B. Now that the MR - # is updated, we need to find out what line in C->D corresponds to the - # line the note was originally created on, so that we can update the diff note's - # records and continue to display it in the right place in the diffs. - # If we cannot find this line in the new diff, this means the diff note is now - # outdated, and we will display that fact to the user. - # - # In the new diff, the file the diff note was originally created on may - # have been renamed, deleted or even created, if the file existed in A and B, - # but was removed in C, and restored in D. - # - # Every diff note stores a Position object that defines a specific location, - # identified by paths and line numbers, within a specific diff, identified - # by start, head and base commit ids. - # - # For diff notes for diff A->B, the position looks like this: - # Position - # start_sha - ID of commit A - # head_sha - ID of commit B - # base_sha - ID of base commit of A and B - # old_path - path as of A (nil if file was newly created) - # new_path - path as of B (nil if file was deleted) - # old_line - line number as of A (nil if file was newly created) - # new_line - line number as of B (nil if file was deleted) - # - # We can easily update `start_sha` and `head_sha` to hold the IDs of - # commits C and D, and can trivially determine `base_sha` based on those, - # but need to find the paths and line numbers as of C and D. - # - # If the file was unchanged or newly created in A->B, the path as of D can be found - # by generating diff B->D ("head to head"), finding the diff file with - # `diff_file.old_path == position.new_path`, and taking `diff_file.new_path`. - # The path as of C can be found by taking diff C->D, finding the diff file - # with that same `new_path` and taking `diff_file.old_path`. - # The line number as of D can be found by using the LineMapper on diff B->D - # and providing the line number as of B. - # The line number as of C can be found by using the LineMapper on diff C->D - # and providing the line number as of D. - # - # If the file was deleted in A->B, the path as of C can be found - # by generating diff A->C ("base to base"), finding the diff file with - # `diff_file.old_path == position.old_path`, and taking `diff_file.new_path`. - # The path as of D can be found by taking diff C->D, finding the diff file - # with `old_path` set to that `diff_file.new_path` and taking `diff_file.new_path`. - # The line number as of C can be found by using the LineMapper on diff A->C - # and providing the line number as of A. - # The line number as of D can be found by using the LineMapper on diff C->D - # and providing the line number as of C. + strategy = old_position.on_text? ? LineStrategy : ImageStrategy - if ab_position.added? - trace_added_line(ab_position) - elsif ab_position.removed? - trace_removed_line(ab_position) - else # unchanged - trace_unchanged_line(ab_position) - end - end - - private - - def trace_added_line(ab_position) - b_path = ab_position.new_path - b_line = ab_position.new_line - - bd_diff = bd_diffs.diff_file_with_old_path(b_path) - - d_path = bd_diff&.new_path || b_path - d_line = LineMapper.new(bd_diff).old_to_new(b_line) - - if d_line - cd_diff = cd_diffs.diff_file_with_new_path(d_path) - - c_path = cd_diff&.old_path || d_path - c_line = LineMapper.new(cd_diff).new_to_old(d_line) - - if c_line - # If the line is still in D but also in C, it has turned from an - # added line into an unchanged one. - new_position = position(cd_diff, c_line, d_line) - if valid_position?(new_position) - # If the line is still in the MR, we don't treat this as outdated. - { position: new_position, outdated: false } - else - # If the line is no longer in the MR, we unfortunately cannot show - # the current state on the CD diff, so we treat it as outdated. - ac_diff = ac_diffs.diff_file_with_new_path(c_path) - - { position: position(ac_diff, nil, c_line), outdated: true } - end - else - # If the line is still in D and not in C, it is still added. - { position: position(cd_diff, nil, d_line), outdated: false } - end - else - # If the line is no longer in D, it has been removed from the MR. - { position: position(bd_diff, b_line, nil), outdated: true } - end - end - - def trace_removed_line(ab_position) - a_path = ab_position.old_path - a_line = ab_position.old_line - - ac_diff = ac_diffs.diff_file_with_old_path(a_path) - - c_path = ac_diff&.new_path || a_path - c_line = LineMapper.new(ac_diff).old_to_new(a_line) - - if c_line - cd_diff = cd_diffs.diff_file_with_old_path(c_path) - - d_path = cd_diff&.new_path || c_path - d_line = LineMapper.new(cd_diff).old_to_new(c_line) - - if d_line - # If the line is still in C but also in D, it has turned from a - # removed line into an unchanged one. - bd_diff = bd_diffs.diff_file_with_new_path(d_path) - - { position: position(bd_diff, nil, d_line), outdated: true } - else - # If the line is still in C and not in D, it is still removed. - { position: position(cd_diff, c_line, nil), outdated: false } - end - else - # If the line is no longer in C, it has been removed outside of the MR. - { position: position(ac_diff, a_line, nil), outdated: true } - end - end - - def trace_unchanged_line(ab_position) - a_path = ab_position.old_path - a_line = ab_position.old_line - b_path = ab_position.new_path - b_line = ab_position.new_line - - ac_diff = ac_diffs.diff_file_with_old_path(a_path) - - c_path = ac_diff&.new_path || a_path - c_line = LineMapper.new(ac_diff).old_to_new(a_line) - - bd_diff = bd_diffs.diff_file_with_old_path(b_path) - - d_line = LineMapper.new(bd_diff).old_to_new(b_line) - - cd_diff = cd_diffs.diff_file_with_old_path(c_path) - - if c_line && d_line - # If the line is still in C and D, it is still unchanged. - new_position = position(cd_diff, c_line, d_line) - if valid_position?(new_position) - # If the line is still in the MR, we don't treat this as outdated. - { position: new_position, outdated: false } - else - # If the line is no longer in the MR, we unfortunately cannot show - # the current state on the CD diff or any change on the BD diff, - # so we treat it as outdated. - { position: nil, outdated: true } - end - elsif d_line # && !c_line - # If the line is still in D but no longer in C, it has turned from - # an unchanged line into an added one. - # We don't treat this as outdated since the line is still in the MR. - { position: position(cd_diff, nil, d_line), outdated: false } - else # !d_line && (c_line || !c_line) - # If the line is no longer in D, it has turned from an unchanged line - # into a removed one. - { position: position(bd_diff, b_line, nil), outdated: true } - end + strategy.new(self).trace(old_position) end def ac_diffs @@ -216,18 +42,12 @@ module Gitlab @cd_diffs ||= compare(new_diff_refs.start_sha, new_diff_refs.head_sha) end + private + def compare(start_sha, head_sha, straight: false) compare = CompareService.new(project, head_sha).execute(project, start_sha, straight: straight) compare.diffs(paths: paths, expanded: true) end - - def position(diff_file, old_line, new_line) - Position.new(diff_file: diff_file, old_line: old_line, new_line: new_line) - end - - def valid_position?(position) - !!position.diff_line(project.repository) - end end end end diff --git a/lib/gitlab/diff/position_tracer/base_strategy.rb b/lib/gitlab/diff/position_tracer/base_strategy.rb new file mode 100644 index 00000000000..65049daabf4 --- /dev/null +++ b/lib/gitlab/diff/position_tracer/base_strategy.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module Diff + class PositionTracer + class BaseStrategy + attr_reader :tracer + + delegate \ + :project, + :ac_diffs, + :bd_diffs, + :cd_diffs, + to: :tracer + + def initialize(tracer) + @tracer = tracer + end + + def trace(position) + raise NotImplementedError + end + end + end + end +end diff --git a/lib/gitlab/diff/position_tracer/image_strategy.rb b/lib/gitlab/diff/position_tracer/image_strategy.rb new file mode 100644 index 00000000000..79244a17951 --- /dev/null +++ b/lib/gitlab/diff/position_tracer/image_strategy.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + module Diff + class PositionTracer + class ImageStrategy < BaseStrategy + def trace(position) + b_path = position.new_path + + # If file exists in B->D (e.g. updated, renamed, removed), let the + # note become outdated. + bd_diff = bd_diffs.diff_file_with_old_path(b_path) + + return { position: new_position(position, bd_diff), outdated: true } if bd_diff + + # If file still exists in the new diff, update the position. + cd_diff = cd_diffs.diff_file_with_new_path(bd_diff&.new_path || b_path) + + return { position: new_position(position, cd_diff), outdated: false } if cd_diff + + # If file exists in A->C (e.g. rebased and same changes were present + # in target branch), let the note become outdated. + ac_diff = ac_diffs.diff_file_with_old_path(position.old_path) + + return { position: new_position(position, ac_diff), outdated: true } if ac_diff + + # If ever there's a case that the file no longer exists in any diff, + # don't set a change position and let the note become outdated. + # + # This should never happen given the file should exist in one of the + # diffs above. + { outdated: true } + end + + private + + def new_position(position, diff_file) + Position.new( + diff_file: diff_file, + x: position.x, + y: position.y, + width: position.width, + height: position.height, + position_type: position.position_type + ) + end + end + end + end +end diff --git a/lib/gitlab/diff/position_tracer/line_strategy.rb b/lib/gitlab/diff/position_tracer/line_strategy.rb new file mode 100644 index 00000000000..8db0fc6f963 --- /dev/null +++ b/lib/gitlab/diff/position_tracer/line_strategy.rb @@ -0,0 +1,201 @@ +# frozen_string_literal: true + +module Gitlab + module Diff + class PositionTracer + class LineStrategy < BaseStrategy + def trace(position) + # Suppose we have an MR with source branch `feature` and target branch `master`. + # When the MR was created, the head of `master` was commit A, and the + # head of `feature` was commit B, resulting in the original diff A->B. + # Since creation, `master` was updated to C. + # Now `feature` is being updated to D, and the newly generated MR diff is C->D. + # It is possible that C and D are direct descendants of A and B respectively, + # but this isn't necessarily the case as rebases and merges come into play. + # + # Suppose we have a diff note on the original diff A->B. Now that the MR + # is updated, we need to find out what line in C->D corresponds to the + # line the note was originally created on, so that we can update the diff note's + # records and continue to display it in the right place in the diffs. + # If we cannot find this line in the new diff, this means the diff note is now + # outdated, and we will display that fact to the user. + # + # In the new diff, the file the diff note was originally created on may + # have been renamed, deleted or even created, if the file existed in A and B, + # but was removed in C, and restored in D. + # + # Every diff note stores a Position object that defines a specific location, + # identified by paths and line numbers, within a specific diff, identified + # by start, head and base commit ids. + # + # For diff notes for diff A->B, the position looks like this: + # Position + # start_sha - ID of commit A + # head_sha - ID of commit B + # base_sha - ID of base commit of A and B + # old_path - path as of A (nil if file was newly created) + # new_path - path as of B (nil if file was deleted) + # old_line - line number as of A (nil if file was newly created) + # new_line - line number as of B (nil if file was deleted) + # + # We can easily update `start_sha` and `head_sha` to hold the IDs of + # commits C and D, and can trivially determine `base_sha` based on those, + # but need to find the paths and line numbers as of C and D. + # + # If the file was unchanged or newly created in A->B, the path as of D can be found + # by generating diff B->D ("head to head"), finding the diff file with + # `diff_file.old_path == position.new_path`, and taking `diff_file.new_path`. + # The path as of C can be found by taking diff C->D, finding the diff file + # with that same `new_path` and taking `diff_file.old_path`. + # The line number as of D can be found by using the LineMapper on diff B->D + # and providing the line number as of B. + # The line number as of C can be found by using the LineMapper on diff C->D + # and providing the line number as of D. + # + # If the file was deleted in A->B, the path as of C can be found + # by generating diff A->C ("base to base"), finding the diff file with + # `diff_file.old_path == position.old_path`, and taking `diff_file.new_path`. + # The path as of D can be found by taking diff C->D, finding the diff file + # with `old_path` set to that `diff_file.new_path` and taking `diff_file.new_path`. + # The line number as of C can be found by using the LineMapper on diff A->C + # and providing the line number as of A. + # The line number as of D can be found by using the LineMapper on diff C->D + # and providing the line number as of C. + + if position.added? + trace_added_line(position) + elsif position.removed? + trace_removed_line(position) + else # unchanged + trace_unchanged_line(position) + end + end + + private + + def trace_added_line(position) + b_path = position.new_path + b_line = position.new_line + + bd_diff = bd_diffs.diff_file_with_old_path(b_path) + + d_path = bd_diff&.new_path || b_path + d_line = LineMapper.new(bd_diff).old_to_new(b_line) + + if d_line + cd_diff = cd_diffs.diff_file_with_new_path(d_path) + + c_path = cd_diff&.old_path || d_path + c_line = LineMapper.new(cd_diff).new_to_old(d_line) + + if c_line + # If the line is still in D but also in C, it has turned from an + # added line into an unchanged one. + new_position = new_position(cd_diff, c_line, d_line) + if valid_position?(new_position) + # If the line is still in the MR, we don't treat this as outdated. + { position: new_position, outdated: false } + else + # If the line is no longer in the MR, we unfortunately cannot show + # the current state on the CD diff, so we treat it as outdated. + ac_diff = ac_diffs.diff_file_with_new_path(c_path) + + { position: new_position(ac_diff, nil, c_line), outdated: true } + end + else + # If the line is still in D and not in C, it is still added. + { position: new_position(cd_diff, nil, d_line), outdated: false } + end + else + # If the line is no longer in D, it has been removed from the MR. + { position: new_position(bd_diff, b_line, nil), outdated: true } + end + end + + def trace_removed_line(position) + a_path = position.old_path + a_line = position.old_line + + ac_diff = ac_diffs.diff_file_with_old_path(a_path) + + c_path = ac_diff&.new_path || a_path + c_line = LineMapper.new(ac_diff).old_to_new(a_line) + + if c_line + cd_diff = cd_diffs.diff_file_with_old_path(c_path) + + d_path = cd_diff&.new_path || c_path + d_line = LineMapper.new(cd_diff).old_to_new(c_line) + + if d_line + # If the line is still in C but also in D, it has turned from a + # removed line into an unchanged one. + bd_diff = bd_diffs.diff_file_with_new_path(d_path) + + { position: new_position(bd_diff, nil, d_line), outdated: true } + else + # If the line is still in C and not in D, it is still removed. + { position: new_position(cd_diff, c_line, nil), outdated: false } + end + else + # If the line is no longer in C, it has been removed outside of the MR. + { position: new_position(ac_diff, a_line, nil), outdated: true } + end + end + + def trace_unchanged_line(position) + a_path = position.old_path + a_line = position.old_line + b_path = position.new_path + b_line = position.new_line + + ac_diff = ac_diffs.diff_file_with_old_path(a_path) + + c_path = ac_diff&.new_path || a_path + c_line = LineMapper.new(ac_diff).old_to_new(a_line) + + bd_diff = bd_diffs.diff_file_with_old_path(b_path) + + d_line = LineMapper.new(bd_diff).old_to_new(b_line) + + cd_diff = cd_diffs.diff_file_with_old_path(c_path) + + if c_line && d_line + # If the line is still in C and D, it is still unchanged. + new_position = new_position(cd_diff, c_line, d_line) + if valid_position?(new_position) + # If the line is still in the MR, we don't treat this as outdated. + { position: new_position, outdated: false } + else + # If the line is no longer in the MR, we unfortunately cannot show + # the current state on the CD diff or any change on the BD diff, + # so we treat it as outdated. + { position: nil, outdated: true } + end + elsif d_line # && !c_line + # If the line is still in D but no longer in C, it has turned from + # an unchanged line into an added one. + # We don't treat this as outdated since the line is still in the MR. + { position: new_position(cd_diff, nil, d_line), outdated: false } + else # !d_line && (c_line || !c_line) + # If the line is no longer in D, it has turned from an unchanged line + # into a removed one. + { position: new_position(bd_diff, b_line, nil), outdated: true } + end + end + + def new_position(diff_file, old_line, new_line) + Position.new( + diff_file: diff_file, + old_line: old_line, + new_line: new_line + ) + end + + def valid_position?(position) + !!position.diff_line(project.repository) + end + end + end + end +end diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index 01fd261404b..86e532766b1 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -7,7 +7,7 @@ module Gitlab CANONICAL_CE_PROJECT_URL = 'https://gitlab.com/gitlab-org/gitlab-ce'.freeze CANONICAL_EE_REPO_URL = 'https://gitlab.com/gitlab-org/gitlab-ee.git'.freeze CHECK_DIR = Rails.root.join('ee_compat_check') - IGNORED_FILES_REGEX = /VERSION|CHANGELOG\.md/i.freeze + IGNORED_FILES_REGEX = /VERSION|CHANGELOG\.md|doc\/.+/i.freeze PLEASE_READ_THIS_BANNER = %Q{ ============================================================ ===================== PLEASE READ THIS ===================== diff --git a/lib/gitlab/email/hook/disable_email_interceptor.rb b/lib/gitlab/email/hook/disable_email_interceptor.rb index 6b6b1d85109..58dc1527c7a 100644 --- a/lib/gitlab/email/hook/disable_email_interceptor.rb +++ b/lib/gitlab/email/hook/disable_email_interceptor.rb @@ -7,7 +7,7 @@ module Gitlab def self.delivering_email(message) message.perform_deliveries = false - Rails.logger.info "Emails disabled! Interceptor prevented sending mail #{message.subject}" + Rails.logger.info "Emails disabled! Interceptor prevented sending mail #{message.subject}" # rubocop:disable Gitlab/RailsLogger end end end diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index 5a61a7f5d60..88729babb2b 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -59,7 +59,7 @@ module Gitlab begin CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8') rescue ArgumentError => e - Rails.logger.warn("Ignoring error converting #{detect[:encoding]} into UTF8: #{e.message}") + Rails.logger.warn("Ignoring error converting #{detect[:encoding]} into UTF8: #{e.message}") # rubocop:disable Gitlab/RailsLogger '' end diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb index 17fbecbd097..d09dcdbb337 100644 --- a/lib/gitlab/etag_caching/router.rb +++ b/lib/gitlab/etag_caching/router.rb @@ -61,6 +61,10 @@ module Gitlab Gitlab::EtagCaching::Router::Route.new( %r(#{RESERVED_WORDS_PREFIX}/import/gitea/realtime_changes\.json\z), 'realtime_changes_import_gitea' + ), + Gitlab::EtagCaching::Router::Route.new( + %r(#{RESERVED_WORDS_PREFIX}/merge_requests/\d+/cached_widget\.json\z), + 'merge_request_widget' ) ].freeze diff --git a/lib/gitlab/exclusive_lease_helpers.rb b/lib/gitlab/exclusive_lease_helpers.rb index 7961d4bbd6e..61eb030563d 100644 --- a/lib/gitlab/exclusive_lease_helpers.rb +++ b/lib/gitlab/exclusive_lease_helpers.rb @@ -15,17 +15,18 @@ module Gitlab raise ArgumentError, 'Key needs to be specified' unless key lease = Gitlab::ExclusiveLease.new(key, timeout: ttl) + retried = false until uuid = lease.try_obtain # Keep trying until we obtain the lease. To prevent hammering Redis too # much we'll wait for a bit. sleep(sleep_sec) - break if (retries -= 1) < 0 + (retries -= 1) < 0 ? break : retried ||= true end raise FailedToObtainLockError, 'Failed to obtain a lock' unless uuid - yield + yield(retried) ensure Gitlab::ExclusiveLease.cancel(key, uuid) end diff --git a/lib/gitlab/external_authorization/client.rb b/lib/gitlab/external_authorization/client.rb index 60aab2e7044..7985e6dcf7b 100644 --- a/lib/gitlab/external_authorization/client.rb +++ b/lib/gitlab/external_authorization/client.rb @@ -48,7 +48,8 @@ module Gitlab @body ||= begin body = { user_identifier: @user.email, - project_classification_label: @label + project_classification_label: @label, + identities: @user.identities.map { |identity| { provider: identity.provider, extern_uid: identity.extern_uid } } } if @user.ldap_identity diff --git a/lib/gitlab/favicon.rb b/lib/gitlab/favicon.rb index 6e31064f737..519213e143c 100644 --- a/lib/gitlab/favicon.rb +++ b/lib/gitlab/favicon.rb @@ -5,8 +5,8 @@ module Gitlab class << self def main image_name = - if appearance_favicon.exists? - appearance_favicon.url + if appearance.favicon.exists? + appearance.favicon_path elsif Gitlab::Utils.to_boolean(ENV['CANARY']) 'favicon-yellow.png' elsif Rails.env.development? @@ -57,10 +57,6 @@ module Gitlab def appearance Gitlab::SafeRequestStore[:appearance] ||= (Appearance.current || Appearance.new) end - - def appearance_favicon - appearance.favicon - end end end end diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb index 2d1c9ac40ae..6b52d6e88e5 100644 --- a/lib/gitlab/gfm/uploads_rewriter.rb +++ b/lib/gitlab/gfm/uploads_rewriter.rb @@ -27,7 +27,15 @@ module Gitlab klass = target_parent.is_a?(Namespace) ? NamespaceFileUploader : FileUploader moved = klass.copy_to(file, target_parent) - moved.markdown_link + + moved_markdown = moved.markdown_link + + # Prevents rewrite of plain links as embedded + if was_embedded?(markdown) + moved_markdown + else + moved_markdown.sub(/\A!/, "") + end end end @@ -43,6 +51,10 @@ module Gitlab referenced_files.compact.select(&:exists?) end + def was_embedded?(markdown) + markdown.starts_with?("!") + end + private def find_file(project, secret, file) diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 44a62586a23..df9f33baec2 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -9,6 +9,7 @@ module Gitlab # https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012 EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'.freeze BLANK_SHA = ('0' * 40).freeze + COMMIT_ID = /\A[0-9a-f]{40}\z/.freeze TAG_REF_PREFIX = "refs/tags/".freeze BRANCH_REF_PREFIX = "refs/heads/".freeze @@ -65,6 +66,10 @@ module Gitlab ref == BLANK_SHA end + def commit_id?(ref) + COMMIT_ID.match?(ref) + end + def version Gitlab::Git::Version.git_version end diff --git a/lib/gitlab/git/raw_diff_change.rb b/lib/gitlab/git/raw_diff_change.rb index e1002af40f6..9a41f04a4db 100644 --- a/lib/gitlab/git/raw_diff_change.rb +++ b/lib/gitlab/git/raw_diff_change.rb @@ -11,8 +11,8 @@ module Gitlab if raw_change.is_a?(Gitaly::GetRawChangesResponse::RawChange) @blob_id = raw_change.blob_id @blob_size = raw_change.size - @old_path = raw_change.old_path.presence - @new_path = raw_change.new_path.presence + @old_path = raw_change.old_path_bytes.presence + @new_path = raw_change.new_path_bytes.presence @operation = raw_change.operation&.downcase || :unknown else parse(raw_change) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 8a2e711ec4e..27032602828 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -15,11 +15,6 @@ module Gitlab SEARCH_CONTEXT_LINES = 3 REV_LIST_COMMIT_LIMIT = 2_000 - # In https://gitlab.com/gitlab-org/gitaly/merge_requests/698 - # We copied these two prefixes into gitaly-go, so don't change these - # or things will break! (REBASE_WORKTREE_PREFIX and SQUASH_WORKTREE_PREFIX) - REBASE_WORKTREE_PREFIX = 'rebase'.freeze - SQUASH_WORKTREE_PREFIX = 'squash'.freeze GITALY_INTERNAL_URL = 'ssh://gitaly/internal.git'.freeze GITLAB_PROJECTS_TIMEOUT = Gitlab.config.gitlab_shell.git_timeout EMPTY_REPOSITORY_CHECKSUM = '0000000000000000000000000000000000000000'.freeze @@ -60,6 +55,10 @@ module Gitlab @name = @relative_path.split("/").last end + def to_s + "<#{self.class.name}: #{self.gl_project_path}>" + end + def ==(other) other.is_a?(self.class) && [storage, relative_path] == [other.storage, other.relative_path] end @@ -469,6 +468,18 @@ module Gitlab end end + # Returns path to url mappings for submodules + # + # Ex. + # @repository.submodule_urls_for('master') + # # => { 'rack' => 'git@localhost:rack.git' } + # + def submodule_urls_for(ref) + wrapped_gitaly_errors do + gitaly_submodule_urls_for(ref) + end + end + # Return total commits count accessible from passed ref def commit_count(ref) wrapped_gitaly_errors do @@ -541,9 +552,9 @@ module Gitlab tags.find { |tag| tag.name == name } end - def merge_to_ref(user, source_sha, branch, target_ref, message) + def merge_to_ref(user, source_sha, branch, target_ref, message, first_parent_ref) wrapped_gitaly_errors do - gitaly_operation_client.user_merge_to_ref(user, source_sha, branch, target_ref, message) + gitaly_operation_client.user_merge_to_ref(user, source_sha, branch, target_ref, message, first_parent_ref) end end @@ -683,17 +694,16 @@ module Gitlab attributes(path)[name] end - # Check .gitattributes for a given ref + # Returns parsed .gitattributes for a given ref # - # This only checks the root .gitattributes file, + # This only parses the root .gitattributes file, # it does not traverse subfolders to find additional .gitattributes files # # This method is around 30 times slower than `attributes`, which uses # `$GIT_DIR/info/attributes`. Consider caching AttributesAtRefParser # and reusing that for multiple calls instead of this method. - def attributes_at(ref, file_path) - parser = AttributesAtRefParser.new(self, ref) - parser.attributes(file_path) + def attributes_at(ref) + AttributesAtRefParser.new(self, ref) end def languages(ref = nil) @@ -867,13 +877,13 @@ module Gitlab def multi_action( user, branch_name:, message:, actions:, author_email: nil, author_name: nil, - start_branch_name: nil, start_repository: self, + start_branch_name: nil, start_sha: nil, start_repository: self, force: false) wrapped_gitaly_errors do gitaly_operation_client.user_commit_files(user, branch_name, message, actions, author_email, author_name, - start_branch_name, start_repository, force) + start_branch_name, start_repository, force, start_sha) end end # rubocop:enable Metrics/ParameterLists @@ -942,7 +952,7 @@ module Gitlab gitaly_repository_client.cleanup if exists? end rescue Gitlab::Git::CommandError => e # Don't fail if we can't cleanup - Rails.logger.error("Unable to clean repository on storage #{storage} with relative path #{relative_path}: #{e.message}") + Rails.logger.error("Unable to clean repository on storage #{storage} with relative path #{relative_path}: #{e.message}") # rubocop:disable Gitlab/RailsLogger Gitlab::Metrics.counter( :failed_repository_cleanup_total, 'Number of failed repository cleanup events' @@ -1065,12 +1075,16 @@ module Gitlab return unless commit_object && commit_object.type == :COMMIT + urls = gitaly_submodule_urls_for(ref) + urls && urls[path] + end + + def gitaly_submodule_urls_for(ref) gitmodules = gitaly_commit_client.tree_entry(ref, '.gitmodules', Gitlab::Git::Blob::MAX_DATA_DISPLAY_SIZE) return unless gitmodules - found_module = GitmodulesParser.new(gitmodules.data).parse[path] - - found_module && found_module['url'] + submodules = GitmodulesParser.new(gitmodules.data).parse + submodules.transform_values { |submodule| submodule['url'] } end # Returns true if the given ref name exists diff --git a/lib/gitlab/git/rugged_impl/blob.rb b/lib/gitlab/git/rugged_impl/blob.rb index 11ee4ebda4b..5c73c0c66a9 100644 --- a/lib/gitlab/git/rugged_impl/blob.rb +++ b/lib/gitlab/git/rugged_impl/blob.rb @@ -11,11 +11,12 @@ module Gitlab module Blob module ClassMethods extend ::Gitlab::Utils::Override + include Gitlab::Git::RuggedImpl::UseRugged override :tree_entry def tree_entry(repository, sha, path, limit) - if Feature.enabled?(:rugged_tree_entry) - rugged_tree_entry(repository, sha, path, limit) + if use_rugged?(repository, :rugged_tree_entry) + execute_rugged_call(:rugged_tree_entry, repository, sha, path, limit) else super end diff --git a/lib/gitlab/git/rugged_impl/commit.rb b/lib/gitlab/git/rugged_impl/commit.rb index bce4fa14fb4..0eff35ab1c4 100644 --- a/lib/gitlab/git/rugged_impl/commit.rb +++ b/lib/gitlab/git/rugged_impl/commit.rb @@ -12,6 +12,7 @@ module Gitlab module Commit module ClassMethods extend ::Gitlab::Utils::Override + include Gitlab::Git::RuggedImpl::UseRugged def rugged_find(repo, commit_id) obj = repo.rev_parse_target(commit_id) @@ -34,8 +35,8 @@ module Gitlab override :find_commit def find_commit(repo, commit_id) - if Feature.enabled?(:rugged_find_commit) - rugged_find(repo, commit_id) + if use_rugged?(repo, :rugged_find_commit) + execute_rugged_call(:rugged_find, repo, commit_id) else super end @@ -43,8 +44,8 @@ module Gitlab override :batch_by_oid def batch_by_oid(repo, oids) - if Feature.enabled?(:rugged_list_commits_by_oid) - rugged_batch_by_oid(repo, oids) + if use_rugged?(repo, :rugged_list_commits_by_oid) + execute_rugged_call(:rugged_batch_by_oid, repo, oids) else super end @@ -52,6 +53,7 @@ module Gitlab end extend ::Gitlab::Utils::Override + include Gitlab::Git::RuggedImpl::UseRugged override :init_commit def init_commit(raw_commit) @@ -65,8 +67,8 @@ module Gitlab override :commit_tree_entry def commit_tree_entry(path) - if Feature.enabled?(:rugged_commit_tree_entry) - rugged_tree_entry(path) + if use_rugged?(@repository, :rugged_commit_tree_entry) + execute_rugged_call(:rugged_tree_entry, path) else super end diff --git a/lib/gitlab/git/rugged_impl/repository.rb b/lib/gitlab/git/rugged_impl/repository.rb index e91b0ddcd31..8fde93e71e2 100644 --- a/lib/gitlab/git/rugged_impl/repository.rb +++ b/lib/gitlab/git/rugged_impl/repository.rb @@ -11,6 +11,7 @@ module Gitlab module RuggedImpl module Repository extend ::Gitlab::Utils::Override + include Gitlab::Git::RuggedImpl::UseRugged FEATURE_FLAGS = %i(rugged_find_commit rugged_tree_entries rugged_tree_entry rugged_commit_is_ancestor rugged_commit_tree_entry rugged_list_commits_by_oid).freeze @@ -46,8 +47,8 @@ module Gitlab override :ancestor? def ancestor?(from, to) - if Feature.enabled?(:rugged_commit_is_ancestor) - rugged_is_ancestor?(from, to) + if use_rugged?(self, :rugged_commit_is_ancestor) + execute_rugged_call(:rugged_is_ancestor?, from, to) else super end diff --git a/lib/gitlab/git/rugged_impl/tree.rb b/lib/gitlab/git/rugged_impl/tree.rb index 9c37bb01961..389c9d32ccb 100644 --- a/lib/gitlab/git/rugged_impl/tree.rb +++ b/lib/gitlab/git/rugged_impl/tree.rb @@ -11,11 +11,12 @@ module Gitlab module Tree module ClassMethods extend ::Gitlab::Utils::Override + include Gitlab::Git::RuggedImpl::UseRugged override :tree_entries def tree_entries(repository, sha, path, recursive) - if Feature.enabled?(:rugged_tree_entries) - tree_entries_with_flat_path_from_rugged(repository, sha, path, recursive) + if use_rugged?(repository, :rugged_tree_entries) + execute_rugged_call(:tree_entries_with_flat_path_from_rugged, repository, sha, path, recursive) else super end diff --git a/lib/gitlab/git/rugged_impl/use_rugged.rb b/lib/gitlab/git/rugged_impl/use_rugged.rb new file mode 100644 index 00000000000..80b75689334 --- /dev/null +++ b/lib/gitlab/git/rugged_impl/use_rugged.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module Git + module RuggedImpl + module UseRugged + def use_rugged?(repo, feature_key) + feature = Feature.get(feature_key) + return feature.enabled? if Feature.persisted?(feature) + + Gitlab::GitalyClient.can_use_disk?(repo.storage) + end + + def execute_rugged_call(method_name, *args) + Gitlab::GitalyClient::StorageSettings.allow_disk_access do + start = Gitlab::Metrics::System.monotonic_time + + result = send(method_name, *args) # rubocop:disable GitlabSecurity/PublicSend + + duration = Gitlab::Metrics::System.monotonic_time - start + + if Gitlab::RuggedInstrumentation.active? + Gitlab::RuggedInstrumentation.increment_query_count + Gitlab::RuggedInstrumentation.query_time += duration + + Gitlab::RuggedInstrumentation.add_call_details( + feature: method_name, + args: args, + duration: duration, + backtrace: Gitlab::Profiler.clean_backtrace(caller)) + end + + result + end + end + end + end + end +end diff --git a/lib/gitlab/git_logger.rb b/lib/gitlab/git_logger.rb index dac4ddd320f..545451f0dc9 100644 --- a/lib/gitlab/git_logger.rb +++ b/lib/gitlab/git_logger.rb @@ -1,13 +1,9 @@ # frozen_string_literal: true module Gitlab - class GitLogger < Gitlab::Logger + class GitLogger < JsonLogger def self.file_name_noext - 'githost' - end - - def format_message(severity, timestamp, progname, msg) - "#{timestamp.to_s(:long)} -> #{severity} -> #{msg}\n" + 'git_json' end end end diff --git a/lib/gitlab/git_post_receive.rb b/lib/gitlab/git_post_receive.rb index d98b85fecc4..2a8bcd015a8 100644 --- a/lib/gitlab/git_post_receive.rb +++ b/lib/gitlab/git_post_receive.rb @@ -27,6 +27,29 @@ module Gitlab end end + def includes_branches? + enum_for(:changes_refs).any? do |_oldrev, _newrev, ref| + Gitlab::Git.branch_ref?(ref) + end + end + + def includes_tags? + enum_for(:changes_refs).any? do |_oldrev, _newrev, ref| + Gitlab::Git.tag_ref?(ref) + end + end + + def includes_default_branch? + # If the branch doesn't have a default branch yet, we presume the + # first branch pushed will be the default. + return true unless project.default_branch.present? + + enum_for(:changes_refs).any? do |_oldrev, _newrev, ref| + Gitlab::Git.branch_ref?(ref) && + Gitlab::Git.branch_name(ref) == project.default_branch + end + end + private def deserialize_changes(changes) diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index e683d4e5bbe..e6cbfb00f60 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -30,18 +30,10 @@ module Gitlab SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION' MAXIMUM_GITALY_CALLS = 30 CLIENT_NAME = (Sidekiq.server? ? 'gitlab-sidekiq' : 'gitlab-web').freeze - - SERVER_FEATURE_CATFILE_CACHE = 'catfile-cache'.freeze - # Server feature flags should use '_' to separate words. - SERVER_FEATURE_FLAGS = [SERVER_FEATURE_CATFILE_CACHE].freeze + GITALY_METADATA_FILENAME = '.gitaly-metadata' MUTEX = Mutex.new - define_histogram :gitaly_controller_action_duration_seconds do - docstring "Gitaly endpoint histogram by controller and action combination" - base_labels Gitlab::Metrics::Transaction::BASE_LABELS.merge(gitaly_service: nil, rpc: nil) - end - def self.stub(name, storage) MUTEX.synchronize do @stubs ||= {} @@ -75,7 +67,7 @@ module Gitlab File.read(cert_file).scan(PEM_REGEX).map do |cert| OpenSSL::X509::Certificate.new(cert).to_pem rescue OpenSSL::OpenSSLError => e - Rails.logger.error "Could not load certificate #{cert_file} #{e}" + Rails.logger.error "Could not load certificate #{cert_file} #{e}" # rubocop:disable Gitlab/RailsLogger Gitlab::Sentry.track_exception(e, extra: { cert_file: cert_file }) nil end.compact @@ -165,10 +157,6 @@ module Gitlab # Keep track, separately, for the performance bar self.query_time += duration - gitaly_controller_action_duration_seconds.observe( - current_transaction_labels.merge(gitaly_service: service.to_s, rpc: rpc.to_s), - duration) - if peek_enabled? add_call_details(feature: "#{service}##{rpc}", duration: duration, request: request_hash, rpc: rpc, backtrace: Gitlab::Profiler.clean_backtrace(caller)) @@ -223,9 +211,8 @@ module Gitlab metadata['call_site'] = feature.to_s if feature metadata['gitaly-servers'] = address_metadata(remote_storage) if remote_storage metadata['x-gitlab-correlation-id'] = Labkit::Correlation::CorrelationId.current_id if Labkit::Correlation::CorrelationId.current_id - metadata['gitaly-session-id'] = session_id if feature_enabled?(SERVER_FEATURE_CATFILE_CACHE) - - metadata.merge!(server_feature_flags) + metadata['gitaly-session-id'] = session_id + metadata.merge!(Feature::Gitaly.server_feature_flags) result = { metadata: metadata } @@ -244,12 +231,6 @@ module Gitlab Gitlab::SafeRequestStore[:gitaly_session_id] ||= SecureRandom.uuid end - def self.server_feature_flags - SERVER_FEATURE_FLAGS.map do |f| - ["gitaly-feature-#{f.tr('_', '-')}", feature_enabled?(f).to_s] - end.to_h - end - def self.token(storage) params = Gitlab.config.repositories.storages[storage] raise "storage not found: #{storage.inspect}" if params.nil? @@ -257,15 +238,9 @@ module Gitlab params['gitaly_token'].presence || Gitlab.config.gitaly['token'] end - def self.feature_enabled?(feature_name) - Feature::FlipperFeature.table_exists? && Feature.enabled?("gitaly_#{feature_name}") - rescue ActiveRecord::NoDatabaseError - false - end - # Ensures that Gitaly is not being abuse through n+1 misuse etc def self.enforce_gitaly_request_limits(call_site) - # Only count limits in request-response environments (not sidekiq for example) + # Only count limits in request-response environments return unless Gitlab::SafeRequestStore.active? # This is this actual number of times this call was made. Used for information purposes only @@ -295,7 +270,7 @@ module Gitlab # check if the limit is being exceeded while testing in those environments # In that case we can use a feature flag to indicate that we do want to # enforce request limits. - return true if feature_enabled?('enforce_requests_limits') + return true if Feature::Gitaly.enabled?('enforce_requests_limits') !(Rails.env.production? || ENV["GITALY_DISABLE_REQUEST_LIMITS"]) end @@ -403,6 +378,45 @@ module Gitlab 0 end + def self.storage_metadata_file_path(storage) + Gitlab::GitalyClient::StorageSettings.allow_disk_access do + File.join( + Gitlab.config.repositories.storages[storage].legacy_disk_path, GITALY_METADATA_FILENAME + ) + end + end + + def self.can_use_disk?(storage) + cached_value = MUTEX.synchronize do + @can_use_disk ||= {} + @can_use_disk[storage] + end + + return cached_value unless cached_value.nil? + + gitaly_filesystem_id = filesystem_id(storage) + direct_filesystem_id = filesystem_id_from_disk(storage) + + MUTEX.synchronize do + @can_use_disk[storage] = gitaly_filesystem_id.present? && + gitaly_filesystem_id == direct_filesystem_id + end + end + + def self.filesystem_id(storage) + response = Gitlab::GitalyClient::ServerService.new(storage).info + storage_status = response.storage_statuses.find { |status| status.storage_name == storage } + storage_status.filesystem_id + end + + def self.filesystem_id_from_disk(storage) + metadata_file = File.read(storage_metadata_file_path(storage)) + metadata_hash = JSON.parse(metadata_file) + metadata_hash['gitaly_filesystem_id'] + rescue Errno::ENOENT, Errno::EACCES, JSON::ParserError + nil + end + def self.timeout(timeout_name) Gitlab::CurrentSettings.current_application_settings[timeout_name] end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index d21b98d36ea..a80ce462ab0 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -271,26 +271,30 @@ module Gitlab end def find_commit(revision) - if Gitlab::SafeRequestStore.active? - # We don't use Gitlab::SafeRequestStore.fetch(key) { ... } directly - # because `revision` can be a branch name, so we can't use it as a key - # as it could point to another commit later on (happens a lot in - # tests). - key = { - storage: @gitaly_repo.storage_name, - relative_path: @gitaly_repo.relative_path, - commit_id: revision - } - return Gitlab::SafeRequestStore[key] if Gitlab::SafeRequestStore.exist?(key) - - commit = call_find_commit(revision) - return unless commit - - key[:commit_id] = commit.id unless GitalyClient.ref_name_caching_allowed? + return call_find_commit(revision) unless Gitlab::SafeRequestStore.active? + + # We don't use Gitlab::SafeRequestStore.fetch(key) { ... } directly + # because `revision` can be a branch name, so we can't use it as a key + # as it could point to another commit later on (happens a lot in + # tests). + key = { + storage: @gitaly_repo.storage_name, + relative_path: @gitaly_repo.relative_path, + commit_id: revision + } + return Gitlab::SafeRequestStore[key] if Gitlab::SafeRequestStore.exist?(key) + + commit = call_find_commit(revision) + + if GitalyClient.ref_name_caching_allowed? Gitlab::SafeRequestStore[key] = commit - else - call_find_commit(revision) + return commit end + + return unless commit + + key[:commit_id] = commit.id + Gitlab::SafeRequestStore[key] = commit end # rubocop: disable CodeReuse/ActiveRecord diff --git a/lib/gitlab/gitaly_client/notification_service.rb b/lib/gitlab/gitaly_client/notification_service.rb deleted file mode 100644 index 873c3e4086d..00000000000 --- a/lib/gitlab/gitaly_client/notification_service.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module GitalyClient - class NotificationService - # 'repository' is a Gitlab::Git::Repository - def initialize(repository) - @gitaly_repo = repository.gitaly_repository - @storage = repository.storage - end - - def post_receive - GitalyClient.call( - @storage, - :notification_service, - :post_receive, - Gitaly::PostReceiveRequest.new(repository: @gitaly_repo) - ) - end - end - end -end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index b42e6cbad8d..33ca428a942 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -100,14 +100,15 @@ module Gitlab end end - def user_merge_to_ref(user, source_sha, branch, target_ref, message) + def user_merge_to_ref(user, source_sha, branch, target_ref, message, first_parent_ref) request = Gitaly::UserMergeToRefRequest.new( repository: @gitaly_repo, source_sha: source_sha, branch: encode_binary(branch), target_ref: encode_binary(target_ref), user: Gitlab::Git::User.from_gitlab(user).to_gitaly, - message: encode_binary(message) + message: encode_binary(message), + first_parent_ref: encode_binary(first_parent_ref) ) response = GitalyClient.call(@repository.storage, :operation_service, :user_merge_to_ref, request) @@ -324,11 +325,11 @@ module Gitlab # rubocop:disable Metrics/ParameterLists def user_commit_files( user, branch_name, commit_message, actions, author_email, author_name, - start_branch_name, start_repository, force = false) + start_branch_name, start_repository, force = false, start_sha = nil) req_enum = Enumerator.new do |y| header = user_commit_files_request_header(user, branch_name, commit_message, actions, author_email, author_name, - start_branch_name, start_repository, force) + start_branch_name, start_repository, force, start_sha) y.yield Gitaly::UserCommitFilesRequest.new(header: header) @@ -444,7 +445,7 @@ module Gitlab # rubocop:disable Metrics/ParameterLists def user_commit_files_request_header( user, branch_name, commit_message, actions, author_email, author_name, - start_branch_name, start_repository, force) + start_branch_name, start_repository, force, start_sha) Gitaly::UserCommitFilesRequestHeader.new( repository: @gitaly_repo, @@ -455,7 +456,8 @@ module Gitlab commit_author_email: encode_binary(author_email), start_branch_name: encode_binary(start_branch_name), start_repository: start_repository.gitaly_repository, - force: force + force: force, + start_sha: encode_binary(start_sha) ) end # rubocop:enable Metrics/ParameterLists diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index d8e9dccb644..ca3e5b51ecc 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -5,7 +5,7 @@ module Gitlab class RepositoryService include Gitlab::EncodingHelper - MAX_MSG_SIZE = 128.kilobytes.freeze + MAX_MSG_SIZE = 128.kilobytes def initialize(repository) @repository = repository diff --git a/lib/gitlab/gitaly_client/storage_settings.rb b/lib/gitlab/gitaly_client/storage_settings.rb index 78ef6bfc0ec..7d1206e551b 100644 --- a/lib/gitlab/gitaly_client/storage_settings.rb +++ b/lib/gitlab/gitaly_client/storage_settings.rb @@ -34,7 +34,7 @@ module Gitlab def self.disk_access_denied? return false if rugged_enabled? - !temporarily_allowed?(ALLOW_KEY) && GitalyClient.feature_enabled?(DISK_ACCESS_DENIED_FLAG) + !temporarily_allowed?(ALLOW_KEY) && Feature::Gitaly.enabled?(DISK_ACCESS_DENIED_FLAG) rescue false # Err on the side of caution, don't break gitlab for people end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index a61beafae0d..826b35d685c 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -40,7 +40,7 @@ module Gitlab # otherwise hitting the rate limit will result in a thread # being blocked in a `sleep()` call for up to an hour. def initialize(token, per_page: 100, parallel: true) - @octokit = Octokit::Client.new( + @octokit = ::Octokit::Client.new( access_token: token, per_page: per_page, api_endpoint: api_endpoint @@ -139,7 +139,7 @@ module Gitlab begin yield - rescue Octokit::TooManyRequests + rescue ::Octokit::TooManyRequests raise_or_wait_for_rate_limit # This retry will only happen when running in sequential mode as we'll diff --git a/lib/gitlab/github_import/importer/lfs_objects_importer.rb b/lib/gitlab/github_import/importer/lfs_objects_importer.rb index 6046e30d4ef..30763492235 100644 --- a/lib/gitlab/github_import/importer/lfs_objects_importer.rb +++ b/lib/gitlab/github_import/importer/lfs_objects_importer.rb @@ -29,7 +29,7 @@ module Gitlab yield object end rescue StandardError => e - Rails.logger.error("The Lfs import process failed. #{e.message}") + Rails.logger.error("The Lfs import process failed. #{e.message}") # rubocop:disable Gitlab/RailsLogger end end end diff --git a/lib/gitlab/github_import/importer/pull_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests_importer.rb index a52866c4b08..929fceaacf2 100644 --- a/lib/gitlab/github_import/importer/pull_requests_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_importer.rb @@ -40,7 +40,7 @@ module Gitlab pname = project.path_with_namespace - Rails.logger + Rails.logger # rubocop:disable Gitlab/RailsLogger .info("GitHub importer finished updating repository for #{pname}") repository_updates_counter.increment diff --git a/lib/gitlab/github_import/importer/releases_importer.rb b/lib/gitlab/github_import/importer/releases_importer.rb index 0e7c9ee0d00..9d925581441 100644 --- a/lib/gitlab/github_import/importer/releases_importer.rb +++ b/lib/gitlab/github_import/importer/releases_importer.rb @@ -36,6 +36,7 @@ module Gitlab description: description_for(release), created_at: release.created_at, updated_at: release.updated_at, + released_at: release.published_at, project_id: project.id } end diff --git a/lib/gitlab/global_id.rb b/lib/gitlab/global_id.rb new file mode 100644 index 00000000000..cc82b6c5897 --- /dev/null +++ b/lib/gitlab/global_id.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Gitlab + module GlobalId + def self.build(object = nil, model_name: nil, id: nil, params: nil) + if object + model_name ||= object.class.name + id ||= object.id + end + + ::URI::GID.build(app: GlobalID.app, model_name: model_name, model_id: id, params: params) + end + end +end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 582c3065189..92917028851 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -16,8 +16,8 @@ module Gitlab gon.shortcuts_path = Gitlab::Routing.url_helpers.help_page_path('shortcuts') gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class - if Gitlab::CurrentSettings.clientside_sentry_enabled - gon.sentry_dsn = Gitlab::CurrentSettings.clientside_sentry_dsn + if Gitlab.config.sentry.enabled + gon.sentry_dsn = Gitlab.config.sentry.clientside_dsn gon.sentry_environment = Gitlab.config.sentry.environment end diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb index 5ff415b6126..1d317c389d2 100644 --- a/lib/gitlab/gpg/commit.rb +++ b/lib/gitlab/gpg/commit.rb @@ -52,12 +52,13 @@ module Gitlab def using_keychain Gitlab::Gpg.using_tmp_keychain do - # first we need to get the keyid from the signature to query the gpg - # key belonging to the keyid. + # first we need to get the fingerprint from the signature to query the gpg + # key belonging to the fingerprint. # This way we can add the key to the temporary keychain and extract # the proper signature. - # NOTE: the invoked method is #fingerprint but it's only returning - # 16 characters (the format used by keyid) instead of 40. + # NOTE: the invoked method is #fingerprint but versions of GnuPG + # prior to 2.2.13 return 16 characters (the format used by keyid) + # instead of 40. fingerprint = verified_signature&.fingerprint break unless fingerprint @@ -128,11 +129,13 @@ module Gitlab gpg_key&.verified_user_infos&.first || gpg_key&.user_infos&.first || {} end - # rubocop: disable CodeReuse/ActiveRecord - def find_gpg_key(keyid) - GpgKey.find_by(primary_keyid: keyid) || GpgKeySubkey.find_by(keyid: keyid) + def find_gpg_key(fingerprint) + if fingerprint.length > 16 + GpgKey.find_by_fingerprint(fingerprint) || GpgKeySubkey.find_by_fingerprint(fingerprint) + else + GpgKey.find_by_primary_keyid(fingerprint) || GpgKeySubkey.find_by_keyid(fingerprint) + end end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/grape_logging/loggers/client_env_logger.rb b/lib/gitlab/grape_logging/loggers/client_env_logger.rb new file mode 100644 index 00000000000..3acc6f6a2ef --- /dev/null +++ b/lib/gitlab/grape_logging/loggers/client_env_logger.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +# This is a fork of +# https://github.com/aserafin/grape_logging/blob/master/lib/grape_logging/loggers/client_env.rb +# to use remote_ip instead of ip. +module Gitlab + module GrapeLogging + module Loggers + class ClientEnvLogger < ::GrapeLogging::Loggers::Base + def parameters(request, _) + { remote_ip: request.env["HTTP_X_FORWARDED_FOR"] || request.env["REMOTE_ADDR"], ua: request.env["HTTP_USER_AGENT"] } + end + end + end + end +end diff --git a/lib/gitlab/grape_logging/loggers/perf_logger.rb b/lib/gitlab/grape_logging/loggers/perf_logger.rb index 18ea3a8d2f3..7e86b35a215 100644 --- a/lib/gitlab/grape_logging/loggers/perf_logger.rb +++ b/lib/gitlab/grape_logging/loggers/perf_logger.rb @@ -6,11 +6,30 @@ module Gitlab module Loggers class PerfLogger < ::GrapeLogging::Loggers::Base def parameters(_, _) + gitaly_data.merge(rugged_data) + end + + def gitaly_data + gitaly_calls = Gitlab::GitalyClient.get_request_count + + return {} if gitaly_calls.zero? + { gitaly_calls: Gitlab::GitalyClient.get_request_count, gitaly_duration: Gitlab::GitalyClient.query_time_ms } end + + def rugged_data + rugged_calls = Gitlab::RuggedInstrumentation.query_count + + return {} if rugged_calls.zero? + + { + rugged_calls: rugged_calls, + rugged_duration_ms: Gitlab::RuggedInstrumentation.query_time_ms + } + end end end end diff --git a/lib/gitlab/graphql.rb b/lib/gitlab/graphql.rb index 8a59e83974f..74c04e5380e 100644 --- a/lib/gitlab/graphql.rb +++ b/lib/gitlab/graphql.rb @@ -3,9 +3,5 @@ module Gitlab module Graphql StandardGraphqlError = Class.new(StandardError) - - def self.enabled? - Feature.enabled?(:graphql, default_enabled: true) - end end end diff --git a/lib/gitlab/graphql/authorize.rb b/lib/gitlab/graphql/authorize.rb index f8d0208e275..e83b567308b 100644 --- a/lib/gitlab/graphql/authorize.rb +++ b/lib/gitlab/graphql/authorize.rb @@ -8,7 +8,7 @@ module Gitlab extend ActiveSupport::Concern def self.use(schema_definition) - schema_definition.instrument(:field, Instrumentation.new, after_built_ins: true) + schema_definition.instrument(:field, Gitlab::Graphql::Authorize::Instrumentation.new, after_built_ins: true) end end end diff --git a/lib/gitlab/graphql/authorize/authorize_field_service.rb b/lib/gitlab/graphql/authorize/authorize_field_service.rb index 619ce100421..3b5dde2fde5 100644 --- a/lib/gitlab/graphql/authorize/authorize_field_service.rb +++ b/lib/gitlab/graphql/authorize/authorize_field_service.rb @@ -39,6 +39,8 @@ module Gitlab type = node_type_for_basic_connection(type) end + type = type.unwrap if type.kind.non_null? + Array.wrap(type.metadata[:authorize]) end diff --git a/lib/gitlab/graphql/authorize/authorize_resource.rb b/lib/gitlab/graphql/authorize/authorize_resource.rb index b367a97105c..ef5caaf5b0e 100644 --- a/lib/gitlab/graphql/authorize/authorize_resource.rb +++ b/lib/gitlab/graphql/authorize/authorize_resource.rb @@ -27,12 +27,6 @@ module Gitlab raise NotImplementedError, "Implement #find_object in #{self.class.name}" end - def authorized_find(*args) - object = find_object(*args) - - object if authorized?(object) - end - def authorized_find!(*args) object = find_object(*args) authorize!(object) @@ -48,6 +42,12 @@ module Gitlab end def authorized?(object) + # Sanity check. We don't want to accidentally allow a developer to authorize + # without first adding permissions to authorize against + if self.class.required_permissions.empty? + raise Gitlab::Graphql::Errors::ArgumentError, "#{self.class.name} has no authorizations" + end + self.class.required_permissions.all? do |ability| # The actions could be performed across multiple objects. In which # case the current user is common, and we could benefit from the diff --git a/lib/gitlab/graphql/calls_gitaly.rb b/lib/gitlab/graphql/calls_gitaly.rb new file mode 100644 index 00000000000..40cd74a34f2 --- /dev/null +++ b/lib/gitlab/graphql/calls_gitaly.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + # Wraps the field resolution to count Gitaly calls before and after. + # Raises an error if the field calls Gitaly but hadn't declared such. + module CallsGitaly + extend ActiveSupport::Concern + + def self.use(schema_definition) + schema_definition.instrument(:field, Gitlab::Graphql::CallsGitaly::Instrumentation.new, after_built_ins: true) + end + end + end +end diff --git a/lib/gitlab/graphql/calls_gitaly/instrumentation.rb b/lib/gitlab/graphql/calls_gitaly/instrumentation.rb new file mode 100644 index 00000000000..fbd5e348c7d --- /dev/null +++ b/lib/gitlab/graphql/calls_gitaly/instrumentation.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module CallsGitaly + class Instrumentation + # Check if any `calls_gitaly: true` declarations need to be added + # Do nothing if a constant complexity was provided + def instrument(_type, field) + type_object = field.metadata[:type_class] + return field unless type_object.respond_to?(:calls_gitaly?) + return field if type_object.constant_complexity? || type_object.calls_gitaly? + + old_resolver_proc = field.resolve_proc + + gitaly_wrapped_resolve = -> (typed_object, args, ctx) do + previous_gitaly_call_count = Gitlab::GitalyClient.get_request_count + result = old_resolver_proc.call(typed_object, args, ctx) + current_gitaly_call_count = Gitlab::GitalyClient.get_request_count + calls_gitaly_check(type_object, current_gitaly_call_count - previous_gitaly_call_count) + result + end + + field.redefine do + resolve(gitaly_wrapped_resolve) + end + end + + def calls_gitaly_check(type_object, calls) + return if calls < 1 + + # Will inform you if there needs to be `calls_gitaly: true` as a kwarg in the field declaration + # if there is at least 1 Gitaly call involved with the field resolution. + error = RuntimeError.new("Gitaly is called for field '#{type_object.name}' on #{type_object.owner.try(:name)} - please either specify a constant complexity or add `calls_gitaly: true` to the field declaration") + Gitlab::Sentry.track_exception(error) + end + end + end + end +end diff --git a/lib/gitlab/graphql/copy_field_description.rb b/lib/gitlab/graphql/copy_field_description.rb new file mode 100644 index 00000000000..edd73083ff2 --- /dev/null +++ b/lib/gitlab/graphql/copy_field_description.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module CopyFieldDescription + extend ActiveSupport::Concern + + class_methods do + # Returns the `description` for property of field `field_name` on type. + # This can be used to ensure, for example, that mutation argument descriptions + # are always identical to the corresponding query field descriptions. + # + # E.g.: + # argument :name, GraphQL::STRING_TYPE, description: copy_field_description(Types::UserType, :name) + def copy_field_description(type, field_name) + type.fields[field_name.to_s.camelize(:lower)].description + end + end + end + end +end diff --git a/lib/gitlab/graphql/docs/helper.rb b/lib/gitlab/graphql/docs/helper.rb new file mode 100644 index 00000000000..ac2a78c0f28 --- /dev/null +++ b/lib/gitlab/graphql/docs/helper.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +return if Rails.env.production? + +module Gitlab + module Graphql + module Docs + # Helper with functions to be used by HAML templates + # This includes graphql-docs gem helpers class. + # You can check the included module on: https://github.com/gjtorikian/graphql-docs/blob/v1.6.0/lib/graphql-docs/helpers.rb + module Helper + include GraphQLDocs::Helpers + + def auto_generated_comment + <<-MD.strip_heredoc + <!--- + This documentation is auto generated by a script. + + Please do not edit this file directly, check compile_docs task on lib/tasks/gitlab/graphql.rake. + ---> + MD + end + + # Some fields types are arrays of other types and are displayed + # on docs wrapped in square brackets, for example: [String!]. + # This makes GitLab docs renderer thinks they are links so here + # we change them to be rendered as: String! => Array. + def render_field_type(type) + array_type = type[/\[(.+)\]/, 1] + + if array_type + "#{array_type} => Array" + else + type + end + end + + # We are ignoring connections and built in types for now, + # they should be added when queries are generated. + def objects + graphql_object_types.select do |object_type| + !object_type[:name]["Connection"] && + !object_type[:name]["Edge"] && + !object_type[:name]["__"] + end + end + end + end + end +end diff --git a/lib/gitlab/graphql/docs/renderer.rb b/lib/gitlab/graphql/docs/renderer.rb new file mode 100644 index 00000000000..f47a372aa19 --- /dev/null +++ b/lib/gitlab/graphql/docs/renderer.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +return if Rails.env.production? + +module Gitlab + module Graphql + module Docs + # Gitlab renderer for graphql-docs. + # Uses HAML templates to parse markdown and generate .md files. + # It uses graphql-docs helpers and schema parser, more information in https://github.com/gjtorikian/graphql-docs. + # + # Arguments: + # schema - the GraphQL schema defition. For GitLab should be: GitlabSchema.graphql_definition + # output_dir: The folder where the markdown files will be saved + # template: The path of the haml template to be parsed + class Renderer + include Gitlab::Graphql::Docs::Helper + + def initialize(schema, output_dir:, template:) + @output_dir = output_dir + @template = template + @layout = Haml::Engine.new(File.read(template)) + @parsed_schema = GraphQLDocs::Parser.new(schema, {}).parse + end + + def render + contents = @layout.render(self) + + write_file(contents) + end + + private + + def write_file(contents) + filename = File.join(@output_dir, 'index.md') + + FileUtils.mkdir_p(@output_dir) + File.write(filename, contents) + end + end + end + end +end diff --git a/lib/gitlab/graphql/docs/templates/default.md.haml b/lib/gitlab/graphql/docs/templates/default.md.haml new file mode 100644 index 00000000000..cc22d43ab4f --- /dev/null +++ b/lib/gitlab/graphql/docs/templates/default.md.haml @@ -0,0 +1,25 @@ +-# haml-lint:disable UnnecessaryStringOutput + += auto_generated_comment + +:plain + # GraphQL API Resources + + This documentation is self-generated based on GitLab current GraphQL schema. + + The API can be explored interactively using the [GraphiQL IDE](../index.md#graphiql). + + ## Objects +\ +- objects.each do |type| + - unless type[:fields].empty? + = "### #{type[:name]}" + \ + ~ "| Name | Type | Description |" + ~ "| --- | ---- | ---------- |" + - type[:fields].each do |field| + = "| `#{field[:name]}` | #{render_field_type(field[:type][:info])} | #{field[:description]} |" + \ + + + diff --git a/lib/gitlab/graphql/errors.rb b/lib/gitlab/graphql/errors.rb index fe74549e322..40b90310e8b 100644 --- a/lib/gitlab/graphql/errors.rb +++ b/lib/gitlab/graphql/errors.rb @@ -6,6 +6,7 @@ module Gitlab BaseError = Class.new(GraphQL::ExecutionError) ArgumentError = Class.new(BaseError) ResourceNotAvailable = Class.new(BaseError) + MutationError = Class.new(BaseError) end end end diff --git a/lib/gitlab/graphql/find_argument_in_parent.rb b/lib/gitlab/graphql/find_argument_in_parent.rb new file mode 100644 index 00000000000..1f83f8fce7a --- /dev/null +++ b/lib/gitlab/graphql/find_argument_in_parent.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module FindArgumentInParent + # Searches up the GraphQL AST and returns the first matching argument + # passed to a node + def self.find(parent, argument, limit_depth: nil) + argument = argument.to_s.camelize(:lower).to_sym + depth = 0 + + while parent.respond_to?(:parent) + args = node_args(parent) + return args[argument] if args.key?(argument) + + depth += 1 + return if limit_depth && depth >= limit_depth + + parent = parent.parent + end + end + + class << self + private + + def node_args(node) + node.irep_node.arguments + end + end + end + end +end diff --git a/lib/gitlab/graphql/loaders/pipeline_for_sha_loader.rb b/lib/gitlab/graphql/loaders/pipeline_for_sha_loader.rb new file mode 100644 index 00000000000..81c5cabf451 --- /dev/null +++ b/lib/gitlab/graphql/loaders/pipeline_for_sha_loader.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Loaders + class PipelineForShaLoader + attr_accessor :project, :sha + + def initialize(project, sha) + @project, @sha = project, sha + end + + def find_last + BatchLoader.for(sha).batch(key: project) do |shas, loader, args| + pipelines = args[:key].ci_pipelines.latest_for_shas(shas) + + pipelines.each do |pipeline| + loader.call(pipeline.sha, pipeline) + end + end + end + end + end + end +end diff --git a/lib/gitlab/graphql/markdown_field.rb b/lib/gitlab/graphql/markdown_field.rb new file mode 100644 index 00000000000..7be6810f7ba --- /dev/null +++ b/lib/gitlab/graphql/markdown_field.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module MarkdownField + extend ActiveSupport::Concern + + prepended do + def self.markdown_field(name, **kwargs) + if kwargs[:resolver].present? || kwargs[:resolve].present? + raise ArgumentError, 'Only `method` is allowed to specify the markdown field' + end + + method_name = kwargs.delete(:method) || name.to_s.sub(/_html$/, '') + kwargs[:resolve] = Gitlab::Graphql::MarkdownField::Resolver.new(method_name.to_sym).proc + + kwargs[:description] ||= "The GitLab Flavored Markdown rendering of `#{method_name}`" + # Adding complexity to rendered notes since that could cause queries. + kwargs[:complexity] ||= 5 + + field name, GraphQL::STRING_TYPE, **kwargs + end + end + end + end +end diff --git a/lib/gitlab/graphql/markdown_field/resolver.rb b/lib/gitlab/graphql/markdown_field/resolver.rb new file mode 100644 index 00000000000..11a01b95ad1 --- /dev/null +++ b/lib/gitlab/graphql/markdown_field/resolver.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module MarkdownField + class Resolver + attr_reader :method_name + + def initialize(method_name) + @method_name = method_name + end + + def proc + -> (object, _args, ctx) do + # We need to `dup` the context so the MarkdownHelper doesn't modify it + ::MarkupHelper.markdown_field(object, method_name, ctx.to_h.dup) + end + end + end + end + end +end diff --git a/lib/gitlab/graphql/mount_mutation.rb b/lib/gitlab/graphql/mount_mutation.rb index 9048967d4e1..b10e963170a 100644 --- a/lib/gitlab/graphql/mount_mutation.rb +++ b/lib/gitlab/graphql/mount_mutation.rb @@ -6,11 +6,12 @@ module Gitlab extend ActiveSupport::Concern class_methods do - def mount_mutation(mutation_class) + def mount_mutation(mutation_class, **custom_kwargs) # Using an underscored field name symbol will make `graphql-ruby` # standardize the field name field mutation_class.graphql_name.underscore.to_sym, - mutation: mutation_class + mutation: mutation_class, + **custom_kwargs end end end diff --git a/lib/gitlab/graphql/representation/submodule_tree_entry.rb b/lib/gitlab/graphql/representation/submodule_tree_entry.rb new file mode 100644 index 00000000000..65716dff75d --- /dev/null +++ b/lib/gitlab/graphql/representation/submodule_tree_entry.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Representation + class SubmoduleTreeEntry < SimpleDelegator + class << self + def decorate(submodules, tree) + repository = tree.repository + submodule_links = Gitlab::SubmoduleLinks.new(repository) + + submodules.map do |submodule| + self.new(submodule, submodule_links.for(submodule, tree.sha)) + end + end + end + + def initialize(submodule, submodule_links) + @submodule_links = submodule_links + + super(submodule) + end + + def web_url + @submodule_links.first + end + + def tree_url + @submodule_links.last + end + end + end + end +end diff --git a/lib/gitlab/hashed_storage/migrator.rb b/lib/gitlab/hashed_storage/migrator.rb index 1f0deebea39..6a8e16f5a85 100644 --- a/lib/gitlab/hashed_storage/migrator.rb +++ b/lib/gitlab/hashed_storage/migrator.rb @@ -62,6 +62,7 @@ module Gitlab # Flag a project to be migrated to Hashed Storage # # @param [Project] project that will be migrated + # rubocop:disable Gitlab/RailsLogger def migrate(project) Rails.logger.info "Starting storage migration of #{project.full_path} (ID=#{project.id})..." @@ -69,10 +70,12 @@ module Gitlab rescue => err Rails.logger.error("#{err.message} migrating storage of #{project.full_path} (ID=#{project.id}), trace - #{err.backtrace}") end + # rubocop:enable Gitlab/RailsLogger # Flag a project to be rolled-back to Legacy Storage # # @param [Project] project that will be rolled-back + # rubocop:disable Gitlab/RailsLogger def rollback(project) Rails.logger.info "Starting storage rollback of #{project.full_path} (ID=#{project.id})..." @@ -80,6 +83,7 @@ module Gitlab rescue => err Rails.logger.error("#{err.message} rolling-back storage of #{project.full_path} (ID=#{project.id}), trace - #{err.backtrace}") end + # rubocop:enable Gitlab/RailsLogger # Returns whether we have any pending storage migration # diff --git a/lib/gitlab/hashed_storage/rake_helper.rb b/lib/gitlab/hashed_storage/rake_helper.rb index 87a31a37e3f..14727b03ce9 100644 --- a/lib/gitlab/hashed_storage/rake_helper.rb +++ b/lib/gitlab/hashed_storage/rake_helper.rb @@ -19,8 +19,12 @@ module Gitlab ENV['ID_TO'] end + def self.using_ranges? + !range_from.nil? && !range_to.nil? + end + def self.range_single_item? - !range_from.nil? && range_from == range_to + using_ranges? && range_from == range_to end # rubocop: disable CodeReuse/ActiveRecord diff --git a/lib/gitlab/health_checks/db_check.rb b/lib/gitlab/health_checks/db_check.rb index 2bcd25cd3cc..ec4b97eaca4 100644 --- a/lib/gitlab/health_checks/db_check.rb +++ b/lib/gitlab/health_checks/db_check.rb @@ -18,11 +18,7 @@ module Gitlab def check catch_timeout 10.seconds do - if Gitlab::Database.postgresql? - ActiveRecord::Base.connection.execute('SELECT 1 as ping')&.first&.[]('ping')&.to_s - else - ActiveRecord::Base.connection.execute('SELECT 1 as ping')&.first&.first&.to_s - end + ActiveRecord::Base.connection.execute('SELECT 1 as ping')&.first&.[]('ping')&.to_s end end end diff --git a/lib/gitlab/health_checks/simple_abstract_check.rb b/lib/gitlab/health_checks/simple_abstract_check.rb index 3588260d6eb..5a1e8c2a1dd 100644 --- a/lib/gitlab/health_checks/simple_abstract_check.rb +++ b/lib/gitlab/health_checks/simple_abstract_check.rb @@ -18,7 +18,7 @@ module Gitlab def metrics result, elapsed = with_timing(&method(:check)) - Rails.logger.error("#{human_name} check returned unexpected result #{result}") unless successful?(result) + Rails.logger.error("#{human_name} check returned unexpected result #{result}") unless successful?(result) # rubocop:disable Gitlab/RailsLogger [ metric("#{metric_prefix}_timeout", result.is_a?(Timeout::Error) ? 1 : 0), metric("#{metric_prefix}_success", successful?(result) ? 1 : 0), diff --git a/lib/gitlab/hook_data/issue_builder.rb b/lib/gitlab/hook_data/issue_builder.rb index cfc9ebe4f92..e5f86ca02b5 100644 --- a/lib/gitlab/hook_data/issue_builder.rb +++ b/lib/gitlab/hook_data/issue_builder.rb @@ -45,7 +45,7 @@ module Gitlab human_time_estimate: issue.human_time_estimate, assignee_ids: issue.assignee_ids, assignee_id: issue.assignee_ids.first, # This key is deprecated - labels: issue.labels + labels: issue.labels_hook_attrs } issue.attributes.with_indifferent_access.slice(*self.class.safe_hook_attributes) diff --git a/lib/gitlab/http.rb b/lib/gitlab/http.rb index db2b4dde244..58bce613a98 100644 --- a/lib/gitlab/http.rb +++ b/lib/gitlab/http.rb @@ -10,9 +10,9 @@ module Gitlab RedirectionTooDeep = Class.new(StandardError) HTTP_ERRORS = [ - SocketError, OpenSSL::SSL::SSLError, Errno::ECONNRESET, - Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Net::OpenTimeout, - Net::ReadTimeout, Gitlab::HTTP::BlockedUrlError, + SocketError, OpenSSL::SSL::SSLError, OpenSSL::OpenSSLError, + Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, + Net::OpenTimeout, Net::ReadTimeout, Gitlab::HTTP::BlockedUrlError, Gitlab::HTTP::RedirectionTooDeep ].freeze diff --git a/lib/gitlab/http_connection_adapter.rb b/lib/gitlab/http_connection_adapter.rb index 41eab3658bc..84eb60f3a5d 100644 --- a/lib/gitlab/http_connection_adapter.rb +++ b/lib/gitlab/http_connection_adapter.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # This class is part of the Gitlab::HTTP wrapper. Depending on the value -# of the global setting allow_local_requests_from_hooks_and_services this adapter +# of the global setting allow_local_requests_from_web_hooks_and_services this adapter # will allow/block connection to internal IPs and/or urls. # # This functionality can be overridden by providing the setting the option @@ -38,7 +38,7 @@ module Gitlab end def allow_settings_local_requests? - Gitlab::CurrentSettings.allow_local_requests_from_hooks_and_services? + Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services? end end end diff --git a/lib/gitlab/import/database_helpers.rb b/lib/gitlab/import/database_helpers.rb index 5b3f30d894a..aaade39dd62 100644 --- a/lib/gitlab/import/database_helpers.rb +++ b/lib/gitlab/import/database_helpers.rb @@ -6,9 +6,7 @@ module Gitlab # Inserts a raw row and returns the ID of the inserted row. # # attributes - The attributes/columns to set. - # relation - An ActiveRecord::Relation to use for finding the ID of the row - # when using MySQL. - # rubocop: disable CodeReuse/ActiveRecord + # relation - An ActiveRecord::Relation to use for finding the table name def insert_and_return_id(attributes, relation) # We use bulk_insert here so we can bypass any queries executed by # callbacks or validation rules, as doing this wouldn't scale when @@ -16,12 +14,8 @@ module Gitlab result = Gitlab::Database .bulk_insert(relation.table_name, [attributes], return_ids: true) - # MySQL doesn't support returning the IDs of a bulk insert in a way that - # is not a pain, so in this case we'll issue an extra query instead. - result.first || - relation.where(iid: attributes[:iid]).limit(1).pluck(:id).first + result.first end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index f63a5ece71e..bb46bd657e8 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -4,7 +4,9 @@ module Gitlab module ImportExport extend self - # For every version update, the version history in import_export.md has to be kept up to date. + # For every version update the version history in these docs must be kept up to date: + # - development/import_export.md + # - user/project/settings/import_export.md VERSION = '0.2.4'.freeze FILENAME_LIMIT = 50 @@ -28,6 +30,14 @@ module Gitlab "project.bundle" end + def lfs_objects_filename + "lfs-objects.json" + end + + def lfs_objects_storage + "lfs-objects" + end + def config_file Rails.root.join('lib/gitlab/import_export/import_export.yml') end diff --git a/lib/gitlab/import_export/attribute_cleaner.rb b/lib/gitlab/import_export/attribute_cleaner.rb index c28a1674018..b2fe9592c06 100644 --- a/lib/gitlab/import_export/attribute_cleaner.rb +++ b/lib/gitlab/import_export/attribute_cleaner.rb @@ -3,7 +3,7 @@ module Gitlab module ImportExport class AttributeCleaner - ALLOWED_REFERENCES = RelationFactory::PROJECT_REFERENCES + RelationFactory::USER_REFERENCES + ['group_id'] + ALLOWED_REFERENCES = RelationFactory::PROJECT_REFERENCES + RelationFactory::USER_REFERENCES + %w[group_id commit_id] PROHIBITED_REFERENCES = Regexp.union(/\Acached_markdown_version\Z/, /_id\Z/, /_html\Z/).freeze def self.clean(*args) diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb index 409243e68a5..42cd94add79 100644 --- a/lib/gitlab/import_export/attributes_finder.rb +++ b/lib/gitlab/import_export/attributes_finder.rb @@ -45,7 +45,7 @@ module Gitlab end def key_from_hash(value) - value.is_a?(Hash) ? value.keys.first : value + value.is_a?(Hash) ? value.first.first : value end end end diff --git a/lib/gitlab/import_export/config.rb b/lib/gitlab/import_export/config.rb new file mode 100644 index 00000000000..f6cd4eb5e0c --- /dev/null +++ b/lib/gitlab/import_export/config.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + class Config + # Returns a Hash of the YAML file, including EE specific data if EE is + # used. + def to_h + hash = parse_yaml + ee_hash = hash['ee'] + + if merge? && ee_hash + ee_hash.each do |key, value| + if key == 'project_tree' + merge_project_tree(value, hash[key]) + else + merge_attributes_list(value, hash[key]) + end + end + end + + # We don't want to expose this section after this point, as it is no + # longer needed. + hash.delete('ee') + + hash + end + + # Merges a project relationships tree into the target tree. + # + # @param [Array<Hash|Symbol>] source_values + # @param [Array<Hash|Symbol>] target_values + def merge_project_tree(source_values, target_values) + source_values.each do |value| + if value.is_a?(Hash) + # Examples: + # + # { 'project_tree' => [{ 'labels' => [...] }] } + # { 'notes' => [:author, { 'events' => [:push_event_payload] }] } + value.each do |key, val| + target = target_values + .find { |h| h.is_a?(Hash) && h[key] } + + if target + merge_project_tree(val, target[key]) + else + target_values << { key => val.dup } + end + end + else + # Example: :priorities, :author, etc + target_values << value + end + end + end + + # Merges a Hash containing a flat list of attributes, such as the entries + # in a `excluded_attributes` section. + # + # @param [Hash] source_values + # @param [Hash] target_values + def merge_attributes_list(source_values, target_values) + source_values.each do |key, values| + target_values[key] ||= [] + target_values[key].concat(values) + end + end + + def merge? + Gitlab.ee? + end + + def parse_yaml + YAML.load_file(Gitlab::ImportExport.config_file) + end + end + end +end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 71c44af9254..bd0f3e70749 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -1,7 +1,11 @@ # Model relationships to be included in the project import/export +# +# This list _must_ only contain relationships that are available to both CE and +# EE. EE specific relationships must be defined in the `ee` section further +# down below. project_tree: - labels: - :priorities + - :priorities - milestones: - events: - :push_event_payload @@ -15,29 +19,29 @@ project_tree: - :push_event_payload - label_links: - label: - :priorities + - :priorities - milestone: - events: - :push_event_payload - resource_label_events: - label: - :priorities + - :priorities - :issue_assignees - snippets: - :award_emoji - notes: - :author + - :author - releases: - :links - project_members: - :user - merge_requests: - :metrics - - :suggestions - notes: - :author - events: - :push_event_payload + - :suggestions - merge_request_diff: - :merge_request_diff_commits - :merge_request_diff_files @@ -46,13 +50,13 @@ project_tree: - :timelogs - label_links: - label: - :priorities + - :priorities - milestone: - events: - :push_event_payload - resource_label_events: - label: - :priorities + - :priorities - ci_pipelines: - notes: - :author @@ -76,6 +80,10 @@ project_tree: - :ci_cd_settings - :error_tracking_setting - :metrics_setting + - boards: + - lists: + - label: + - :priorities # Only include the following attributes for the models specified. included_attributes: @@ -121,12 +129,23 @@ excluded_attributes: - :bfg_object_map - :detected_repository_languages - :tag_list + - :mirror_user_id + - :mirror_trigger_builds + - :only_mirror_protected_branches + - :pull_mirror_available_overridden + - :mirror_overwrites_diverged_branches + - :packages_enabled + - :mirror_last_update_at + - :mirror_last_successful_update_at + - :emails_disabled namespaces: - :runners_token - :runners_token_encrypted project_import_state: - :last_error - :jid + - :last_update_at + - :last_successful_update_at prometheus_metrics: - :common - :identifier @@ -146,6 +165,7 @@ excluded_attributes: - :milestone_id - :ref_fetched - :merge_jid + - :rebase_jid - :latest_merge_request_diff_id award_emoji: - :awardable_id @@ -201,3 +221,14 @@ methods: - :action project_badges: - :type + lists: + - :list_type + +# EE specific relationships and settings to include. All of this will be merged +# into the previous structures if EE is used. +ee: + project_tree: + - protected_branches: + - :unprotect_access_levels + - protected_environments: + - :deploy_access_levels diff --git a/lib/gitlab/import_export/json_hash_builder.rb b/lib/gitlab/import_export/json_hash_builder.rb index b145f37c052..a92e3862361 100644 --- a/lib/gitlab/import_export/json_hash_builder.rb +++ b/lib/gitlab/import_export/json_hash_builder.rb @@ -27,7 +27,7 @@ module Gitlab # {:merge_requests=>[:merge_request_diff, :notes]} def process_model_objects(model_object_hash) json_config_hash = {} - current_key = model_object_hash.keys.first + current_key = model_object_hash.first.first model_object_hash.values.flatten.each do |model_object| @attributes_finder.parse(current_key) { |hash| json_config_hash[current_key] ||= hash } diff --git a/lib/gitlab/import_export/lfs_restorer.rb b/lib/gitlab/import_export/lfs_restorer.rb index 345c7880e30..1de8a5bf9ec 100644 --- a/lib/gitlab/import_export/lfs_restorer.rb +++ b/lib/gitlab/import_export/lfs_restorer.rb @@ -3,6 +3,10 @@ module Gitlab module ImportExport class LfsRestorer + include Gitlab::Utils::StrongMemoize + + attr_accessor :project, :shared + def initialize(project:, shared:) @project = project @shared = shared @@ -17,7 +21,7 @@ module Gitlab true rescue => e - @shared.error(e) + shared.error(e) false end @@ -29,16 +33,57 @@ module Gitlab lfs_object = LfsObject.find_or_initialize_by(oid: oid, size: size) lfs_object.file = File.open(path) unless lfs_object.file&.exists? + lfs_object.save! if lfs_object.changed? - @project.all_lfs_objects << lfs_object + repository_types(oid).each do |repository_type| + LfsObjectsProject.create!( + project: project, + lfs_object: lfs_object, + repository_type: repository_type + ) + end + end + + def repository_types(oid) + # We allow support for imports created before the `lfs-objects.json` + # file was generated. In this case, the restorer will link an LFS object + # with a single `lfs_objects_projects` relation. + # + # This allows us backwards-compatibility without version bumping. + # See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/30830#note_192608870 + return ['project'] unless has_lfs_json? + + lfs_json[oid] end def lfs_file_paths @lfs_file_paths ||= Dir.glob("#{lfs_storage_path}/*") end + def has_lfs_json? + strong_memoize(:has_lfs_json) do + File.exist?(lfs_json_path) + end + end + + def lfs_json + return {} unless has_lfs_json? + + @lfs_json ||= + begin + json = IO.read(lfs_json_path) + ActiveSupport::JSON.decode(json) + rescue + raise Gitlab::ImportExport::Error.new('Incorrect JSON format') + end + end + def lfs_storage_path - File.join(@shared.export_path, 'lfs-objects') + File.join(shared.export_path, ImportExport.lfs_objects_storage) + end + + def lfs_json_path + File.join(shared.export_path, ImportExport.lfs_objects_filename) end end end diff --git a/lib/gitlab/import_export/lfs_saver.rb b/lib/gitlab/import_export/lfs_saver.rb index 954f6f00078..18c590e1ca9 100644 --- a/lib/gitlab/import_export/lfs_saver.rb +++ b/lib/gitlab/import_export/lfs_saver.rb @@ -5,25 +5,40 @@ module Gitlab class LfsSaver include Gitlab::ImportExport::CommandLineUtil + attr_accessor :lfs_json, :project, :shared + + BATCH_SIZE = 100 + def initialize(project:, shared:) @project = project @shared = shared + @lfs_json = {} end def save - @project.all_lfs_objects.each do |lfs_object| - save_lfs_object(lfs_object) + project.all_lfs_objects.find_in_batches(batch_size: BATCH_SIZE) do |batch| + batch.each do |lfs_object| + save_lfs_object(lfs_object) + end + + append_lfs_json_for_batch(batch) if write_lfs_json_enabled? end + write_lfs_json if write_lfs_json_enabled? + true rescue => e - @shared.error(e) + shared.error(e) false end private + def write_lfs_json_enabled? + ::Feature.enabled?(:export_lfs_objects_projects, default_enabled: true) + end + def save_lfs_object(lfs_object) if lfs_object.local_store? copy_file_for_lfs_object(lfs_object) @@ -45,12 +60,36 @@ module Gitlab copy_files(lfs_object.file.path, destination_path_for_object(lfs_object)) end + def append_lfs_json_for_batch(lfs_objects_batch) + lfs_objects_projects = LfsObjectsProject + .select('lfs_objects.oid, array_agg(distinct lfs_objects_projects.repository_type) as repository_types') + .joins(:lfs_object) + .where(project: project, lfs_object: lfs_objects_batch) + .group('lfs_objects.oid') + + lfs_objects_projects.each do |group| + oid = group.oid + + lfs_json[oid] ||= [] + lfs_json[oid] += group.repository_types + end + end + + def write_lfs_json + mkdir_p(shared.export_path) + File.write(lfs_json_path, lfs_json.to_json) + end + def destination_path_for_object(lfs_object) File.join(lfs_export_path, lfs_object.oid) end def lfs_export_path - File.join(@shared.export_path, 'lfs-objects') + File.join(shared.export_path, ImportExport.lfs_objects_storage) + end + + def lfs_json_path + File.join(shared.export_path, ImportExport.lfs_objects_filename) end end end diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index a154de5419e..4e976cfca3a 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -35,7 +35,7 @@ module Gitlab end def include?(old_author_id) - map.keys.include?(old_author_id) && map[old_author_id] != default_user_id + map.has_key?(old_author_id) && map[old_author_id] != default_user_id end private @@ -50,6 +50,8 @@ module Gitlab @project.project_members.destroy_all # rubocop: disable DestroyAll ProjectMember.create!(user: @user, access_level: ProjectMember::MAINTAINER, source_id: @project.id, importing: true) + rescue => e + raise e, "Error adding importer user to project members. #{e.message}" end def add_team_member(member, existing_user = nil) diff --git a/lib/gitlab/import_export/merge_request_parser.rb b/lib/gitlab/import_export/merge_request_parser.rb index deb2f59f05f..0b534a5bafc 100644 --- a/lib/gitlab/import_export/merge_request_parser.rb +++ b/lib/gitlab/import_export/merge_request_parser.rb @@ -43,7 +43,7 @@ module Gitlab target_ref = Gitlab::Git::BRANCH_REF_PREFIX + @merge_request.source_branch unless @project.repository.fetch_source_branch!(@project.repository, @diff_head_sha, target_ref) - Rails.logger.warn("Import/Export warning: Failed to create #{target_ref} for MR: #{@merge_request.iid}") + Rails.logger.warn("Import/Export warning: Failed to create #{target_ref} for MR: #{@merge_request.iid}") # rubocop:disable Gitlab/RailsLogger end end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 20caadb89c0..91fe4e5d074 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -20,7 +20,7 @@ module Gitlab json = IO.read(@path) @tree_hash = ActiveSupport::JSON.decode(json) rescue => e - Rails.logger.error("Import/Export error: #{e.message}") + Rails.logger.error("Import/Export error: #{e.message}") # rubocop:disable Gitlab/RailsLogger raise Gitlab::ImportExport::Error.new('Incorrect JSON format') end @@ -130,6 +130,7 @@ module Gitlab def visibility_level level = override_params['visibility_level'] || json_params['visibility_level'] || @project.visibility_level level = @project.group.visibility_level if @project.group && level.to_i > @project.group.visibility_level + level = Gitlab::VisibilityLevel::PRIVATE if level == Gitlab::VisibilityLevel::INTERNAL && Gitlab::CurrentSettings.restricted_visibility_levels.include?(level) { 'visibility_level' => level } end diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb index bc0d18e03fa..8bdf6ca491d 100644 --- a/lib/gitlab/import_export/reader.rb +++ b/lib/gitlab/import_export/reader.rb @@ -7,7 +7,7 @@ module Gitlab def initialize(shared:) @shared = shared - config_hash = YAML.load_file(Gitlab::ImportExport.config_file).deep_symbolize_keys + config_hash = ImportExport::Config.new.to_h.deep_symbolize_keys @tree = config_hash[:project_tree] @attributes_finder = Gitlab::ImportExport::AttributesFinder.new(included_attributes: config_hash[:included_attributes], excluded_attributes: config_hash[:excluded_attributes], diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index efd3f550a22..0be49e27acb 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -28,7 +28,7 @@ module Gitlab links: 'Releases::Link', metrics_setting: 'ProjectMetricsSetting' }.freeze - USER_REFERENCES = %w[author_id assignee_id updated_by_id merged_by_id latest_closed_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id closed_by_id].freeze + USER_REFERENCES = %w[author_id assignee_id updated_by_id merged_by_id latest_closed_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id closed_by_id owner_id].freeze PROJECT_REFERENCES = %w[project_id source_project_id target_project_id].freeze @@ -78,6 +78,9 @@ module Gitlab def create return if unknown_service? + # Do not import legacy triggers + return if !Feature.enabled?(:use_legacy_pipeline_triggers, @project) && legacy_trigger? + setup_models generate_imported_object @@ -182,7 +185,7 @@ module Gitlab return unless EXISTING_OBJECT_CHECK.include?(@relation_name) return unless @relation_hash['group_id'] - @relation_hash['group_id'] = @project.group&.id + @relation_hash['group_id'] = @project.namespace_id end def reset_tokens! @@ -278,6 +281,10 @@ module Gitlab !Object.const_defined?(parsed_relation_hash['type']) end + def legacy_trigger? + @relation_name == 'Ci::Trigger' && @relation_hash['owner_id'].nil? + end + def find_or_create_object! return relation_class.find_or_create_by(project_id: @project.id) if @relation_name == :project_feature diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb index 72f575db095..bea7a7cce65 100644 --- a/lib/gitlab/import_export/saver.rb +++ b/lib/gitlab/import_export/saver.rb @@ -18,7 +18,7 @@ module Gitlab if compress_and_save remove_export_path - Rails.logger.info("Saved project export #{archive_file}") + Rails.logger.info("Saved project export #{archive_file}") # rubocop:disable Gitlab/RailsLogger save_upload else diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb index 6d978d00ea5..86ea7a30e69 100644 --- a/lib/gitlab/import_export/version_checker.rb +++ b/lib/gitlab/import_export/version_checker.rb @@ -36,7 +36,7 @@ module Gitlab def different_version?(version) Gem::Version.new(version) != Gem::Version.new(Gitlab::ImportExport.version) rescue => e - Rails.logger.error("Import/Export error: #{e.message}") + Rails.logger.error("Import/Export error: #{e.message}") # rubocop:disable Gitlab/RailsLogger raise Gitlab::ImportExport::Error.new('Incorrect VERSION format') end end diff --git a/lib/gitlab/instrumentation_helper.rb b/lib/gitlab/instrumentation_helper.rb new file mode 100644 index 00000000000..e6a5facb2a5 --- /dev/null +++ b/lib/gitlab/instrumentation_helper.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module InstrumentationHelper + extend self + + KEYS = %i(gitaly_calls gitaly_duration rugged_calls rugged_duration_ms).freeze + + def add_instrumentation_data(payload) + gitaly_calls = Gitlab::GitalyClient.get_request_count + + if gitaly_calls > 0 + payload[:gitaly_calls] = gitaly_calls + payload[:gitaly_duration] = Gitlab::GitalyClient.query_time_ms + end + + rugged_calls = Gitlab::RuggedInstrumentation.query_count + + if rugged_calls > 0 + payload[:rugged_calls] = rugged_calls + payload[:rugged_duration_ms] = Gitlab::RuggedInstrumentation.query_time_ms + end + end + end +end diff --git a/lib/gitlab/issuable_metadata.rb b/lib/gitlab/issuable_metadata.rb index 351d15605e0..be73bcd5506 100644 --- a/lib/gitlab/issuable_metadata.rb +++ b/lib/gitlab/issuable_metadata.rb @@ -2,7 +2,7 @@ module Gitlab module IssuableMetadata - def issuable_meta_data(issuable_collection, collection_type) + def issuable_meta_data(issuable_collection, collection_type, user = nil) # ActiveRecord uses Object#extend for null relations. if !(issuable_collection.singleton_class < ActiveRecord::NullRelation) && issuable_collection.respond_to?(:limit_value) && @@ -23,7 +23,7 @@ module Gitlab issuable_votes_count = ::AwardEmoji.votes_for_collection(issuable_ids, collection_type) issuable_merge_requests_count = if collection_type == 'Issue' - ::MergeRequestsClosingIssues.count_for_collection(issuable_ids) + ::MergeRequestsClosingIssues.count_for_collection(issuable_ids, user) else [] end diff --git a/lib/gitlab/json_cache.rb b/lib/gitlab/json_cache.rb index e4bc437d787..84c6817f3c7 100644 --- a/lib/gitlab/json_cache.rb +++ b/lib/gitlab/json_cache.rb @@ -22,10 +22,10 @@ module Gitlab expanded_cache_key = [namespace, key].compact if cache_key_with_version - expanded_cache_key << Rails.version + expanded_cache_key << [Gitlab::VERSION, Rails.version] end - expanded_cache_key.join(':') + expanded_cache_key.flatten.join(':').freeze end def expire(key) @@ -34,7 +34,7 @@ module Gitlab def read(key, klass = nil) value = backend.read(cache_key(key)) - value = parse_value(value, klass) if value + value = parse_value(value, klass) unless value.nil? value end @@ -58,7 +58,7 @@ module Gitlab private def parse_value(raw, klass) - value = ActiveSupport::JSON.decode(raw) + value = ActiveSupport::JSON.decode(raw.to_s) case value when Hash then parse_entry(value, klass) diff --git a/lib/gitlab/kubernetes.rb b/lib/gitlab/kubernetes.rb index d46b5e3aee3..22bd00751bc 100644 --- a/lib/gitlab/kubernetes.rb +++ b/lib/gitlab/kubernetes.rb @@ -37,15 +37,18 @@ module Gitlab # Filters an array of pods (as returned by the kubernetes API) by their project and environment def filter_by_project_environment(items, app, env) - pods = filter_by_annotation(items, { + filter_by_annotation(items, { 'app.gitlab.com/app' => app, 'app.gitlab.com/env' => env }) - return pods unless pods.empty? + end - filter_by_label(items, { - 'app' => env, # deprecated: replaced by app.gitlab.com/env - }) + def filter_by_legacy_label(items, app, env) + legacy_items = filter_by_label(items, { app: env }) + + non_legacy_items = filter_by_project_environment(legacy_items, app, env) + + legacy_items - non_legacy_items end # Converts a pod (as returned by the kubernetes API) into a terminal diff --git a/lib/gitlab/kubernetes/default_namespace.rb b/lib/gitlab/kubernetes/default_namespace.rb new file mode 100644 index 00000000000..c95362b024b --- /dev/null +++ b/lib/gitlab/kubernetes/default_namespace.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + class DefaultNamespace + attr_reader :cluster, :project + + delegate :platform_kubernetes, to: :cluster + + ## + # Ideally we would just use an environment record here instead of + # passing a project and name/slug separately, but we need to be able + # to look up namespaces before the environment has been persisted. + def initialize(cluster, project:) + @cluster = cluster + @project = project + end + + def from_environment_name(name) + from_environment_slug(generate_slug(name)) + end + + def from_environment_slug(slug) + default_platform_namespace(slug) || default_project_namespace(slug) + end + + private + + def default_platform_namespace(slug) + return unless platform_kubernetes&.namespace.present? + + if cluster.managed? && cluster.namespace_per_environment? + "#{platform_kubernetes.namespace}-#{slug}" + else + platform_kubernetes.namespace + end + end + + def default_project_namespace(slug) + namespace_slug = "#{project.path}-#{project.id}".downcase + + if cluster.namespace_per_environment? + namespace_slug += "-#{slug}" + end + + Gitlab::NamespaceSanitizer.sanitize(namespace_slug) + end + + ## + # Environment slug can be predicted given an environment + # name, so even if the environment isn't persisted yet we + # still know what to look for. + def generate_slug(name) + Gitlab::Slug::Environment.new(name).generate + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm.rb b/lib/gitlab/kubernetes/helm.rb index 42c4745ff98..6e4286589c1 100644 --- a/lib/gitlab/kubernetes/helm.rb +++ b/lib/gitlab/kubernetes/helm.rb @@ -3,8 +3,8 @@ module Gitlab module Kubernetes module Helm - HELM_VERSION = '2.12.3'.freeze - KUBECTL_VERSION = '1.11.7'.freeze + HELM_VERSION = '2.14.3'.freeze + KUBECTL_VERSION = '1.11.10'.freeze NAMESPACE = 'gitlab-managed-apps'.freeze SERVICE_ACCOUNT = 'tiller'.freeze CLUSTER_ROLE_BINDING = 'tiller-admin'.freeze diff --git a/lib/gitlab/kubernetes/helm/client_command.rb b/lib/gitlab/kubernetes/helm/client_command.rb index 9940272a8bf..6ae68306a9b 100644 --- a/lib/gitlab/kubernetes/helm/client_command.rb +++ b/lib/gitlab/kubernetes/helm/client_command.rb @@ -13,15 +13,27 @@ module Gitlab end def wait_for_tiller_command + helm_check = ['helm', 'version', *optional_tls_flags].shelljoin # This is necessary to give Tiller time to restart after upgrade. # Ideally we'd be able to use --wait but cannot because of # https://github.com/helm/helm/issues/4855 - 'for i in $(seq 1 30); do helm version && break; sleep 1s; echo "Retrying ($i)..."; done' + "for i in $(seq 1 30); do #{helm_check} && break; sleep 1s; echo \"Retrying ($i)...\"; done" end def repository_command ['helm', 'repo', 'add', name, repository].shelljoin if repository end + + def optional_tls_flags + return [] unless files.key?(:'ca.pem') + + [ + '--tls', + '--tls-ca-cert', "#{files_dir}/ca.pem", + '--tls-cert', "#{files_dir}/cert.pem", + '--tls-key', "#{files_dir}/key.pem" + ] + end end end end diff --git a/lib/gitlab/kubernetes/helm/delete_command.rb b/lib/gitlab/kubernetes/helm/delete_command.rb index 876994d2678..dcf22e7abb6 100644 --- a/lib/gitlab/kubernetes/helm/delete_command.rb +++ b/lib/gitlab/kubernetes/helm/delete_command.rb @@ -7,19 +7,24 @@ module Gitlab include BaseCommand include ClientCommand + attr_reader :predelete, :postdelete attr_accessor :name, :files - def initialize(name:, rbac:, files:) + def initialize(name:, rbac:, files:, predelete: nil, postdelete: nil) @name = name @files = files @rbac = rbac + @predelete = predelete + @postdelete = postdelete end def generate_script super + [ init_command, wait_for_tiller_command, - delete_command + predelete, + delete_command, + postdelete ].compact.join("\n") end @@ -38,17 +43,6 @@ module Gitlab command.shelljoin end - - def optional_tls_flags - return [] unless files.key?(:'ca.pem') - - [ - '--tls', - '--tls-ca-cert', "#{files_dir}/ca.pem", - '--tls-cert', "#{files_dir}/cert.pem", - '--tls-key', "#{files_dir}/key.pem" - ] - end end end end diff --git a/lib/gitlab/kubernetes/helm/install_command.rb b/lib/gitlab/kubernetes/helm/install_command.rb index e33ba9305ce..f572bc43533 100644 --- a/lib/gitlab/kubernetes/helm/install_command.rb +++ b/lib/gitlab/kubernetes/helm/install_command.rb @@ -27,9 +27,9 @@ module Gitlab wait_for_tiller_command, repository_command, repository_update_command, - preinstall_command, + preinstall, install_command, - postinstall_command + postinstall ].compact.join("\n") end @@ -58,14 +58,6 @@ module Gitlab command.shelljoin end - def preinstall_command - preinstall.join("\n") if preinstall - end - - def postinstall_command - postinstall.join("\n") if postinstall - end - def install_flag ['--install'] end @@ -95,17 +87,6 @@ module Gitlab ['--version', version] end - - def optional_tls_flags - return [] unless files.key?(:'ca.pem') - - [ - '--tls', - '--tls-ca-cert', "#{files_dir}/ca.pem", - '--tls-cert', "#{files_dir}/cert.pem", - '--tls-key', "#{files_dir}/key.pem" - ] - end end end end diff --git a/lib/gitlab/kubernetes/helm/reset_command.rb b/lib/gitlab/kubernetes/helm/reset_command.rb new file mode 100644 index 00000000000..a35ffa34c58 --- /dev/null +++ b/lib/gitlab/kubernetes/helm/reset_command.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + module Helm + class ResetCommand + include BaseCommand + include ClientCommand + + attr_reader :name, :files + + def initialize(name:, rbac:, files:) + @name = name + @files = files + @rbac = rbac + end + + def generate_script + super + [ + reset_helm_command, + delete_tiller_replicaset + ].join("\n") + end + + def rbac? + @rbac + end + + def pod_name + "uninstall-#{name}" + end + + private + + # This method can be delete once we upgrade Helm to > 12.13.0 + # https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/27096#note_159695900 + # + # Tracking this method to be removed here: + # https://gitlab.com/gitlab-org/gitlab-ce/issues/52791#note_199374155 + def delete_tiller_replicaset + delete_args = %w[replicaset -n gitlab-managed-apps -l name=tiller] + + Gitlab::Kubernetes::KubectlCmd.delete(*delete_args) + end + + def reset_helm_command + command = %w[helm reset] + optional_tls_flags + + command.shelljoin + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/kube_client.rb b/lib/gitlab/kubernetes/kube_client.rb index de14df56555..64317225ec6 100644 --- a/lib/gitlab/kubernetes/kube_client.rb +++ b/lib/gitlab/kubernetes/kube_client.rb @@ -59,6 +59,13 @@ module Gitlab # RBAC methods delegates to the apis/rbac.authorization.k8s.io api # group client + delegate :create_role, + :get_role, + :update_role, + to: :rbac_client + + # RBAC methods delegates to the apis/rbac.authorization.k8s.io api + # group client delegate :create_role_binding, :get_role_binding, :update_role_binding, @@ -121,7 +128,7 @@ module Gitlab private def validate_url! - return if Gitlab::CurrentSettings.allow_local_requests_from_hooks_and_services? + return if Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services? Gitlab::UrlBlocker.validate!(api_prefix, allow_local_network: false) end diff --git a/lib/gitlab/kubernetes/kubectl_cmd.rb b/lib/gitlab/kubernetes/kubectl_cmd.rb new file mode 100644 index 00000000000..981eb5681dc --- /dev/null +++ b/lib/gitlab/kubernetes/kubectl_cmd.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + module KubectlCmd + class << self + def delete(*args) + %w(kubectl delete).concat(args).shelljoin + end + + def apply_file(filename, *args) + raise ArgumentError, "filename is not present" unless filename.present? + + %w(kubectl apply -f).concat([filename], args).shelljoin + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/role.rb b/lib/gitlab/kubernetes/role.rb new file mode 100644 index 00000000000..096f60f0372 --- /dev/null +++ b/lib/gitlab/kubernetes/role.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + class Role + def initialize(name:, namespace:, rules:) + @name = name + @namespace = namespace + @rules = rules + end + + def generate + ::Kubeclient::Resource.new( + metadata: { name: name, namespace: namespace }, + rules: rules + ) + end + + private + + attr_reader :name, :namespace, :rules + end + end +end diff --git a/lib/gitlab/kubernetes/role_binding.rb b/lib/gitlab/kubernetes/role_binding.rb index cb0cb42d007..0404fb4453c 100644 --- a/lib/gitlab/kubernetes/role_binding.rb +++ b/lib/gitlab/kubernetes/role_binding.rb @@ -3,9 +3,10 @@ module Gitlab module Kubernetes class RoleBinding - def initialize(name:, role_name:, namespace:, service_account_name:) + def initialize(name:, role_name:, role_kind:, namespace:, service_account_name:) @name = name @role_name = role_name + @role_kind = role_kind @namespace = namespace @service_account_name = service_account_name end @@ -20,7 +21,7 @@ module Gitlab private - attr_reader :name, :role_name, :namespace, :service_account_name + attr_reader :name, :role_name, :role_kind, :namespace, :service_account_name def metadata { name: name, namespace: namespace } @@ -29,7 +30,7 @@ module Gitlab def role_ref { apiGroup: 'rbac.authorization.k8s.io', - kind: 'ClusterRole', + kind: role_kind, name: role_name } end diff --git a/lib/gitlab/legacy_github_import/client.rb b/lib/gitlab/legacy_github_import/client.rb index bbdd094e33b..b23efd64dee 100644 --- a/lib/gitlab/legacy_github_import/client.rb +++ b/lib/gitlab/legacy_github_import/client.rb @@ -101,7 +101,7 @@ module Gitlab # GitHub Rate Limit API returns 404 when the rate limit is # disabled. In this case we just want to return gracefully # instead of spitting out an error. - rescue Octokit::NotFound + rescue ::Octokit::NotFound nil end diff --git a/lib/gitlab/legacy_github_import/importer.rb b/lib/gitlab/legacy_github_import/importer.rb index 70b18221a66..751726d4810 100644 --- a/lib/gitlab/legacy_github_import/importer.rb +++ b/lib/gitlab/legacy_github_import/importer.rb @@ -55,7 +55,12 @@ module Gitlab import_pull_requests import_issues import_comments(:issues) - import_comments(:pull_requests) + + # Gitea doesn't have an API endpoint for pull requests comments + unless project.gitea_import? + import_comments(:pull_requests) + end + import_wiki # Gitea doesn't have a Release API yet diff --git a/lib/gitlab/legacy_github_import/release_formatter.rb b/lib/gitlab/legacy_github_import/release_formatter.rb index 746786b5a66..fdab6b512ea 100644 --- a/lib/gitlab/legacy_github_import/release_formatter.rb +++ b/lib/gitlab/legacy_github_import/release_formatter.rb @@ -10,6 +10,7 @@ module Gitlab name: raw_data.name, description: raw_data.body, created_at: raw_data.created_at, + released_at: raw_data.published_at, updated_at: raw_data.created_at } end diff --git a/lib/gitlab/lets_encrypt.rb b/lib/gitlab/lets_encrypt.rb new file mode 100644 index 00000000000..08ad2ab91b0 --- /dev/null +++ b/lib/gitlab/lets_encrypt.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Gitlab + module LetsEncrypt + def self.enabled? + Gitlab::CurrentSettings.lets_encrypt_terms_of_service_accepted + end + end +end diff --git a/lib/gitlab/lets_encrypt/client.rb b/lib/gitlab/lets_encrypt/client.rb index 66aea137012..ad2921ed555 100644 --- a/lib/gitlab/lets_encrypt/client.rb +++ b/lib/gitlab/lets_encrypt/client.rb @@ -34,14 +34,6 @@ module Gitlab acme_client.terms_of_service end - def enabled? - return false unless Feature.enabled?(:pages_auto_ssl) - - return false unless private_key - - Gitlab::CurrentSettings.lets_encrypt_terms_of_service_accepted - end - private def acme_client @@ -65,7 +57,7 @@ module Gitlab end def ensure_account - raise 'Acme integration is disabled' unless enabled? + raise 'Acme integration is disabled' unless ::Gitlab::LetsEncrypt.enabled? @acme_account ||= acme_client.new_account(contact: contact, terms_of_service_agreed: true) end diff --git a/lib/gitlab/markdown_cache/active_record/extension.rb b/lib/gitlab/markdown_cache/active_record/extension.rb index f3abe631779..233d3bf1ac7 100644 --- a/lib/gitlab/markdown_cache/active_record/extension.rb +++ b/lib/gitlab/markdown_cache/active_record/extension.rb @@ -26,10 +26,6 @@ module Gitlab attrs end - def changed_markdown_fields - changed_attributes.keys.map(&:to_s) & cached_markdown_fields.markdown_fields.map(&:to_s) - end - def write_markdown_field(field_name, value) write_attribute(field_name, value) end diff --git a/lib/gitlab/markdown_cache/redis/extension.rb b/lib/gitlab/markdown_cache/redis/extension.rb index 97fc23343b4..af3237f4ba6 100644 --- a/lib/gitlab/markdown_cache/redis/extension.rb +++ b/lib/gitlab/markdown_cache/redis/extension.rb @@ -36,8 +36,8 @@ module Gitlab false end - def changed_markdown_fields - [] + def changed_attributes + {} end def cached_markdown diff --git a/lib/gitlab/metrics/dashboard/base_service.rb b/lib/gitlab/metrics/dashboard/base_service.rb deleted file mode 100644 index 090014aa597..00000000000 --- a/lib/gitlab/metrics/dashboard/base_service.rb +++ /dev/null @@ -1,63 +0,0 @@ -# frozen_string_literal: true - -# Searches a projects repository for a metrics dashboard and formats the output. -# Expects any custom dashboards will be located in `.gitlab/dashboards` -module Gitlab - module Metrics - module Dashboard - class BaseService < ::BaseService - PROCESSING_ERROR = Gitlab::Metrics::Dashboard::Stages::BaseStage::DashboardProcessingError - NOT_FOUND_ERROR = Gitlab::Template::Finders::RepoTemplateFinder::FileNotFoundError - - def get_dashboard - success(dashboard: process_dashboard) - rescue NOT_FOUND_ERROR - error("#{dashboard_path} could not be found.", :not_found) - rescue PROCESSING_ERROR => e - error(e.message, :unprocessable_entity) - end - - # Summary of all known dashboards for the service. - # @return [Array<Hash>] ex) [{ path: String, default: Boolean }] - def self.all_dashboard_paths(_project) - raise NotImplementedError - end - - private - - # Returns a new dashboard Hash, supplemented with DB info - def process_dashboard - Gitlab::Metrics::Dashboard::Processor - .new(project, params[:environment], raw_dashboard) - .process(insert_project_metrics: insert_project_metrics?) - end - - # @return [String] Relative filepath of the dashboard yml - def dashboard_path - params[:dashboard_path] - end - - # Returns an un-processed dashboard from the cache. - def raw_dashboard - Gitlab::Metrics::Dashboard::Cache.fetch(cache_key) { get_raw_dashboard } - end - - # @return [Hash] an unmodified dashboard - def get_raw_dashboard - raise NotImplementedError - end - - # @return [String] - def cache_key - raise NotImplementedError - end - - # Determines whether custom metrics should be included - # in the processed output. - def insert_project_metrics? - false - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/defaults.rb b/lib/gitlab/metrics/dashboard/defaults.rb new file mode 100644 index 00000000000..3c39a7c6911 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/defaults.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +# Central point for managing default attributes from within +# the metrics dashboard module. +module Gitlab + module Metrics + module Dashboard + module Defaults + DEFAULT_PANEL_TYPE = 'area-chart' + DEFAULT_PANEL_WEIGHT = 0 + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/errors.rb b/lib/gitlab/metrics/dashboard/errors.rb new file mode 100644 index 00000000000..1739a4e6738 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/errors.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +# Central point for managing errors from within the metrics +# dashboard module. Handles errors from dashboard retrieval +# and processing steps, as well as defines shared error classes. +module Gitlab + module Metrics + module Dashboard + module Errors + PanelNotFoundError = Class.new(StandardError) + + PROCESSING_ERROR = Gitlab::Metrics::Dashboard::Stages::BaseStage::DashboardProcessingError + NOT_FOUND_ERROR = Gitlab::Template::Finders::RepoTemplateFinder::FileNotFoundError + + def handle_errors(error) + case error + when PROCESSING_ERROR + error(error.message, :unprocessable_entity) + when NOT_FOUND_ERROR + error("#{dashboard_path} could not be found.", :not_found) + when PanelNotFoundError + error(error.message, :not_found) + else + raise error + end + end + + def panels_not_found!(opts) + raise PanelNotFoundError.new("No panels matching properties #{opts}") + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/finder.rb b/lib/gitlab/metrics/dashboard/finder.rb index 49ea5c7d4f2..66c4d662a6c 100644 --- a/lib/gitlab/metrics/dashboard/finder.rb +++ b/lib/gitlab/metrics/dashboard/finder.rb @@ -9,17 +9,44 @@ module Gitlab class Finder class << self # Returns a formatted dashboard packed with DB info. + # @param project [Project] + # @param user [User] + # @param environment [Environment] + # @param options - embedded [Boolean] Determines whether the + # dashboard is to be rendered as part of an + # issue or location other than the primary + # metrics dashboard UI. Returns only the + # Memory/CPU charts of the system dash. + # @param options - dashboard_path [String] Path at which the + # dashboard can be found. Nil values will + # default to the system dashboard. + # @param options - group [String] Title of the group + # to which a panel might belong. Used by + # embedded dashboards. + # @param options - title [String] Title of the panel. + # Used by embedded dashboards. + # @param options - y_label [String] Y-Axis label of + # a panel. Used by embedded dashboards. # @return [Hash] - def find(project, user, environment, dashboard_path = nil) - service = system_dashboard?(dashboard_path) ? system_service : project_service - - service - .new(project, user, environment: environment, dashboard_path: dashboard_path) + def find(project, user, environment, options = {}) + service_for(options) + .new(project, user, options.merge(environment: environment)) .get_dashboard end + # Returns a dashboard without any supplemental info. + # Returns only full, yml-defined dashboards. + # @return [Hash] + def find_raw(project, dashboard_path: nil) + service_for(dashboard_path: dashboard_path) + .new(project, nil, dashboard_path: dashboard_path) + .raw_dashboard + end + # Summary of all known dashboards. - # @return [Array<Hash>] ex) [{ path: String, default: Boolean }] + # @return [Array<Hash>] ex) [{ path: String, + # display_name: String, + # default: Boolean }] def find_all_paths(project) project.repository.metrics_dashboard_paths end @@ -36,15 +63,15 @@ module Gitlab private def system_service - Gitlab::Metrics::Dashboard::SystemDashboardService + ::Metrics::Dashboard::SystemDashboardService end def project_service - Gitlab::Metrics::Dashboard::ProjectDashboardService + ::Metrics::Dashboard::ProjectDashboardService end - def system_dashboard?(filepath) - !filepath || system_service.system_dashboard?(filepath) + def service_for(options) + Gitlab::Metrics::Dashboard::ServiceSelector.call(options) end end end diff --git a/lib/gitlab/metrics/dashboard/project_dashboard_service.rb b/lib/gitlab/metrics/dashboard/project_dashboard_service.rb deleted file mode 100644 index e88658e4f9f..00000000000 --- a/lib/gitlab/metrics/dashboard/project_dashboard_service.rb +++ /dev/null @@ -1,39 +0,0 @@ -# frozen_string_literal: true - -# Searches a projects repository for a metrics dashboard and formats the output. -# Expects any custom dashboards will be located in `.gitlab/dashboards` -# Use Gitlab::Metrics::Dashboard::Finder to retrive dashboards. -module Gitlab - module Metrics - module Dashboard - class ProjectDashboardService < Gitlab::Metrics::Dashboard::BaseService - DASHBOARD_ROOT = ".gitlab/dashboards" - - class << self - def all_dashboard_paths(project) - file_finder(project) - .list_files_for(DASHBOARD_ROOT) - .map { |filepath| { path: filepath, default: false } } - end - - def file_finder(project) - Gitlab::Template::Finders::RepoTemplateFinder.new(project, DASHBOARD_ROOT, '.yml') - end - end - - private - - # Searches the project repo for a custom-defined dashboard. - def get_raw_dashboard - yml = self.class.file_finder(project).read(dashboard_path) - - YAML.safe_load(yml) - end - - def cache_key - "project_#{project.id}_metrics_dashboard_#{dashboard_path}" - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/service_selector.rb b/lib/gitlab/metrics/dashboard/service_selector.rb new file mode 100644 index 00000000000..934ba9145a2 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/service_selector.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +# Responsible for determining which dashboard service should +# be used to fetch or generate a dashboard hash. +# The services can be considered in two categories - embeds +# and dashboards. Embeds are all portions of dashboards. +module Gitlab + module Metrics + module Dashboard + class ServiceSelector + SERVICES = ::Metrics::Dashboard + + class << self + include Gitlab::Utils::StrongMemoize + + # Returns a class which inherits from the BaseService + # class that can be used to obtain a dashboard. + # @return [Gitlab::Metrics::Dashboard::Services::BaseService] + def call(params) + return SERVICES::CustomMetricEmbedService if custom_metric_embed?(params) + return SERVICES::DynamicEmbedService if dynamic_embed?(params) + return SERVICES::DefaultEmbedService if params[:embedded] + return SERVICES::SystemDashboardService if system_dashboard?(params[:dashboard_path]) + return SERVICES::ProjectDashboardService if params[:dashboard_path] + + default_service + end + + private + + def default_service + SERVICES::SystemDashboardService + end + + def system_dashboard?(filepath) + SERVICES::SystemDashboardService.system_dashboard?(filepath) + end + + def custom_metric_embed?(params) + SERVICES::CustomMetricEmbedService.valid_params?(params) + end + + def dynamic_embed?(params) + SERVICES::DynamicEmbedService.valid_params?(params) + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/stages/base_stage.rb b/lib/gitlab/metrics/dashboard/stages/base_stage.rb index 0db7b176e8d..514ed50e58d 100644 --- a/lib/gitlab/metrics/dashboard/stages/base_stage.rb +++ b/lib/gitlab/metrics/dashboard/stages/base_stage.rb @@ -5,11 +5,11 @@ module Gitlab module Dashboard module Stages class BaseStage + include Gitlab::Metrics::Dashboard::Defaults + DashboardProcessingError = Class.new(StandardError) LayoutError = Class.new(DashboardProcessingError) - DEFAULT_PANEL_TYPE = 'area-chart' - attr_reader :project, :environment, :dashboard def initialize(project, environment, dashboard) diff --git a/lib/gitlab/metrics/dashboard/stages/project_metrics_inserter.rb b/lib/gitlab/metrics/dashboard/stages/project_metrics_inserter.rb index 221610a14d1..643be309992 100644 --- a/lib/gitlab/metrics/dashboard/stages/project_metrics_inserter.rb +++ b/lib/gitlab/metrics/dashboard/stages/project_metrics_inserter.rb @@ -97,7 +97,7 @@ module Gitlab end def new_metric(metric) - metric.queries.first.merge(metric_id: metric.id) + metric.to_metric_hash end end end diff --git a/lib/gitlab/metrics/dashboard/system_dashboard_service.rb b/lib/gitlab/metrics/dashboard/system_dashboard_service.rb deleted file mode 100644 index 67509ed4230..00000000000 --- a/lib/gitlab/metrics/dashboard/system_dashboard_service.rb +++ /dev/null @@ -1,47 +0,0 @@ -# frozen_string_literal: true - -# Fetches the system metrics dashboard and formats the output. -# Use Gitlab::Metrics::Dashboard::Finder to retrive dashboards. -module Gitlab - module Metrics - module Dashboard - class SystemDashboardService < Gitlab::Metrics::Dashboard::BaseService - SYSTEM_DASHBOARD_PATH = 'config/prometheus/common_metrics.yml' - - class << self - def all_dashboard_paths(_project) - [{ - path: SYSTEM_DASHBOARD_PATH, - default: true - }] - end - - def system_dashboard?(filepath) - filepath == SYSTEM_DASHBOARD_PATH - end - end - - private - - def dashboard_path - SYSTEM_DASHBOARD_PATH - end - - # Returns the base metrics shipped with every GitLab service. - def get_raw_dashboard - yml = File.read(Rails.root.join(dashboard_path)) - - YAML.safe_load(yml) - end - - def cache_key - "metrics_dashboard_#{dashboard_path}" - end - - def insert_project_metrics? - true - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/url.rb b/lib/gitlab/metrics/dashboard/url.rb new file mode 100644 index 00000000000..94f8b2e02b1 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/url.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +# Manages url matching for metrics dashboards. +module Gitlab + module Metrics + module Dashboard + class Url + class << self + # Matches urls for a metrics dashboard. This could be + # either the /metrics endpoint or the /metrics_dashboard + # endpoint. + # + # EX - https://<host>/<namespace>/<project>/environments/<env_id>/metrics + def regex + %r{ + (?<url> + #{Regexp.escape(Gitlab.config.gitlab.url)} + \/#{Project.reference_pattern} + (?:\/\-)? + \/environments + \/(?<environment>\d+) + \/metrics + (?<query> + \?[a-zA-Z0-9%.()+_=-]+ + (&[a-zA-Z0-9%.()+_=-]+)* + )? + (?<anchor>\#[a-z0-9_-]+)? + ) + }x + end + + # Parses query params out from full url string into hash. + # + # Ex) 'https://<root>/<project>/<environment>/metrics?title=Title&group=Group' + # --> { title: 'Title', group: 'Group' } + def parse_query(url) + query_string = URI.parse(url).query.to_s + + CGI.parse(query_string) + .transform_values { |value| value.first } + .symbolize_keys + end + + # Builds a metrics dashboard url based on the passed in arguments + def build_dashboard_url(*args) + Gitlab::Routing.url_helpers.metrics_dashboard_namespace_project_environment_url(*args) + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/samplers/base_sampler.rb b/lib/gitlab/metrics/samplers/base_sampler.rb index 6a062e93f0f..d7d848d2833 100644 --- a/lib/gitlab/metrics/samplers/base_sampler.rb +++ b/lib/gitlab/metrics/samplers/base_sampler.rb @@ -19,7 +19,7 @@ module Gitlab def safe_sample sample rescue => e - Rails.logger.warn("#{self.class}: #{e}, stopping") + Rails.logger.warn("#{self.class}: #{e}, stopping") # rubocop:disable Gitlab/RailsLogger stop end diff --git a/lib/gitlab/metrics/samplers/influx_sampler.rb b/lib/gitlab/metrics/samplers/influx_sampler.rb index 5138b37f83e..1eae0a7bf45 100644 --- a/lib/gitlab/metrics/samplers/influx_sampler.rb +++ b/lib/gitlab/metrics/samplers/influx_sampler.rb @@ -15,19 +15,14 @@ module Gitlab @last_step = nil @metrics = [] - - @last_minor_gc = Delta.new(GC.stat[:minor_gc_count]) - @last_major_gc = Delta.new(GC.stat[:major_gc_count]) end def sample sample_memory_usage sample_file_descriptors - sample_gc flush ensure - GC::Profiler.clear @metrics.clear end @@ -43,23 +38,6 @@ module Gitlab add_metric('file_descriptors', value: System.file_descriptor_count) end - def sample_gc - time = GC::Profiler.total_time * 1000.0 - stats = GC.stat.merge(total_time: time) - - # We want the difference of GC runs compared to the last sample, not the - # total amount since the process started. - stats[:minor_gc_count] = - @last_minor_gc.compared_with(stats[:minor_gc_count]) - - stats[:major_gc_count] = - @last_major_gc.compared_with(stats[:major_gc_count]) - - stats[:count] = stats[:minor_gc_count] + stats[:major_gc_count] - - add_metric('gc_statistics', stats) - end - def add_metric(series, values, tags = {}) prefix = sidekiq? ? 'sidekiq_' : 'rails_' diff --git a/lib/gitlab/metrics/samplers/puma_sampler.rb b/lib/gitlab/metrics/samplers/puma_sampler.rb index 25e40c70230..8a24d4f3663 100644 --- a/lib/gitlab/metrics/samplers/puma_sampler.rb +++ b/lib/gitlab/metrics/samplers/puma_sampler.rb @@ -15,7 +15,6 @@ module Gitlab puma_workers: ::Gitlab::Metrics.gauge(:puma_workers, 'Total number of workers'), puma_running_workers: ::Gitlab::Metrics.gauge(:puma_running_workers, 'Number of active workers'), puma_stale_workers: ::Gitlab::Metrics.gauge(:puma_stale_workers, 'Number of stale workers'), - puma_phase: ::Gitlab::Metrics.gauge(:puma_phase, 'Phase number (increased during phased restarts)'), puma_running: ::Gitlab::Metrics.gauge(:puma_running, 'Number of running threads'), puma_queued_connections: ::Gitlab::Metrics.gauge(:puma_queued_connections, 'Number of connections in that worker\'s "todo" set waiting for a worker thread'), puma_active_connections: ::Gitlab::Metrics.gauge(:puma_active_connections, 'Number of threads processing a request'), @@ -43,7 +42,7 @@ module Gitlab def puma_stats Puma.stats rescue NoMethodError - Rails.logger.info "PumaSampler: stats are not available yet, waiting for Puma to boot" + Rails.logger.info "PumaSampler: stats are not available yet, waiting for Puma to boot" # rubocop:disable Gitlab/RailsLogger nil end @@ -54,7 +53,6 @@ module Gitlab last_status = worker['last_status'] labels = { worker: "worker_#{worker['index']}" } - metrics[:puma_phase].set(labels, worker['phase']) set_worker_metrics(last_status, labels) if last_status.present? end end @@ -76,7 +74,6 @@ module Gitlab metrics[:puma_workers].set(labels, stats['workers']) metrics[:puma_running_workers].set(labels, stats['booted_workers']) metrics[:puma_stale_workers].set(labels, stats['old_workers']) - metrics[:puma_phase].set(labels, stats['phase']) end def set_worker_metrics(stats, labels = {}) diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index 17eacbd21d8..3bfa3da35e0 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -6,6 +6,16 @@ module Gitlab module Metrics module Samplers class RubySampler < BaseSampler + GC_REPORT_BUCKETS = [0.001, 0.002, 0.005, 0.01, 0.05, 0.1, 0.5].freeze + + def initialize(interval) + GC::Profiler.clear + + metrics[:process_start_time_seconds].set(labels, Time.now.to_i) + + super + end + def metrics @metrics ||= init_metrics end @@ -24,18 +34,18 @@ module Gitlab def init_metrics metrics = { - file_descriptors: ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum), - memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels, :livesum), + file_descriptors: ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels), + memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels), process_cpu_seconds_total: ::Gitlab::Metrics.gauge(with_prefix(:process, :cpu_seconds_total), 'Process CPU seconds total'), process_max_fds: ::Gitlab::Metrics.gauge(with_prefix(:process, :max_fds), 'Process max fds'), - process_resident_memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:process, :resident_memory_bytes), 'Memory used', labels, :livesum), + process_resident_memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:process, :resident_memory_bytes), 'Memory used', labels), process_start_time_seconds: ::Gitlab::Metrics.gauge(with_prefix(:process, :start_time_seconds), 'Process start time seconds'), sampler_duration: ::Gitlab::Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels), - total_time: ::Gitlab::Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels) + gc_duration_seconds: ::Gitlab::Metrics.histogram(with_prefix(:gc, :duration_seconds), 'GC time', labels, GC_REPORT_BUCKETS) } GC.stat.keys.each do |key| - metrics[key] = ::Gitlab::Metrics.gauge(with_prefix(:gc_stat, key), to_doc_string(key), labels, :livesum) + metrics[key] = ::Gitlab::Metrics.gauge(with_prefix(:gc_stat, key), to_doc_string(key), labels) end metrics @@ -44,48 +54,41 @@ module Gitlab def sample start_time = System.monotonic_time - metrics[:file_descriptors].set(labels.merge(worker_label), System.file_descriptor_count) - metrics[:process_cpu_seconds_total].set(labels.merge(worker_label), ::Gitlab::Metrics::System.cpu_time) - metrics[:process_max_fds].set(labels.merge(worker_label), ::Gitlab::Metrics::System.max_open_file_descriptors) - metrics[:process_start_time_seconds].set(labels.merge(worker_label), ::Gitlab::Metrics::System.process_start_time) + metrics[:file_descriptors].set(labels, System.file_descriptor_count) + metrics[:process_cpu_seconds_total].set(labels, ::Gitlab::Metrics::System.cpu_time) + metrics[:process_max_fds].set(labels, ::Gitlab::Metrics::System.max_open_file_descriptors) set_memory_usage_metrics sample_gc metrics[:sampler_duration].increment(labels, System.monotonic_time - start_time) - ensure - GC::Profiler.clear end private def sample_gc - # Collect generic GC stats. + # Observe all GC samples + sample_gc_reports.each do |report| + metrics[:gc_duration_seconds].observe(labels, report[:GC_TIME]) + end + + # Collect generic GC stats GC.stat.each do |key, value| metrics[key].set(labels, value) end + end - # Collect the GC time since last sample in float seconds. - metrics[:total_time].increment(labels, GC::Profiler.total_time) + def sample_gc_reports + GC::Profiler.enable + GC::Profiler.raw_data + ensure + GC::Profiler.clear end def set_memory_usage_metrics memory_usage = System.memory_usage - memory_labels = labels.merge(worker_label) - metrics[:memory_bytes].set(memory_labels, memory_usage) - metrics[:process_resident_memory_bytes].set(memory_labels, memory_usage) - end - - def worker_label - return { worker: 'sidekiq' } if Sidekiq.server? - return {} unless defined?(Unicorn::Worker) - - worker_no = ::Prometheus::Client::Support::Unicorn.worker_id - if worker_no - { worker: worker_no } - else - { worker: 'master' } - end + metrics[:memory_bytes].set(labels, memory_usage) + metrics[:process_resident_memory_bytes].set(labels, memory_usage) end end end diff --git a/lib/gitlab/metrics/samplers/unicorn_sampler.rb b/lib/gitlab/metrics/samplers/unicorn_sampler.rb index 9af7e0afed4..355f938704e 100644 --- a/lib/gitlab/metrics/samplers/unicorn_sampler.rb +++ b/lib/gitlab/metrics/samplers/unicorn_sampler.rb @@ -54,7 +54,16 @@ module Gitlab end def unicorn_workers_count - `pgrep -f '[u]nicorn_rails worker.+ #{Rails.root.to_s}'`.split.count + http_servers.sum(&:worker_processes) # rubocop: disable CodeReuse/ActiveRecord + end + + # Traversal of ObjectSpace is expensive, on fully loaded application + # it takes around 80ms. The instances of HttpServers are not a subject + # to change so we can cache the list of servers. + def http_servers + return [] unless defined?(::Unicorn::HttpServer) + + @http_servers ||= ObjectSpace.each_object(::Unicorn::HttpServer).to_a end end end diff --git a/lib/gitlab/metrics/subscribers/rails_cache.rb b/lib/gitlab/metrics/subscribers/rails_cache.rb index 01db507761b..2ee7144fe2f 100644 --- a/lib/gitlab/metrics/subscribers/rails_cache.rb +++ b/lib/gitlab/metrics/subscribers/rails_cache.rb @@ -50,7 +50,8 @@ module Gitlab def observe(key, duration) return unless current_transaction - metric_cache_operation_duration_seconds.observe(current_transaction.labels.merge({ operation: key }), duration / 1000.0) + metric_cache_operations_total.increment(current_transaction.labels.merge({ operation: key })) + metric_cache_operation_duration_seconds.observe({ operation: key }, duration / 1000.0) current_transaction.increment(:cache_duration, duration, false) current_transaction.increment(:cache_count, 1, false) current_transaction.increment("cache_#{key}_duration".to_sym, duration, false) @@ -63,12 +64,20 @@ module Gitlab Transaction.current end + def metric_cache_operations_total + @metric_cache_operations_total ||= ::Gitlab::Metrics.counter( + :gitlab_cache_operations_total, + 'Cache operations', + Transaction::BASE_LABELS + ) + end + def metric_cache_operation_duration_seconds @metric_cache_operation_duration_seconds ||= ::Gitlab::Metrics.histogram( :gitlab_cache_operation_duration_seconds, 'Cache access time', - Transaction::BASE_LABELS.merge({ action: nil }), - [0.001, 0.01, 0.1, 1, 10] + {}, + [0.00001, 0.0001, 0.001, 0.01, 0.1, 1.0] ) end diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index 33c0de91c11..51f48095cb5 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -31,14 +31,6 @@ module Gitlab match[1].to_i end - - def self.process_start_time - fields = File.read('/proc/self/stat').split - - # fields[21] is linux proc stat field "(22) starttime". - # The value is expressed in clock ticks, divide by clock ticks for seconds. - ( fields[21].to_i || 0 ) / clk_tck - end else def self.memory_usage 0.0 @@ -51,23 +43,11 @@ module Gitlab def self.max_open_file_descriptors 0 end - - def self.process_start_time - 0 - end end - # THREAD_CPUTIME is not supported on OS X - if Process.const_defined?(:CLOCK_THREAD_CPUTIME_ID) - def self.cpu_time - Process - .clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID, :float_second) - end - else - def self.cpu_time - Process - .clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID, :float_second) - end + def self.cpu_time + Process + .clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID, :float_second) end # Returns the current real time in a given precision. @@ -83,10 +63,6 @@ module Gitlab def self.monotonic_time Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_second) end - - def self.clk_tck - @clk_tck ||= `getconf CLK_TCK`.to_i - end end end end diff --git a/lib/gitlab/middleware/read_only/controller.rb b/lib/gitlab/middleware/read_only/controller.rb index 817db12ac55..53e55269c6e 100644 --- a/lib/gitlab/middleware/read_only/controller.rb +++ b/lib/gitlab/middleware/read_only/controller.rb @@ -25,7 +25,7 @@ module Gitlab def call if disallowed_request? && Gitlab::Database.read_only? - Rails.logger.debug('GitLab ReadOnly: preventing possible non read-only operation') + Rails.logger.debug('GitLab ReadOnly: preventing possible non read-only operation') # rubocop:disable Gitlab/RailsLogger if json_request? return [403, { 'Content-Type' => APPLICATION_JSON }, [{ 'message' => ERROR_MESSAGE }.to_json]] diff --git a/lib/gitlab/namespaced_session_store.rb b/lib/gitlab/namespaced_session_store.rb index 34520078bfb..f0f24c081c3 100644 --- a/lib/gitlab/namespaced_session_store.rb +++ b/lib/gitlab/namespaced_session_store.rb @@ -4,19 +4,24 @@ module Gitlab class NamespacedSessionStore delegate :[], :[]=, to: :store - def initialize(key) + def initialize(key, session = Session.current) @key = key + @session = session end def initiated? - !Session.current.nil? + !session.nil? end def store - return unless Session.current + return unless session - Session.current[@key] ||= {} - Session.current[@key] + session[@key] ||= {} + session[@key] end + + private + + attr_reader :session end end diff --git a/lib/gitlab/object_hierarchy.rb b/lib/gitlab/object_hierarchy.rb index 38b32770e90..c06f106ffe1 100644 --- a/lib/gitlab/object_hierarchy.rb +++ b/lib/gitlab/object_hierarchy.rb @@ -32,11 +32,6 @@ module Gitlab # Returns the maximum depth starting from the base # A base object with no children has a maximum depth of `1` def max_descendants_depth - unless hierarchy_supported? - # This makes the return value consistent with the case where hierarchy is supported - return descendants_base.exists? ? 1 : nil - end - base_and_descendants(with_depth: true).maximum(DEPTH_COLUMN) end @@ -66,8 +61,6 @@ module Gitlab # each parent. # rubocop: disable CodeReuse/ActiveRecord def base_and_ancestors(upto: nil, hierarchy_order: nil) - return ancestors_base unless hierarchy_supported? - recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(model.all) recursive_query = recursive_query.order(depth: hierarchy_order) if hierarchy_order @@ -81,10 +74,6 @@ module Gitlab # When `with_depth` is `true`, a `depth` column is included where it starts with `1` for the base objects # and incremented as we go down the descendant tree def base_and_descendants(with_depth: false) - unless hierarchy_supported? - return with_depth ? descendants_base.select("1 as #{DEPTH_COLUMN}", objects_table[Arel.star]) : descendants_base - end - read_only(base_and_descendants_cte(with_depth: with_depth).apply_to(model.all)) end @@ -112,8 +101,6 @@ module Gitlab # If nested objects are not supported, ancestors_base is returned. # rubocop: disable CodeReuse/ActiveRecord def all_objects - return ancestors_base unless hierarchy_supported? - ancestors = base_and_ancestors_cte descendants = base_and_descendants_cte @@ -135,10 +122,6 @@ module Gitlab private - def hierarchy_supported? - Gitlab::Database.postgresql? - end - # rubocop: disable CodeReuse/ActiveRecord def base_and_ancestors_cte(stop_id = nil, hierarchy_order = nil) cte = SQL::RecursiveCTE.new(:base_and_ancestors) diff --git a/lib/gitlab/octokit/middleware.rb b/lib/gitlab/octokit/middleware.rb new file mode 100644 index 00000000000..2dd7d08a58b --- /dev/null +++ b/lib/gitlab/octokit/middleware.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Octokit + class Middleware + def initialize(app) + @app = app + end + + def call(env) + Gitlab::UrlBlocker.validate!(env[:url], { allow_localhost: allow_local_requests?, allow_local_network: allow_local_requests? }) + + @app.call(env) + end + + private + + def allow_local_requests? + Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services? + end + end + end +end diff --git a/lib/gitlab/omniauth_initializer.rb b/lib/gitlab/omniauth_initializer.rb index 2a2083ebae0..ad1377a0892 100644 --- a/lib/gitlab/omniauth_initializer.rb +++ b/lib/gitlab/omniauth_initializer.rb @@ -52,6 +52,16 @@ module Gitlab args[:strategy_class] = args[:strategy_class].constantize end + # Providers that are known to depend on rack-oauth2, like those using + # Omniauth::Strategies::OpenIDConnect, need to be quirked so the + # client_auth_method argument value is passed as a symbol. + if (args[:strategy_class] == OmniAuth::Strategies::OpenIDConnect || + args[:name] == 'openid_connect') && + args[:client_auth_method].is_a?(String) + + args[:client_auth_method] = args[:client_auth_method].to_sym + end + args end diff --git a/lib/gitlab/optimistic_locking.rb b/lib/gitlab/optimistic_locking.rb index 868b2ae641a..0c0f46d3b77 100644 --- a/lib/gitlab/optimistic_locking.rb +++ b/lib/gitlab/optimistic_locking.rb @@ -5,6 +5,7 @@ module Gitlab module_function def retry_lock(subject, retries = 100, &block) + # TODO(Observability): We should be recording details of the number of retries and the duration of the total execution here ActiveRecord::Base.transaction do yield(subject) end diff --git a/lib/gitlab/pages_client.rb b/lib/gitlab/pages_client.rb index d74fdba2241..281eafb142f 100644 --- a/lib/gitlab/pages_client.rb +++ b/lib/gitlab/pages_client.rb @@ -110,7 +110,7 @@ module Gitlab end rescue Errno::EACCES => ex # TODO stop rescuing this exception in GitLab 11.0 https://gitlab.com/gitlab-org/gitlab-ce/issues/45672 - Rails.logger.error("Could not write pages admin token file: #{ex}") + Rails.logger.error("Could not write pages admin token file: #{ex}") # rubocop:disable Gitlab/RailsLogger rescue Errno::EEXIST # Another process wrote the token file concurrently with us. Use their token, not ours. end diff --git a/lib/gitlab/patch/active_record_query_cache.rb b/lib/gitlab/patch/active_record_query_cache.rb new file mode 100644 index 00000000000..71d66bdbe02 --- /dev/null +++ b/lib/gitlab/patch/active_record_query_cache.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +# Fixes a bug where the query cache isn't aware of the shared +# ActiveRecord connection used in tests +# https://github.com/rails/rails/issues/36587 + +# To be removed with https://gitlab.com/gitlab-org/gitlab-ce/issues/64413 + +module Gitlab + module Patch + module ActiveRecordQueryCache + # rubocop:disable Gitlab/ModuleWithInstanceVariables + def enable_query_cache! + @query_cache_enabled[connection_cache_key(current_thread)] = true + connection.enable_query_cache! if active_connection? + end + + def disable_query_cache! + @query_cache_enabled.delete connection_cache_key(current_thread) + connection.disable_query_cache! if active_connection? + end + + def query_cache_enabled + @query_cache_enabled[connection_cache_key(current_thread)] + end + + def active_connection? + @thread_cached_conns[connection_cache_key(current_thread)] + end + + private + + def current_thread + @lock_thread || Thread.current + end + # rubocop:enable Gitlab/ModuleWithInstanceVariables + end + end +end diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index a07b1246bee..f96466b2b00 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -53,7 +53,6 @@ module Gitlab sent_notifications slash-command-logo.png snippets - u unsubscribes uploads users @@ -176,6 +175,10 @@ module Gitlab @project_git_route_regex ||= /#{project_route_regex}\.git/.freeze end + def project_wiki_git_route_regex + @project_wiki_git_route_regex ||= /#{PATH_REGEX_STR}\.wiki/.freeze + end + def full_namespace_path_regex @full_namespace_path_regex ||= %r{\A#{full_namespace_route_regex}/\z} end diff --git a/lib/gitlab/performance_bar.rb b/lib/gitlab/performance_bar.rb index 4b0c7b5c7f8..07439d8e011 100644 --- a/lib/gitlab/performance_bar.rb +++ b/lib/gitlab/performance_bar.rb @@ -3,7 +3,8 @@ module Gitlab module PerformanceBar ALLOWED_USER_IDS_KEY = 'performance_bar_allowed_user_ids:v2'.freeze - EXPIRY_TIME = 5.minutes + EXPIRY_TIME_L1_CACHE = 1.minute + EXPIRY_TIME_L2_CACHE = 5.minutes def self.enabled?(user = nil) return true if Rails.env.development? @@ -19,20 +20,31 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def self.allowed_user_ids - Rails.cache.fetch(ALLOWED_USER_IDS_KEY, expires_in: EXPIRY_TIME) do - group = Group.find_by_id(allowed_group_id) + l1_cache_backend.fetch(ALLOWED_USER_IDS_KEY, expires_in: EXPIRY_TIME_L1_CACHE) do + l2_cache_backend.fetch(ALLOWED_USER_IDS_KEY, expires_in: EXPIRY_TIME_L2_CACHE) do + group = Group.find_by_id(allowed_group_id) - if group - GroupMembersFinder.new(group).execute.pluck(:user_id) - else - [] + if group + GroupMembersFinder.new(group).execute.pluck(:user_id) + else + [] + end end end end # rubocop: enable CodeReuse/ActiveRecord def self.expire_allowed_user_ids_cache - Rails.cache.delete(ALLOWED_USER_IDS_KEY) + l1_cache_backend.delete(ALLOWED_USER_IDS_KEY) + l2_cache_backend.delete(ALLOWED_USER_IDS_KEY) + end + + def self.l1_cache_backend + Gitlab::ThreadMemoryCache.cache_backend + end + + def self.l2_cache_backend + Rails.cache end end end diff --git a/lib/gitlab/performance_bar/peek_query_tracker.rb b/lib/gitlab/performance_bar/peek_query_tracker.rb deleted file mode 100644 index 3a27e26eaba..00000000000 --- a/lib/gitlab/performance_bar/peek_query_tracker.rb +++ /dev/null @@ -1,45 +0,0 @@ -# frozen_string_literal: true - -# Inspired by https://github.com/peek/peek-pg/blob/master/lib/peek/views/pg.rb -# PEEK_DB_CLIENT is a constant set in config/initializers/peek.rb -module Gitlab - module PerformanceBar - module PeekQueryTracker - def sorted_queries - PEEK_DB_CLIENT.query_details - .sort { |a, b| b[:duration] <=> a[:duration] } - end - - def results - super.merge(queries: sorted_queries) - end - - private - - def setup_subscribers - super - - # Reset each counter when a new request starts - before_request do - PEEK_DB_CLIENT.query_details = [] - end - - subscribe('sql.active_record') do |_, start, finish, _, data| - if Gitlab::SafeRequestStore.store[:peek_enabled] - unless data[:cached] - backtrace = Gitlab::Profiler.clean_backtrace(caller) - track_query(data[:sql].strip, data[:binds], backtrace, start, finish) - end - end - end - end - - def track_query(raw_query, bindings, backtrace, start, finish) - duration = (finish - start) * 1000.0 - query_info = { duration: duration.round(3), sql: raw_query, backtrace: backtrace } - - PEEK_DB_CLIENT.query_details << query_info - end - end - end -end diff --git a/lib/gitlab/performance_bar/redis_adapter_when_peek_enabled.rb b/lib/gitlab/performance_bar/redis_adapter_when_peek_enabled.rb new file mode 100644 index 00000000000..2d997760c46 --- /dev/null +++ b/lib/gitlab/performance_bar/redis_adapter_when_peek_enabled.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +# Adapted from https://github.com/peek/peek/blob/master/lib/peek/adapters/redis.rb +module Gitlab + module PerformanceBar + module RedisAdapterWhenPeekEnabled + def save + super unless ::Peek.request_id.blank? + end + end + end +end diff --git a/lib/gitlab/phabricator_import/cache/map.rb b/lib/gitlab/phabricator_import/cache/map.rb index fa8b37b20ca..6a2841b6a8e 100644 --- a/lib/gitlab/phabricator_import/cache/map.rb +++ b/lib/gitlab/phabricator_import/cache/map.rb @@ -9,9 +9,15 @@ module Gitlab def get_gitlab_model(phabricator_id) cached_info = get(phabricator_id) - return unless cached_info[:classname] && cached_info[:database_id] - cached_info[:classname].constantize.find_by_id(cached_info[:database_id]) + if cached_info[:classname] && cached_info[:database_id] + object = cached_info[:classname].constantize.find_by_id(cached_info[:database_id]) + else + object = yield if block_given? + set_gitlab_model(object, phabricator_id) if object + end + + object end def set_gitlab_model(object, phabricator_id) diff --git a/lib/gitlab/phabricator_import/conduit/user.rb b/lib/gitlab/phabricator_import/conduit/user.rb new file mode 100644 index 00000000000..fc8c3f7cde9 --- /dev/null +++ b/lib/gitlab/phabricator_import/conduit/user.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true +module Gitlab + module PhabricatorImport + module Conduit + class User + MAX_PAGE_SIZE = 100 + + def initialize(phabricator_url:, api_token:) + @client = Client.new(phabricator_url, api_token) + end + + def users(phids) + phids.each_slice(MAX_PAGE_SIZE).map { |limited_phids| get_page(limited_phids) } + end + + private + + def get_page(phids) + UsersResponse.new(get_users(phids)) + end + + def get_users(phids) + client.get('user.search', + params: { constraints: { phids: phids } }) + end + + attr_reader :client + end + end + end +end diff --git a/lib/gitlab/phabricator_import/conduit/users_response.rb b/lib/gitlab/phabricator_import/conduit/users_response.rb new file mode 100644 index 00000000000..3dfb29a7be5 --- /dev/null +++ b/lib/gitlab/phabricator_import/conduit/users_response.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module PhabricatorImport + module Conduit + class UsersResponse + def initialize(conduit_response) + @conduit_response = conduit_response + end + + def users + @users ||= conduit_response.data.map do |user_json| + Gitlab::PhabricatorImport::Representation::User.new(user_json) + end + end + + private + + attr_reader :conduit_response + end + end + end +end diff --git a/lib/gitlab/phabricator_import/issues/task_importer.rb b/lib/gitlab/phabricator_import/issues/task_importer.rb index 40d4392cbc1..77ee11c7cdd 100644 --- a/lib/gitlab/phabricator_import/issues/task_importer.rb +++ b/lib/gitlab/phabricator_import/issues/task_importer.rb @@ -8,9 +8,7 @@ module Gitlab end def execute - # TODO: get the user from the project namespace from the username loaded by Phab-id - # https://gitlab.com/gitlab-org/gitlab-ce/issues/60565 - issue.author = User.ghost + issue.author = user_finder.find(task.author_phid) || User.ghost # TODO: Reformat the description with attachments, escaping accidental # links and add attachments @@ -19,6 +17,10 @@ module Gitlab save! + if owner = user_finder.find(task.owner_phid) + issue.assignees << owner + end + issue end @@ -41,6 +43,10 @@ module Gitlab project.issues.new end + def user_finder + @issue_finder ||= Gitlab::PhabricatorImport::UserFinder.new(project, task.phids) + end + def find_issue_by_phabricator_id(phabricator_id) object_map.get_gitlab_model(phabricator_id) end diff --git a/lib/gitlab/phabricator_import/representation/task.rb b/lib/gitlab/phabricator_import/representation/task.rb index 6aedc71b626..ba93fb37a8e 100644 --- a/lib/gitlab/phabricator_import/representation/task.rb +++ b/lib/gitlab/phabricator_import/representation/task.rb @@ -11,6 +11,18 @@ module Gitlab json['phid'] end + def author_phid + json['fields']['authorPHID'] + end + + def owner_phid + json['fields']['ownerPHID'] + end + + def phids + @phids ||= [author_phid, owner_phid] + end + def issue_attributes @issue_attributes ||= { title: issue_title, diff --git a/lib/gitlab/phabricator_import/representation/user.rb b/lib/gitlab/phabricator_import/representation/user.rb new file mode 100644 index 00000000000..7fd7cecc6ae --- /dev/null +++ b/lib/gitlab/phabricator_import/representation/user.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module PhabricatorImport + module Representation + class User + def initialize(json) + @json = json + end + + def phabricator_id + json['phid'] + end + + def username + json['fields']['username'] + end + + private + + attr_reader :json + end + end + end +end diff --git a/lib/gitlab/phabricator_import/user_finder.rb b/lib/gitlab/phabricator_import/user_finder.rb new file mode 100644 index 00000000000..4b50431e0e0 --- /dev/null +++ b/lib/gitlab/phabricator_import/user_finder.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module PhabricatorImport + class UserFinder + def initialize(project, phids) + @project, @phids = project, phids + @loaded_phids = Set.new + end + + def find(phid) + found_user = object_map.get_gitlab_model(phid) do + find_user_for_phid(phid) + end + + loaded_phids << phid + + found_user + end + + private + + attr_reader :project, :phids, :loaded_phids + + def object_map + @object_map ||= Gitlab::PhabricatorImport::Cache::Map.new(project) + end + + def find_user_for_phid(phid) + phabricator_user = phabricator_users.find { |u| u.phabricator_id == phid } + return unless phabricator_user + + project.authorized_users.find_by_username(phabricator_user.username) + end + + def phabricator_users + @user_responses ||= client.users(users_to_request).flat_map(&:users) + end + + def users_to_request + phids - loaded_phids.to_a + end + + def client + @client ||= + Gitlab::PhabricatorImport::Conduit::User + .new(phabricator_url: project.import_data.data['phabricator_url'], + api_token: project.import_data.credentials[:api_token]) + end + end + end +end diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb index 890228e5e78..ec7671f9a8b 100644 --- a/lib/gitlab/profiler.rb +++ b/lib/gitlab/profiler.rb @@ -21,6 +21,9 @@ module Gitlab lib/gitlab/profiler.rb lib/gitlab/correlation_id.rb lib/gitlab/webpack/dev_server_middleware.rb + lib/gitlab/sidekiq_status/ + lib/gitlab/sidekiq_logging/ + lib/gitlab/sidekiq_middleware/ ].freeze # Takes a URL to profile (can be a fully-qualified URL, or an absolute path) @@ -166,7 +169,7 @@ module Gitlab [model, times.count, times.sum] end - summarised_load_times.sort_by(&:last).reverse.each do |(model, query_count, time)| + summarised_load_times.sort_by(&:last).reverse_each do |(model, query_count, time)| logger.info("#{model} total (#{query_count}): #{time.round(2)}ms") end end diff --git a/lib/gitlab/project_authorizations.rb b/lib/gitlab/project_authorizations.rb new file mode 100644 index 00000000000..a9270cd536e --- /dev/null +++ b/lib/gitlab/project_authorizations.rb @@ -0,0 +1,121 @@ +# frozen_string_literal: true + +# This class relies on Common Table Expressions to efficiently get all data, +# including data for nested groups. +module Gitlab + class ProjectAuthorizations + attr_reader :user + + # user - The User object for which to calculate the authorizations. + def initialize(user) + @user = user + end + + def calculate + cte = recursive_cte + cte_alias = cte.table.alias(Group.table_name) + projects = Project.arel_table + links = ProjectGroupLink.arel_table + + relations = [ + # The project a user has direct access to. + user.projects.select_for_project_authorization, + + # The personal projects of the user. + user.personal_projects.select_as_maintainer_for_project_authorization, + + # Projects that belong directly to any of the groups the user has + # access to. + Namespace + .unscoped + .select([alias_as_column(projects[:id], 'project_id'), + cte_alias[:access_level]]) + .from(cte_alias) + .joins(:projects), + + # Projects shared with any of the namespaces the user has access to. + Namespace + .unscoped + .select([ + links[:project_id], + least(cte_alias[:access_level], links[:group_access], 'access_level') + ]) + .from(cte_alias) + .joins('INNER JOIN project_group_links ON project_group_links.group_id = namespaces.id') + .joins('INNER JOIN projects ON projects.id = project_group_links.project_id') + .joins('INNER JOIN namespaces p_ns ON p_ns.id = projects.namespace_id') + .where('p_ns.share_with_group_lock IS FALSE') + ] + + ProjectAuthorization + .unscoped + .with + .recursive(cte.to_arel) + .select_from_union(relations) + end + + private + + # Builds a recursive CTE that gets all the groups the current user has + # access to, including any nested groups. + def recursive_cte + cte = Gitlab::SQL::RecursiveCTE.new(:namespaces_cte) + members = Member.arel_table + namespaces = Namespace.arel_table + + # Namespaces the user is a member of. + cte << user.groups + .select([namespaces[:id], members[:access_level]]) + .except(:order) + + # Sub groups of any groups the user is a member of. + cte << Group.select([ + namespaces[:id], + greatest(members[:access_level], cte.table[:access_level], 'access_level') + ]) + .joins(join_cte(cte)) + .joins(join_members) + .except(:order) + + cte + end + + # Builds a LEFT JOIN to join optional memberships onto the CTE. + def join_members + members = Member.arel_table + namespaces = Namespace.arel_table + + cond = members[:source_id] + .eq(namespaces[:id]) + .and(members[:source_type].eq('Namespace')) + .and(members[:requested_at].eq(nil)) + .and(members[:user_id].eq(user.id)) + + Arel::Nodes::OuterJoin.new(members, Arel::Nodes::On.new(cond)) + end + + # Builds an INNER JOIN to join namespaces onto the CTE. + def join_cte(cte) + namespaces = Namespace.arel_table + cond = cte.table[:id].eq(namespaces[:parent_id]) + + Arel::Nodes::InnerJoin.new(cte.table, Arel::Nodes::On.new(cond)) + end + + def greatest(left, right, column_alias) + sql_function('GREATEST', [left, right], column_alias) + end + + def least(left, right, column_alias) + sql_function('LEAST', [left, right], column_alias) + end + + def sql_function(name, args, column_alias) + alias_as_column(Arel::Nodes::NamedFunction.new(name, args), column_alias) + end + + def alias_as_column(value, alias_to) + Arel::Nodes::As.new(value, Arel::Nodes::SqlLiteral.new(alias_to)) + end + end +end diff --git a/lib/gitlab/project_authorizations/with_nested_groups.rb b/lib/gitlab/project_authorizations/with_nested_groups.rb deleted file mode 100644 index 2372a316ab0..00000000000 --- a/lib/gitlab/project_authorizations/with_nested_groups.rb +++ /dev/null @@ -1,125 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ProjectAuthorizations - # Calculating new project authorizations when supporting nested groups. - # - # This class relies on Common Table Expressions to efficiently get all data, - # including data for nested groups. As a result this class can only be used - # on PostgreSQL. - class WithNestedGroups - attr_reader :user - - # user - The User object for which to calculate the authorizations. - def initialize(user) - @user = user - end - - def calculate - cte = recursive_cte - cte_alias = cte.table.alias(Group.table_name) - projects = Project.arel_table - links = ProjectGroupLink.arel_table - - relations = [ - # The project a user has direct access to. - user.projects.select_for_project_authorization, - - # The personal projects of the user. - user.personal_projects.select_as_maintainer_for_project_authorization, - - # Projects that belong directly to any of the groups the user has - # access to. - Namespace - .unscoped - .select([alias_as_column(projects[:id], 'project_id'), - cte_alias[:access_level]]) - .from(cte_alias) - .joins(:projects), - - # Projects shared with any of the namespaces the user has access to. - Namespace - .unscoped - .select([links[:project_id], - least(cte_alias[:access_level], - links[:group_access], - 'access_level')]) - .from(cte_alias) - .joins('INNER JOIN project_group_links ON project_group_links.group_id = namespaces.id') - .joins('INNER JOIN projects ON projects.id = project_group_links.project_id') - .joins('INNER JOIN namespaces p_ns ON p_ns.id = projects.namespace_id') - .where('p_ns.share_with_group_lock IS FALSE') - ] - - ProjectAuthorization - .unscoped - .with - .recursive(cte.to_arel) - .select_from_union(relations) - end - - private - - # Builds a recursive CTE that gets all the groups the current user has - # access to, including any nested groups. - def recursive_cte - cte = Gitlab::SQL::RecursiveCTE.new(:namespaces_cte) - members = Member.arel_table - namespaces = Namespace.arel_table - - # Namespaces the user is a member of. - cte << user.groups - .select([namespaces[:id], members[:access_level]]) - .except(:order) - - # Sub groups of any groups the user is a member of. - cte << Group.select([namespaces[:id], - greatest(members[:access_level], - cte.table[:access_level], 'access_level')]) - .joins(join_cte(cte)) - .joins(join_members) - .except(:order) - - cte - end - - # Builds a LEFT JOIN to join optional memberships onto the CTE. - def join_members - members = Member.arel_table - namespaces = Namespace.arel_table - - cond = members[:source_id] - .eq(namespaces[:id]) - .and(members[:source_type].eq('Namespace')) - .and(members[:requested_at].eq(nil)) - .and(members[:user_id].eq(user.id)) - - Arel::Nodes::OuterJoin.new(members, Arel::Nodes::On.new(cond)) - end - - # Builds an INNER JOIN to join namespaces onto the CTE. - def join_cte(cte) - namespaces = Namespace.arel_table - cond = cte.table[:id].eq(namespaces[:parent_id]) - - Arel::Nodes::InnerJoin.new(cte.table, Arel::Nodes::On.new(cond)) - end - - def greatest(left, right, column_alias) - sql_function('GREATEST', [left, right], column_alias) - end - - def least(left, right, column_alias) - sql_function('LEAST', [left, right], column_alias) - end - - def sql_function(name, args, column_alias) - alias_as_column(Arel::Nodes::NamedFunction.new(name, args), column_alias) - end - - def alias_as_column(value, alias_to) - Arel::Nodes::As.new(value, Arel::Nodes::SqlLiteral.new(alias_to)) - end - end - end -end diff --git a/lib/gitlab/project_authorizations/without_nested_groups.rb b/lib/gitlab/project_authorizations/without_nested_groups.rb deleted file mode 100644 index 50b41b17649..00000000000 --- a/lib/gitlab/project_authorizations/without_nested_groups.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ProjectAuthorizations - # Calculating new project authorizations when not supporting nested groups. - class WithoutNestedGroups - attr_reader :user - - # user - The User object for which to calculate the authorizations. - def initialize(user) - @user = user - end - - def calculate - relations = [ - # Projects the user is a direct member of - user.projects.select_for_project_authorization, - - # Personal projects - user.personal_projects.select_as_maintainer_for_project_authorization, - - # Projects of groups the user is a member of - user.groups_projects.select_for_project_authorization, - - # Projects shared with groups the user is a member of - user.groups.joins(:shared_projects).select_for_project_authorization - ] - - ProjectAuthorization - .unscoped - .select_from_union(relations) - end - end - end -end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 0f3b97e2317..2669adb8455 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -29,6 +29,21 @@ module Gitlab end end + def formatted_count(scope) + case scope + when 'blobs' + blobs_count.to_s + when 'notes' + formatted_limited_count(limited_notes_count) + when 'wiki_blobs' + wiki_blobs_count.to_s + when 'commits' + commits_count.to_s + else + super + end + end + def users super.where(id: @project.team.members) # rubocop:disable CodeReuse/ActiveRecord end @@ -108,7 +123,7 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def notes_finder(type) - NotesFinder.new(project, @current_user, search: query, target_type: type).execute.user.order('updated_at DESC') + NotesFinder.new(@current_user, search: query, target_type: type, project: project).execute.user.order('updated_at DESC') end # rubocop: enable CodeReuse/ActiveRecord @@ -134,9 +149,11 @@ module Gitlab project.repository.commit(key) if Commit.valid_hash?(key) end + # rubocop: disable CodeReuse/ActiveRecord def project_ids_relation - project + Project.where(id: project).select(:id).reorder(nil) end + # rubocop: enabled CodeReuse/ActiveRecord def filter_milestones_by_project(milestones) return Milestone.none unless Ability.allowed?(@current_user, :read_milestone, @project) diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb index 99885be8755..fa1d1203842 100644 --- a/lib/gitlab/project_template.rb +++ b/lib/gitlab/project_template.rb @@ -13,11 +13,23 @@ module Gitlab end def archive_path - Rails.root.join("vendor/project_templates/#{name}.tar.gz") + self.class.archive_directory.join(archive_filename) + end + + def archive_filename + "#{name}.tar.gz" end def clone_url - "https://gitlab.com/gitlab-org/project-templates/#{name}.git" + "#{preview}.git" + end + + def project_path + URI.parse(preview).path.sub(%r{\A/}, '') + end + + def uri_encoded_project_path + ERB::Util.url_encode(project_path) end def ==(other) @@ -54,7 +66,7 @@ module Gitlab end def archive_directory - Rails.root.join("vendor_directory/project_templates") + Rails.root.join("vendor/project_templates") end end end diff --git a/lib/gitlab/prometheus/query_variables.rb b/lib/gitlab/prometheus/query_variables.rb index 9cc21129547..ba2d33ee1c1 100644 --- a/lib/gitlab/prometheus/query_variables.rb +++ b/lib/gitlab/prometheus/query_variables.rb @@ -4,12 +4,9 @@ module Gitlab module Prometheus module QueryVariables def self.call(environment) - deployment_platform = environment.deployment_platform - namespace = deployment_platform&.kubernetes_namespace_for(environment.project) || '' - { ci_environment_slug: environment.slug, - kube_namespace: namespace, + kube_namespace: environment.deployment_namespace || '', environment_filter: %{container_name!="POD",environment="#{environment.slug}"} } end diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb index f13156f898e..9fefffefcde 100644 --- a/lib/gitlab/prometheus_client.rb +++ b/lib/gitlab/prometheus_client.rb @@ -3,6 +3,7 @@ module Gitlab # Helper methods to interact with Prometheus network services & resources class PrometheusClient + include Gitlab::Utils::StrongMemoize Error = Class.new(StandardError) QueryError = Class.new(Gitlab::PrometheusClient::Error) @@ -14,10 +15,17 @@ module Gitlab # Minimal value of the `step` parameter for `query_range` in seconds. QUERY_RANGE_MIN_STEP = 60 - attr_reader :rest_client, :headers + # Key translation between RestClient and Gitlab::HTTP (HTTParty) + RESTCLIENT_GITLAB_HTTP_KEYMAP = { + ssl_cert_store: :cert_store + }.freeze - def initialize(rest_client) - @rest_client = rest_client + attr_reader :api_url, :options + private :api_url, :options + + def initialize(api_url, options = {}) + @api_url = api_url.chomp('/') + @options = options end def ping @@ -27,14 +35,10 @@ module Gitlab def proxy(type, args) path = api_path(type) get(path, args) - rescue RestClient::ExceptionWithResponse => ex - if ex.response - ex.response - else - raise PrometheusClient::Error, "Network connection error" - end - rescue RestClient::Exception - raise PrometheusClient::Error, "Network connection error" + rescue Gitlab::HTTP::ResponseError => ex + raise PrometheusClient::Error, "Network connection error" unless ex.response && ex.response.try(:code) + + handle_response(ex.response) end def query(query, time: Time.now) @@ -78,50 +82,58 @@ module Gitlab private def api_path(type) - ['api', 'v1', type].join('/') + [api_url, 'api', 'v1', type].join('/') end def json_api_get(type, args = {}) path = api_path(type) response = get(path, args) handle_response(response) - rescue RestClient::ExceptionWithResponse => ex - if ex.response - handle_exception_response(ex.response) - else - raise PrometheusClient::Error, "Network connection error" + rescue Gitlab::HTTP::ResponseError => ex + raise PrometheusClient::Error, "Network connection error" unless ex.response && ex.response.try(:code) + + handle_response(ex.response) + end + + def gitlab_http_key(key) + RESTCLIENT_GITLAB_HTTP_KEYMAP[key] || key + end + + def mapped_options + options.keys.map { |k| [gitlab_http_key(k), options[k]] }.to_h + end + + def http_options + strong_memoize(:http_options) do + { follow_redirects: false }.merge(mapped_options) end - rescue RestClient::Exception - raise PrometheusClient::Error, "Network connection error" end def get(path, args) - rest_client[path].get(params: args) + Gitlab::HTTP.get(path, { query: args }.merge(http_options) ) rescue SocketError - raise PrometheusClient::Error, "Can't connect to #{rest_client.url}" + raise PrometheusClient::Error, "Can't connect to #{api_url}" rescue OpenSSL::SSL::SSLError - raise PrometheusClient::Error, "#{rest_client.url} contains invalid SSL data" + raise PrometheusClient::Error, "#{api_url} contains invalid SSL data" rescue Errno::ECONNREFUSED raise PrometheusClient::Error, 'Connection refused' end def handle_response(response) - json_data = parse_json(response.body) - if response.code == 200 && json_data['status'] == 'success' - json_data['data'] || {} - else - raise PrometheusClient::Error, "#{response.code} - #{response.body}" - end - end + response_code = response.try(:code) + response_body = response.try(:body) + + raise PrometheusClient::Error, "#{response_code} - #{response_body}" unless response_code + + json_data = parse_json(response_body) if [200, 400].include?(response_code) - def handle_exception_response(response) - if response.code == 200 && response['status'] == 'success' - response['data'] || {} - elsif response.code == 400 - json_data = parse_json(response.body) + case response_code + when 200 + json_data['data'] if response['status'] == 'success' + when 400 raise PrometheusClient::QueryError, json_data['error'] || 'Bad data received' else - raise PrometheusClient::Error, "#{response.code} - #{response.body}" + raise PrometheusClient::Error, "#{response_code} - #{response_body}" end end diff --git a/lib/gitlab/push_options.rb b/lib/gitlab/push_options.rb index 3137676ba4b..682edfc4259 100644 --- a/lib/gitlab/push_options.rb +++ b/lib/gitlab/push_options.rb @@ -4,7 +4,14 @@ module Gitlab class PushOptions VALID_OPTIONS = HashWithIndifferentAccess.new({ merge_request: { - keys: [:create, :merge_when_pipeline_succeeds, :target] + keys: [ + :create, + :description, + :merge_when_pipeline_succeeds, + :remove_source_branch, + :target, + :title + ] }, ci: { keys: [:skip] diff --git a/lib/gitlab/quick_actions/command_definition.rb b/lib/gitlab/quick_actions/command_definition.rb index 93030fd454e..ebdae139315 100644 --- a/lib/gitlab/quick_actions/command_definition.rb +++ b/lib/gitlab/quick_actions/command_definition.rb @@ -3,8 +3,8 @@ module Gitlab module QuickActions class CommandDefinition - attr_accessor :name, :aliases, :description, :explanation, :params, - :condition_block, :parse_params_block, :action_block, :warning, :types + attr_accessor :name, :aliases, :description, :explanation, :execution_message, + :params, :condition_block, :parse_params_block, :action_block, :warning, :types def initialize(name, attributes = {}) @name = name @@ -13,6 +13,7 @@ module Gitlab @description = attributes[:description] || '' @warning = attributes[:warning] || '' @explanation = attributes[:explanation] || '' + @execution_message = attributes[:execution_message] || '' @params = attributes[:params] || [] @condition_block = attributes[:condition_block] @parse_params_block = attributes[:parse_params_block] @@ -48,13 +49,23 @@ module Gitlab end def execute(context, arg) - return if noop? || !available?(context) + return unless executable?(context) count_commands_executed_in(context) execute_block(action_block, context, arg) end + def execute_message(context, arg) + return unless executable?(context) + + if execution_message.respond_to?(:call) + execute_block(execution_message, context, arg) + else + execution_message + end + end + def to_h(context) desc = description if desc.respond_to?(:call) @@ -77,6 +88,10 @@ module Gitlab private + def executable?(context) + !noop? && available?(context) + end + def count_commands_executed_in(context) return unless context.respond_to?(:commands_executed_count=) diff --git a/lib/gitlab/quick_actions/commit_actions.rb b/lib/gitlab/quick_actions/commit_actions.rb index 1018910e8e9..49f5ddf24eb 100644 --- a/lib/gitlab/quick_actions/commit_actions.rb +++ b/lib/gitlab/quick_actions/commit_actions.rb @@ -16,6 +16,13 @@ module Gitlab _("Tags this commit to %{tag_name}.") % { tag_name: tag_name } end end + execution_message do |tag_name, message| + if message.present? + _("Tagged this commit to %{tag_name} with \"%{message}\".") % { tag_name: tag_name, message: message } + else + _("Tagged this commit to %{tag_name}.") % { tag_name: tag_name } + end + end params 'v1.2.3 <message>' parse_params do |tag_name_and_message| tag_name_and_message.split(' ', 2) diff --git a/lib/gitlab/quick_actions/dsl.rb b/lib/gitlab/quick_actions/dsl.rb index ecb2169151e..5abbd377642 100644 --- a/lib/gitlab/quick_actions/dsl.rb +++ b/lib/gitlab/quick_actions/dsl.rb @@ -66,6 +66,35 @@ module Gitlab @explanation = block_given? ? block : text end + # Allows to provide a message about quick action execution result, success or failure. + # This message is shown after quick action execution and after saving the note. + # + # Example: + # + # execution_message do |arguments| + # "Added label(s) #{arguments.join(' ')}" + # end + # command :command_key do |arguments| + # # Awesome code block + # end + # + # Note: The execution_message won't be executed unless the condition block returns true. + # execution_message block is executed always after the command block has run, + # for this reason if the condition block doesn't return true after the command block has + # run you need to set the @execution_message variable inside the command block instead as + # shown in the following example. + # + # Example using instance variable: + # + # command :command_key do |arguments| + # # Awesome code block + # @execution_message[:command_key] = 'command_key executed successfully' + # end + # + def execution_message(text = '', &block) + @execution_message = block_given? ? block : text + end + # Allows to define type(s) that must be met in order for the command # to be returned by `.command_names` & `.command_definitions`. # @@ -121,10 +150,16 @@ module Gitlab # comment. # It accepts aliases and takes a block. # + # You can also set the @execution_message instance variable, on conflicts with + # execution_message method the instance variable has precedence. + # # Example: # # command :my_command, :alias_for_my_command do |arguments| # # Awesome code block + # @updates[:my_command] = 'foo' + # + # @execution_message[:my_command] = 'my_command executed successfully' # end def command(*command_names, &block) define_command(CommandDefinition, *command_names, &block) @@ -158,6 +193,7 @@ module Gitlab description: @description, warning: @warning, explanation: @explanation, + execution_message: @execution_message, params: @params, condition_block: @condition_block, parse_params_block: @parse_params_block, @@ -173,6 +209,7 @@ module Gitlab @description = nil @explanation = nil + @execution_message = nil @params = nil @condition_block = nil @warning = nil diff --git a/lib/gitlab/quick_actions/issuable_actions.rb b/lib/gitlab/quick_actions/issuable_actions.rb index 572c55efcc2..e5d99ebee35 100644 --- a/lib/gitlab/quick_actions/issuable_actions.rb +++ b/lib/gitlab/quick_actions/issuable_actions.rb @@ -12,10 +12,16 @@ module Gitlab included do # Issue, MergeRequest, Epic: quick actions definitions desc do - "Close this #{quick_action_target.to_ability_name.humanize(capitalize: false)}" + _('Close this %{quick_action_target}') % + { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } end explanation do - "Closes this #{quick_action_target.to_ability_name.humanize(capitalize: false)}." + _('Closes this %{quick_action_target}.') % + { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } + end + execution_message do + _('Closed this %{quick_action_target}.') % + { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } end types Issuable condition do @@ -28,10 +34,16 @@ module Gitlab end desc do - "Reopen this #{quick_action_target.to_ability_name.humanize(capitalize: false)}" + _('Reopen this %{quick_action_target}') % + { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } end explanation do - "Reopens this #{quick_action_target.to_ability_name.humanize(capitalize: false)}." + _('Reopens this %{quick_action_target}.') % + { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } + end + execution_message do + _('Reopened this %{quick_action_target}.') % + { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } end types Issuable condition do @@ -45,7 +57,10 @@ module Gitlab desc _('Change title') explanation do |title_param| - _("Changes the title to \"%{title_param}\".") % { title_param: title_param } + _('Changes the title to "%{title_param}".') % { title_param: title_param } + end + execution_message do |title_param| + _('Changed the title to "%{title_param}".') % { title_param: title_param } end params '<New title>' types Issuable @@ -61,7 +76,10 @@ module Gitlab explanation do |labels_param| labels = find_label_references(labels_param) - "Adds #{labels.join(' ')} #{'label'.pluralize(labels.count)}." if labels.any? + if labels.any? + _("Adds %{labels} %{label_text}.") % + { labels: labels.join(' '), label_text: 'label'.pluralize(labels.count) } + end end params '~label1 ~"label 2"' types Issuable @@ -71,21 +89,15 @@ module Gitlab find_labels.any? end command :label do |labels_param| - label_ids = find_label_ids(labels_param) - - if label_ids.any? - @updates[:add_label_ids] ||= [] - @updates[:add_label_ids] += label_ids - - @updates[:add_label_ids].uniq! - end + run_label_command(labels: find_labels(labels_param), command: :label, updates_key: :add_label_ids) end desc _('Remove all or specific label(s)') explanation do |labels_param = nil| - if labels_param.present? - labels = find_label_references(labels_param) - "Removes #{labels.join(' ')} #{'label'.pluralize(labels.count)}." if labels.any? + label_references = labels_param.present? ? find_label_references(labels_param) : [] + if label_references.any? + _("Removes %{label_references} %{label_text}.") % + { label_references: label_references.join(' '), label_text: 'label'.pluralize(label_references.count) } else _('Removes all labels.') end @@ -99,7 +111,9 @@ module Gitlab end command :unlabel do |labels_param = nil| if labels_param.present? - label_ids = find_label_ids(labels_param) + labels = find_labels(labels_param) + label_ids = labels.map(&:id) + label_references = labels_to_reference(labels, :name) if label_ids.any? @updates[:remove_label_ids] ||= [] @@ -109,7 +123,10 @@ module Gitlab end else @updates[:label_ids] = [] + label_references = [] end + + @execution_message[:unlabel] = remove_label_message(label_references) end desc _('Replace all label(s)') @@ -125,18 +142,12 @@ module Gitlab current_user.can?(:"admin_#{quick_action_target.to_ability_name}", parent) end command :relabel do |labels_param| - label_ids = find_label_ids(labels_param) - - if label_ids.any? - @updates[:label_ids] ||= [] - @updates[:label_ids] += label_ids - - @updates[:label_ids].uniq! - end + run_label_command(labels: find_labels(labels_param), command: :relabel, updates_key: :label_ids) end - desc _('Add a todo') - explanation _('Adds a todo.') + desc _('Add a To Do') + explanation _('Adds a To Do.') + execution_message _('Added a To Do.') types Issuable condition do quick_action_target.persisted? && @@ -146,8 +157,9 @@ module Gitlab @updates[:todo_event] = 'add' end - desc _('Mark todo as done') - explanation _('Marks todo as done.') + desc _('Mark To Do as done') + explanation _('Marks To Do as done.') + execution_message _('Marked To Do as done.') types Issuable condition do quick_action_target.persisted? && @@ -159,7 +171,12 @@ module Gitlab desc _('Subscribe') explanation do - "Subscribes to this #{quick_action_target.to_ability_name.humanize(capitalize: false)}." + _('Subscribes to this %{quick_action_target}.') % + { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } + end + execution_message do + _('Subscribed to this %{quick_action_target}.') % + { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } end types Issuable condition do @@ -172,7 +189,12 @@ module Gitlab desc _('Unsubscribe') explanation do - "Unsubscribes from this #{quick_action_target.to_ability_name.humanize(capitalize: false)}." + _('Unsubscribes from this %{quick_action_target}.') % + { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } + end + execution_message do + _('Unsubscribed from this %{quick_action_target}.') % + { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } end types Issuable condition do @@ -187,6 +209,9 @@ module Gitlab explanation do |name| _("Toggles :%{name}: emoji award.") % { name: name } if name end + execution_message do |name| + _("Toggled :%{name}: emoji award.") % { name: name } if name + end params ':emoji:' types Issuable condition do @@ -215,6 +240,41 @@ module Gitlab substitution :tableflip do |comment| "#{comment} #{TABLEFLIP}" end + + private + + def run_label_command(labels:, command:, updates_key:) + return if labels.empty? + + @updates[updates_key] ||= [] + @updates[updates_key] += labels.map(&:id) + @updates[updates_key].uniq! + + label_references = labels_to_reference(labels, :name) + @execution_message[command] = case command + when :relabel + _('Replaced all labels with %{label_references} %{label_text}.') % + { + label_references: label_references.join(' '), + label_text: 'label'.pluralize(label_references.count) + } + when :label + _('Added %{label_references} %{label_text}.') % + { + label_references: label_references.join(' '), + label_text: 'label'.pluralize(labels.count) + } + end + end + + def remove_label_message(label_references) + if label_references.any? + _("Removed %{label_references} %{label_text}.") % + { label_references: label_references.join(' '), label_text: 'label'.pluralize(label_references.count) } + else + _('Removed all labels.') + end + end end end end diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb index 85e62f950c8..da28fbf5be0 100644 --- a/lib/gitlab/quick_actions/issue_actions.rb +++ b/lib/gitlab/quick_actions/issue_actions.rb @@ -12,6 +12,9 @@ module Gitlab explanation do |due_date| _("Sets the due date to %{due_date}.") % { due_date: due_date.strftime('%b %-d, %Y') } if due_date end + execution_message do |due_date| + _("Set the due date to %{due_date}.") % { due_date: due_date.strftime('%b %-d, %Y') } if due_date + end params '<in 2 days | this Friday | December 31st>' types Issue condition do @@ -27,6 +30,7 @@ module Gitlab desc _('Remove due date') explanation _('Removes the due date.') + execution_message _('Removed the due date.') types Issue condition do quick_action_target.persisted? && @@ -49,22 +53,26 @@ module Gitlab current_user.can?(:"update_#{quick_action_target.to_ability_name}", quick_action_target) && quick_action_target.project.boards.count == 1 end - # rubocop: disable CodeReuse/ActiveRecord command :board_move do |target_list_name| - label_ids = find_label_ids(target_list_name) + labels = find_labels(target_list_name) + label_ids = labels.map(&:id) - if label_ids.size == 1 + if label_ids.size > 1 + message = _('Failed to move this issue because only a single label can be provided.') + elsif !Label.on_project_board?(quick_action_target.project_id, label_ids.first) + message = _('Failed to move this issue because label was not found.') + else label_id = label_ids.first - # Ensure this label corresponds to a list on the board - next unless Label.on_project_boards(quick_action_target.project_id).where(id: label_id).exists? - @updates[:remove_label_ids] = - quick_action_target.labels.on_project_boards(quick_action_target.project_id).where.not(id: label_id).pluck(:id) + quick_action_target.labels.on_project_boards(quick_action_target.project_id).where.not(id: label_id).pluck(:id) # rubocop: disable CodeReuse/ActiveRecord @updates[:add_label_ids] = [label_id] + + message = _("Moved issue to %{label} column in the board.") % { label: labels_to_reference(labels).first } end + + @execution_message[:board_move] = message end - # rubocop: enable CodeReuse/ActiveRecord desc _('Mark this issue as a duplicate of another issue') explanation do |duplicate_reference| @@ -81,7 +89,13 @@ module Gitlab if canonical_issue.present? @updates[:canonical_issue_id] = canonical_issue.id + + message = _("Marked this issue as a duplicate of %{duplicate_param}.") % { duplicate_param: duplicate_param } + else + message = _('Failed to mark this issue as a duplicate because referenced issue was not found.') end + + @execution_message[:duplicate] = message end desc _('Move this issue to another project.') @@ -99,12 +113,21 @@ module Gitlab if target_project.present? @updates[:target_project] = target_project + + message = _("Moved this issue to %{path_to_project}.") % { path_to_project: target_project_path } + else + message = _("Failed to move this issue because target project doesn't exist.") end + + @execution_message[:move] = message end - desc _('Make issue confidential.') + desc _('Make issue confidential') explanation do - _('Makes this issue confidential') + _('Makes this issue confidential.') + end + execution_message do + _('Made this issue confidential.') end types Issue condition do @@ -114,12 +137,19 @@ module Gitlab @updates[:confidential] = true end - desc _('Create a merge request.') + desc _('Create a merge request') explanation do |branch_name = nil| if branch_name - _("Creates branch '%{branch_name}' and a merge request to resolve this issue") % { branch_name: branch_name } + _("Creates branch '%{branch_name}' and a merge request to resolve this issue.") % { branch_name: branch_name } + else + _('Creates a branch and a merge request to resolve this issue.') + end + end + execution_message do |branch_name = nil| + if branch_name + _("Created branch '%{branch_name}' and a merge request to resolve this issue.") % { branch_name: branch_name } else - "Creates a branch and a merge request to resolve this issue" + _('Created a branch and a merge request to resolve this issue.') end end params "<branch name>" diff --git a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb index e1579cfddc0..533c74ba9b4 100644 --- a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb +++ b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb @@ -9,12 +9,9 @@ module Gitlab included do # Issue, MergeRequest: quick actions definitions desc _('Assign') - # rubocop: disable CodeReuse/ActiveRecord explanation do |users| - users = quick_action_target.allows_multiple_assignees? ? users : users.take(1) - "Assigns #{users.map(&:to_reference).to_sentence}." + _('Assigns %{assignee_users_sentence}.') % { assignee_users_sentence: assignee_users_sentence(users) } end - # rubocop: enable CodeReuse/ActiveRecord params do quick_action_target.allows_multiple_assignees? ? '@user1 @user2' : '@user' end @@ -26,7 +23,10 @@ module Gitlab extract_users(assignee_param) end command :assign do |users| - next if users.empty? + if users.empty? + @execution_message[:assign] = _("Failed to assign a user because no user was found.") + next + end if quick_action_target.allows_multiple_assignees? @updates[:assignee_ids] ||= quick_action_target.assignees.map(&:id) @@ -34,6 +34,8 @@ module Gitlab else @updates[:assignee_ids] = [users.first.id] end + + @execution_message[:assign] = _('Assigned %{assignee_users_sentence}.') % { assignee_users_sentence: assignee_users_sentence(users) } end desc do @@ -44,9 +46,14 @@ module Gitlab end end explanation do |users = nil| - assignees = quick_action_target.assignees - assignees &= users if users.present? && quick_action_target.allows_multiple_assignees? - "Removes #{'assignee'.pluralize(assignees.size)} #{assignees.map(&:to_reference).to_sentence}." + assignees = assignees_for_removal(users) + _("Removes %{assignee_text} %{assignee_references}.") % + { assignee_text: 'assignee'.pluralize(assignees.size), assignee_references: assignees.map(&:to_reference).to_sentence } + end + execution_message do |users = nil| + assignees = assignees_for_removal(users) + _("Removed %{assignee_text} %{assignee_references}.") % + { assignee_text: 'assignee'.pluralize(assignees.size), assignee_references: assignees.map(&:to_reference).to_sentence } end params do quick_action_target.allows_multiple_assignees? ? '@user1 @user2' : '' @@ -74,6 +81,9 @@ module Gitlab explanation do |milestone| _("Sets the milestone to %{milestone_reference}.") % { milestone_reference: milestone.to_reference } if milestone end + execution_message do |milestone| + _("Set the milestone to %{milestone_reference}.") % { milestone_reference: milestone.to_reference } if milestone + end params '%"milestone"' types Issue, MergeRequest condition do @@ -92,6 +102,9 @@ module Gitlab explanation do _("Removes %{milestone_reference} milestone.") % { milestone_reference: quick_action_target.milestone.to_reference(format: :name) } end + execution_message do + _("Removed %{milestone_reference} milestone.") % { milestone_reference: quick_action_target.milestone.to_reference(format: :name) } + end types Issue, MergeRequest condition do quick_action_target.persisted? && @@ -116,17 +129,22 @@ module Gitlab extract_references(issuable_param, :merge_request).first end command :copy_metadata do |source_issuable| - if source_issuable.present? && source_issuable.project.id == quick_action_target.project.id + if can_copy_metadata?(source_issuable) @updates[:add_label_ids] = source_issuable.labels.map(&:id) @updates[:milestone_id] = source_issuable.milestone.id if source_issuable.milestone + + @execution_message[:copy_metadata] = _("Copied labels and milestone from %{source_issuable_reference}.") % { source_issuable_reference: source_issuable.to_reference } end end desc _('Set time estimate') explanation do |time_estimate| - time_estimate = Gitlab::TimeTrackingFormatter.output(time_estimate) - - _("Sets time estimate to %{time_estimate}.") % { time_estimate: time_estimate } if time_estimate + formatted_time_estimate = format_time_estimate(time_estimate) + _("Sets time estimate to %{time_estimate}.") % { time_estimate: formatted_time_estimate } if formatted_time_estimate + end + execution_message do |time_estimate| + formatted_time_estimate = format_time_estimate(time_estimate) + _("Set time estimate to %{time_estimate}.") % { time_estimate: formatted_time_estimate } if formatted_time_estimate end params '<1w 3d 2h 14m>' types Issue, MergeRequest @@ -144,18 +162,12 @@ module Gitlab desc _('Add or subtract spent time') explanation do |time_spent, time_spent_date| - if time_spent - if time_spent > 0 - verb = _('Adds') - value = time_spent - else - verb = _('Subtracts') - value = -time_spent - end - - _("%{verb} %{time_spent_value} spent time.") % { verb: verb, time_spent_value: Gitlab::TimeTrackingFormatter.output(value) } - end + spend_time_message(time_spent, time_spent_date, false) end + execution_message do |time_spent, time_spent_date| + spend_time_message(time_spent, time_spent_date, true) + end + params '<time(1h30m | -1h30m)> <date(YYYY-MM-DD)>' types Issue, MergeRequest condition do @@ -176,6 +188,7 @@ module Gitlab desc _('Remove time estimate') explanation _('Removes time estimate.') + execution_message _('Removed time estimate.') types Issue, MergeRequest condition do quick_action_target.persisted? && @@ -187,6 +200,7 @@ module Gitlab desc _('Remove spent time') explanation _('Removes spent time.') + execution_message _('Removed spent time.') condition do quick_action_target.persisted? && current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) @@ -197,7 +211,8 @@ module Gitlab end desc _("Lock the discussion") - explanation _("Locks the discussion") + explanation _("Locks the discussion.") + execution_message _("Locked the discussion.") types Issue, MergeRequest condition do quick_action_target.persisted? && @@ -209,7 +224,8 @@ module Gitlab end desc _("Unlock the discussion") - explanation _("Unlocks the discussion") + explanation _("Unlocks the discussion.") + execution_message _("Unlocked the discussion.") types Issue, MergeRequest condition do quick_action_target.persisted? && @@ -219,6 +235,47 @@ module Gitlab command :unlock do @updates[:discussion_locked] = false end + + private + + def assignee_users_sentence(users) + if quick_action_target.allows_multiple_assignees? + users + else + [users.first] + end.map(&:to_reference).to_sentence + end + + def assignees_for_removal(users) + assignees = quick_action_target.assignees + if users.present? && quick_action_target.allows_multiple_assignees? + assignees & users + else + assignees + end + end + + def can_copy_metadata?(source_issuable) + source_issuable.present? && source_issuable.project_id == quick_action_target.project_id + end + + def format_time_estimate(time_estimate) + Gitlab::TimeTrackingFormatter.output(time_estimate) + end + + def spend_time_message(time_spent, time_spent_date, paste_tense) + return unless time_spent + + if time_spent > 0 + verb = paste_tense ? _('Added') : _('Adds') + value = time_spent + else + verb = paste_tense ? _('Subtracted') : _('Subtracts') + value = -time_spent + end + + _("%{verb} %{time_spent_value} spent time.") % { verb: verb, time_spent_value: format_time_estimate(value) } + end end end end diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb index bade59182a1..e9127095a0d 100644 --- a/lib/gitlab/quick_actions/merge_request_actions.rb +++ b/lib/gitlab/quick_actions/merge_request_actions.rb @@ -8,8 +8,9 @@ module Gitlab included do # MergeRequest only quick actions definitions - desc 'Merge (when the pipeline succeeds)' - explanation 'Merges this merge request when the pipeline succeeds.' + desc _('Merge (when the pipeline succeeds)') + explanation _('Merges this merge request when the pipeline succeeds.') + execution_message _('Scheduled to merge this merge request when the pipeline succeeds.') types MergeRequest condition do last_diff_sha = params && params[:merge_request_diff_head_sha] @@ -22,10 +23,22 @@ module Gitlab desc 'Toggle the Work In Progress status' explanation do - verb = quick_action_target.work_in_progress? ? 'Unmarks' : 'Marks' noun = quick_action_target.to_ability_name.humanize(capitalize: false) - "#{verb} this #{noun} as Work In Progress." + if quick_action_target.work_in_progress? + _("Unmarks this %{noun} as Work In Progress.") + else + _("Marks this %{noun} as Work In Progress.") + end % { noun: noun } end + execution_message do + noun = quick_action_target.to_ability_name.humanize(capitalize: false) + if quick_action_target.work_in_progress? + _("Unmarked this %{noun} as Work In Progress.") + else + _("Marked this %{noun} as Work In Progress.") + end % { noun: noun } + end + types MergeRequest condition do quick_action_target.respond_to?(:work_in_progress?) && @@ -36,9 +49,12 @@ module Gitlab @updates[:wip_event] = quick_action_target.work_in_progress? ? 'unwip' : 'wip' end - desc 'Set target branch' + desc _('Set target branch') explanation do |branch_name| - "Sets target branch to #{branch_name}." + _('Sets target branch to %{branch_name}.') % { branch_name: branch_name } + end + execution_message do |branch_name| + _('Set target branch to %{branch_name}.') % { branch_name: branch_name } end params '<Local branch name>' types MergeRequest diff --git a/lib/gitlab/reference_counter.rb b/lib/gitlab/reference_counter.rb index d2dbc6f5ef5..1c43de35816 100644 --- a/lib/gitlab/reference_counter.rb +++ b/lib/gitlab/reference_counter.rb @@ -22,6 +22,7 @@ module Gitlab end end + # rubocop:disable Gitlab/RailsLogger def decrease redis_cmd do |redis| current_value = redis.decr(key) @@ -32,6 +33,7 @@ module Gitlab end end end + # rubocop:enable Gitlab/RailsLogger private @@ -39,7 +41,7 @@ module Gitlab Gitlab::Redis::SharedState.with { |redis| yield(redis) } true rescue => e - Rails.logger.warn("GitLab: An unexpected error occurred in writing to Redis: #{e}") + Rails.logger.warn("GitLab: An unexpected error occurred in writing to Redis: #{e}") # rubocop:disable Gitlab/RailsLogger false end end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 7a1a2eaf6c0..e6372a42dda 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -4,14 +4,6 @@ module Gitlab module Regex extend self - def namespace_name_regex - @namespace_name_regex ||= /\A[\p{Alnum}\p{Pd}_\. ]*\z/.freeze - end - - def namespace_name_regex_message - "can contain only letters, digits, '_', '.', dash and space." - end - def project_name_regex @project_name_regex ||= /\A[\p{Alnum}\u{00A9}-\u{1f9c0}_][\p{Alnum}\p{Pd}\u{00A9}-\u{1f9c0}_\. ]*\z/.freeze end @@ -54,6 +46,18 @@ module Gitlab "can contain only letters, digits, '-', '_', '/', '$', '{', '}', '.', and spaces, but it cannot start or end with '/'" end + def environment_scope_regex_chars + "#{environment_name_regex_chars}\\*" + end + + def environment_scope_regex + @environment_scope_regex ||= /\A[#{environment_scope_regex_chars}]+\z/.freeze + end + + def environment_scope_regex_message + "can contain only letters, digits, '-', '_', '/', '$', '{', '}', '.', '*' and spaces" + end + def kubernetes_namespace_regex /\A[a-z0-9]([-a-z0-9]*[a-z0-9])?\z/ end @@ -102,6 +106,12 @@ module Gitlab }mx end + # Based on Jira's project key format + # https://confluence.atlassian.com/adminjiraserver073/changing-the-project-key-format-861253229.html + def jira_issue_key_regex + @jira_issue_key_regex ||= /[A-Z][A-Z_0-9]+-\d+/ + end + def jira_transition_id_regex @jira_transition_id_regex ||= /\d+/ end diff --git a/lib/gitlab/repository_cache_adapter.rb b/lib/gitlab/repository_cache_adapter.rb index 931298b5117..75503ee1789 100644 --- a/lib/gitlab/repository_cache_adapter.rb +++ b/lib/gitlab/repository_cache_adapter.rb @@ -23,6 +23,37 @@ module Gitlab end end + # Caches and strongly memoizes the method as a Redis Set. + # + # This only works for methods that do not take any arguments. The method + # should return an Array of Strings to be cached. + # + # In addition to overriding the named method, a "name_include?" method is + # defined. This uses the "SISMEMBER" query to efficiently check membership + # without needing to load the entire set into memory. + # + # name - The name of the method to be cached. + # fallback - A value to fall back to if the repository does not exist, or + # in case of a Git error. Defaults to nil. + def cache_method_as_redis_set(name, fallback: nil) + uncached_name = alias_uncached_method(name) + + define_method(name) do + cache_method_output_as_redis_set(name, fallback: fallback) do + __send__(uncached_name) # rubocop:disable GitlabSecurity/PublicSend + end + end + + define_method("#{name}_include?") do |value| + # If the cache isn't populated, we can't rely on it + return redis_set_cache.include?(name, value) if redis_set_cache.exist?(name) + + # Since we have to pull all branch names to populate the cache, use + # the data we already have to answer the query just this once + __send__(name).include?(value) # rubocop:disable GitlabSecurity/PublicSend + end + end + # Caches truthy values from the method. All values are strongly memoized, # and cached in RequestStore. # @@ -84,6 +115,11 @@ module Gitlab raise NotImplementedError end + # RepositorySetCache to be used. Should be overridden by the including class + def redis_set_cache + raise NotImplementedError + end + # List of cached methods. Should be overridden by the including class def cached_methods raise NotImplementedError @@ -100,6 +136,18 @@ module Gitlab end end + # Caches and strongly memoizes the supplied block as a Redis Set. The result + # will be provided as a sorted array. + # + # name - The name of the method to be cached. + # fallback - A value to fall back to if the repository does not exist, or + # in case of a Git error. Defaults to nil. + def cache_method_output_as_redis_set(name, fallback: nil, &block) + memoize_method_output(name, fallback: fallback) do + redis_set_cache.fetch(name, &block).sort + end + end + # Caches truthy values from the supplied block. All values are strongly # memoized, and cached in RequestStore. # @@ -145,7 +193,7 @@ module Gitlab def expire_method_caches(methods) methods.each do |name| unless cached_methods.include?(name.to_sym) - Rails.logger.error "Requested to expire non-existent method '#{name}' for Repository" + Rails.logger.error "Requested to expire non-existent method '#{name}' for Repository" # rubocop:disable Gitlab/RailsLogger next end @@ -154,6 +202,7 @@ module Gitlab clear_memoization(memoizable_name(name)) end + expire_redis_set_method_caches(methods) expire_request_store_method_caches(methods) end @@ -169,6 +218,10 @@ module Gitlab end end + def expire_redis_set_method_caches(methods) + methods.each { |name| redis_set_cache.expire(name) } + end + # All cached repository methods depend on the existence of a Git repository, # so if the repository doesn't exist, we already know not to call it. def fallback_early?(method_name) diff --git a/lib/gitlab/repository_set_cache.rb b/lib/gitlab/repository_set_cache.rb new file mode 100644 index 00000000000..fb634328a95 --- /dev/null +++ b/lib/gitlab/repository_set_cache.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +# Interface to the Redis-backed cache store for keys that use a Redis set +module Gitlab + class RepositorySetCache + attr_reader :repository, :namespace, :expires_in + + def initialize(repository, extra_namespace: nil, expires_in: 2.weeks) + @repository = repository + @namespace = "#{repository.full_path}:#{repository.project.id}" + @namespace = "#{@namespace}:#{extra_namespace}" if extra_namespace + @expires_in = expires_in + end + + def cache_key(type) + [type, namespace, 'set'].join(':') + end + + def expire(key) + with { |redis| redis.del(cache_key(key)) } + end + + def exist?(key) + with { |redis| redis.exists(cache_key(key)) } + end + + def read(key) + with { |redis| redis.smembers(cache_key(key)) } + end + + def write(key, value) + full_key = cache_key(key) + + with do |redis| + redis.multi do + redis.del(full_key) + + # Splitting into groups of 1000 prevents us from creating a too-long + # Redis command + value.in_groups_of(1000, false) { |subset| redis.sadd(full_key, subset) } + + redis.expire(full_key, expires_in) + end + end + + value + end + + def fetch(key, &block) + if exist?(key) + read(key) + else + write(key, yield) + end + end + + def include?(key, value) + with { |redis| redis.sismember(cache_key(key), value) } + end + + private + + def with(&blk) + Gitlab::Redis::Cache.with(&blk) # rubocop:disable CodeReuse/ActiveRecord + end + end +end diff --git a/lib/gitlab/request_profiler.rb b/lib/gitlab/request_profiler.rb index 64593153686..033e451dbee 100644 --- a/lib/gitlab/request_profiler.rb +++ b/lib/gitlab/request_profiler.rb @@ -6,6 +6,21 @@ module Gitlab module RequestProfiler PROFILES_DIR = "#{Gitlab.config.shared.path}/tmp/requests_profiles".freeze + def all + Dir["#{PROFILES_DIR}/*.{html,txt}"].map do |path| + Profile.new(File.basename(path)) + end.select(&:valid?) + end + module_function :all + + def find(name) + file_path = File.join(PROFILES_DIR, name) + return unless File.exist?(file_path) + + Profile.new(name) + end + module_function :find + def profile_token Rails.cache.fetch('profile-token') do Devise.friendly_token diff --git a/lib/gitlab/request_profiler/middleware.rb b/lib/gitlab/request_profiler/middleware.rb index 7615f6f443b..99958d7a211 100644 --- a/lib/gitlab/request_profiler/middleware.rb +++ b/lib/gitlab/request_profiler/middleware.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'ruby-prof' +require 'memory_profiler' module Gitlab module RequestProfiler @@ -28,22 +29,73 @@ module Gitlab end def call_with_profiling(env) + case env['HTTP_X_PROFILE_MODE'] + when 'execution', nil + call_with_call_stack_profiling(env) + when 'memory' + call_with_memory_profiling(env) + else + raise ActionController::BadRequest, invalid_profile_mode(env) + end + end + + def invalid_profile_mode(env) + <<~HEREDOC + Invalid X-Profile-Mode: #{env['HTTP_X_PROFILE_MODE']}. + Supported profile mode request header: + - X-Profile-Mode: execution + - X-Profile-Mode: memory + HEREDOC + end + + def call_with_call_stack_profiling(env) ret = nil - result = RubyProf::Profile.profile do + report = RubyProf::Profile.profile do ret = catch(:warden) do @app.call(env) end end - printer = RubyProf::CallStackPrinter.new(result) - file_name = "#{env['PATH_INFO'].tr('/', '|')}_#{Time.current.to_i}.html" + generate_report(env, 'execution', 'html') do |file| + printer = RubyProf::CallStackPrinter.new(report) + printer.print(file) + end + + handle_request_ret(ret) + end + + def call_with_memory_profiling(env) + ret = nil + report = MemoryProfiler.report do + ret = catch(:warden) do + @app.call(env) + end + end + + generate_report(env, 'memory', 'txt') do |file| + report.pretty_print(to_file: file) + end + + handle_request_ret(ret) + end + + def generate_report(env, report_type, extension) + file_name = "#{env['PATH_INFO'].tr('/', '|')}_#{Time.current.to_i}"\ + "_#{report_type}.#{extension}" file_path = "#{PROFILES_DIR}/#{file_name}" FileUtils.mkdir_p(PROFILES_DIR) - File.open(file_path, 'wb') do |file| - printer.print(file) + + begin + File.open(file_path, 'wb') do |file| + yield(file) + end + rescue + FileUtils.rm(file_path) end + end + def handle_request_ret(ret) if ret.is_a?(Array) ret else diff --git a/lib/gitlab/request_profiler/profile.rb b/lib/gitlab/request_profiler/profile.rb index 46996ef8c51..76c675658b1 100644 --- a/lib/gitlab/request_profiler/profile.rb +++ b/lib/gitlab/request_profiler/profile.rb @@ -3,42 +3,40 @@ module Gitlab module RequestProfiler class Profile - attr_reader :name, :time, :request_path + attr_reader :name, :time, :file_path, :request_path, :profile_mode, :type alias_method :to_param, :name - def self.all - Dir["#{PROFILES_DIR}/*.html"].map do |path| - new(File.basename(path)) - end - end - - def self.find(name) - name_dup = name.dup - name_dup << '.html' unless name.end_with?('.html') - - file_path = "#{PROFILES_DIR}/#{name_dup}" - return unless File.exist?(file_path) - - new(name_dup) - end - def initialize(name) @name = name + @file_path = File.join(PROFILES_DIR, name) set_attributes end - def content - File.read("#{PROFILES_DIR}/#{name}") + def valid? + @request_path.present? + end + + def content_type + case type + when 'html' + 'text/html' + when 'txt' + 'text/plain' + end end private def set_attributes - _, path, timestamp = name.split(/(.*)_(\d+)\.html$/) - @request_path = path.tr('|', '/') - @time = Time.at(timestamp.to_i).utc + matches = name.match(/^(?<path>.*)_(?<timestamp>\d+)(_(?<profile_mode>\w+))?\.(?<type>html|txt)$/) + return unless matches + + @request_path = matches[:path].tr('|', '/') + @time = Time.at(matches[:timestamp].to_i).utc + @profile_mode = matches[:profile_mode] || 'unknown' + @type = matches[:type] end end end diff --git a/lib/gitlab/rugged_instrumentation.rb b/lib/gitlab/rugged_instrumentation.rb new file mode 100644 index 00000000000..8bb8c547ae1 --- /dev/null +++ b/lib/gitlab/rugged_instrumentation.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module RuggedInstrumentation + def self.query_time + SafeRequestStore[:rugged_query_time] ||= 0 + end + + def self.query_time=(duration) + SafeRequestStore[:rugged_query_time] = duration + end + + def self.query_time_ms + (self.query_time * 1000).round(2) + end + + def self.query_count + SafeRequestStore[:rugged_call_count] ||= 0 + end + + def self.increment_query_count + SafeRequestStore[:rugged_call_count] ||= 0 + SafeRequestStore[:rugged_call_count] += 1 + end + + def self.active? + SafeRequestStore.active? + end + + def self.peek_enabled? + SafeRequestStore[:peek_enabled] + end + + def self.add_call_details(details) + return unless peek_enabled? + + Gitlab::SafeRequestStore[:rugged_call_details] ||= [] + Gitlab::SafeRequestStore[:rugged_call_details] << details + end + + def self.list_call_details + return [] unless peek_enabled? + + Gitlab::SafeRequestStore[:rugged_call_details] || [] + end + end +end diff --git a/lib/gitlab/sanitizers/exif.rb b/lib/gitlab/sanitizers/exif.rb index 0928ccdc324..bb4e4ce7bbc 100644 --- a/lib/gitlab/sanitizers/exif.rb +++ b/lib/gitlab/sanitizers/exif.rb @@ -48,7 +48,7 @@ module Gitlab attr_reader :logger - def initialize(logger: Rails.logger) + def initialize(logger: Rails.logger) # rubocop:disable Gitlab/RailsLogger @logger = logger end diff --git a/lib/gitlab/search/found_blob.rb b/lib/gitlab/search/found_blob.rb index a62ab1521a7..fa09ecbdf30 100644 --- a/lib/gitlab/search/found_blob.rb +++ b/lib/gitlab/search/found_blob.rb @@ -28,7 +28,7 @@ module Gitlab @binary_data = opts.fetch(:data, nil) @per_page = opts.fetch(:per_page, 20) @project = opts.fetch(:project, nil) - # Some caller does not have project object (e.g. elastic search), + # Some callers (e.g. Elasticsearch) do not have the Project object, # yet they can trigger many calls in one go, # causing duplicated queries. # Allow those to just pass project_id instead. @@ -93,7 +93,7 @@ module Gitlab data = { id: blob.id, binary_filename: blob.path, - binary_basename: File.basename(blob.path, File.extname(blob.path)), + binary_basename: path_without_extension(blob.path), ref: ref, startline: 1, binary_data: blob.data, @@ -111,6 +111,10 @@ module Gitlab content_match.match(FILENAME_REGEXP) { |matches| matches[:filename] } end + def path_without_extension(path) + Pathname.new(path).sub_ext('').to_s + end + def parsed_content strong_memoize(:parsed_content) do if content_match @@ -137,8 +141,7 @@ module Gitlab filename = matches[:filename] startline = matches[:startline] startline = startline.to_i - index - extname = Regexp.escape(File.extname(filename)) - basename = filename.sub(/#{extname}$/, '') + basename = path_without_extension(filename) end data << line.sub(prefix.to_s, '') diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 7c1e6b1baff..ce4c1611687 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -43,6 +43,29 @@ module Gitlab without_count ? collection.without_count : collection end + def formatted_count(scope) + case scope + when 'projects' + formatted_limited_count(limited_projects_count) + when 'issues' + formatted_limited_count(limited_issues_count) + when 'merge_requests' + formatted_limited_count(limited_merge_requests_count) + when 'milestones' + formatted_limited_count(limited_milestones_count) + when 'users' + formatted_limited_count(limited_users_count) + end + end + + def formatted_limited_count(count) + if count >= COUNT_LIMIT + "#{COUNT_LIMIT - 1}+" + else + count.to_s + end + end + def limited_projects_count @limited_projects_count ||= limited_count(projects) end diff --git a/lib/gitlab/sentry.rb b/lib/gitlab/sentry.rb index 72c44114001..005cb3112b8 100644 --- a/lib/gitlab/sentry.rb +++ b/lib/gitlab/sentry.rb @@ -4,7 +4,7 @@ module Gitlab module Sentry def self.enabled? (Rails.env.production? || Rails.env.development?) && - Gitlab::CurrentSettings.sentry_enabled? + Gitlab.config.sentry.enabled end def self.context(current_user = nil) @@ -39,9 +39,14 @@ module Gitlab # development and test. If you need development and test to behave # just the same as production you can use this instead of # track_exception. + # + # If the exception implements the method `sentry_extra_data` and that method + # returns a Hash, then the return value of that method will be merged into + # `extra`. Exceptions can use this mechanism to provide structured data + # to sentry in addition to their message and back-trace. def self.track_acceptable_exception(exception, issue_url: nil, extra: {}) if enabled? - extra[:issue_url] = issue_url if issue_url + extra = build_extra_data(exception, issue_url, extra) context # Make sure we've set everything we know in the context Raven.capture_exception(exception, tags: default_tags, extra: extra) @@ -58,5 +63,15 @@ module Gitlab locale: I18n.locale } end + + def self.build_extra_data(exception, issue_url, extra) + exception.try(:sentry_extra_data)&.tap do |data| + extra.merge!(data) if data.is_a?(Hash) + end + + extra.merge({ issue_url: issue_url }.compact) + end + + private_class_method :build_extra_data end end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 93182607616..0fa17b3f559 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -78,7 +78,7 @@ module Gitlab true rescue => err # Once the Rugged codes gets removes this can be improved - Rails.logger.error("Failed to add repository #{storage}/#{disk_path}: #{err}") + Rails.logger.error("Failed to add repository #{storage}/#{disk_path}: #{err}") # rubocop:disable Gitlab/RailsLogger false end @@ -153,7 +153,7 @@ module Gitlab !!rm_directory(storage, "#{name}.git") rescue ArgumentError => e - Rails.logger.warn("Repository does not exist: #{e} at: #{name}.git") + Rails.logger.warn("Repository does not exist: #{e} at: #{name}.git") # rubocop:disable Gitlab/RailsLogger false end @@ -238,7 +238,7 @@ module Gitlab def remove_keys_not_found_in_db return unless self.authorized_keys_enabled? - Rails.logger.info("Removing keys not found in DB") + Rails.logger.info("Removing keys not found in DB") # rubocop:disable Gitlab/RailsLogger batch_read_key_ids do |ids_in_file| ids_in_file.uniq! @@ -248,7 +248,7 @@ module Gitlab ids_to_remove = ids_in_file - keys_in_db.pluck(:id) ids_to_remove.each do |id| - Rails.logger.info("Removing key-#{id} not found in DB") + Rails.logger.info("Removing key-#{id} not found in DB") # rubocop:disable Gitlab/RailsLogger remove_key("key-#{id}") end end @@ -368,7 +368,7 @@ module Gitlab return true if status.zero? - Rails.logger.error("gitlab-shell failed with error #{status}: #{output}") + Rails.logger.error("gitlab-shell failed with error #{status}: #{output}") # rubocop:disable Gitlab/RailsLogger false end @@ -465,7 +465,7 @@ module Gitlab end def logger - Rails.logger + Rails.logger # rubocop:disable Gitlab/RailsLogger end end end diff --git a/lib/gitlab/sherlock/query.rb b/lib/gitlab/sherlock/query.rb index 159ce27e702..cbd89b7629f 100644 --- a/lib/gitlab/sherlock/query.rb +++ b/lib/gitlab/sherlock/query.rb @@ -96,12 +96,7 @@ module Gitlab private def raw_explain(query) - explain = - if Gitlab::Database.postgresql? - "EXPLAIN ANALYZE #{query};" - else - "EXPLAIN #{query};" - end + explain = "EXPLAIN ANALYZE #{query};" ActiveRecord::Base.connection.execute(explain) end diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb index fdc0d518c59..60782306ade 100644 --- a/lib/gitlab/sidekiq_logging/structured_logger.rb +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -15,9 +15,9 @@ module Gitlab yield - Sidekiq.logger.info log_job_done(started_at, base_payload) + Sidekiq.logger.info log_job_done(job, started_at, base_payload) rescue => job_exception - Sidekiq.logger.warn log_job_done(started_at, base_payload, job_exception) + Sidekiq.logger.warn log_job_done(job, started_at, base_payload, job_exception) raise end @@ -28,15 +28,26 @@ module Gitlab "#{payload['class']} JID-#{payload['jid']}" end + def add_instrumentation_keys!(job, output_payload) + output_payload.merge!(job.slice(*::Gitlab::InstrumentationHelper::KEYS)) + end + def log_job_start(started_at, payload) payload['message'] = "#{base_message(payload)}: start" payload['job_status'] = 'start' + # Old gitlab-shell messages don't provide enqueued_at/created_at attributes + enqueued_at = payload['enqueued_at'] || payload['created_at'] + if enqueued_at + payload['scheduling_latency_s'] = elapsed_by_absolute_time(Time.iso8601(enqueued_at)) + end + payload end - def log_job_done(started_at, payload, job_exception = nil) + def log_job_done(job, started_at, payload, job_exception = nil) payload = payload.dup + add_instrumentation_keys!(job, payload) payload['duration'] = elapsed(started_at) payload['completed_at'] = Time.now.utc @@ -78,6 +89,10 @@ module Gitlab end end + def elapsed_by_absolute_time(start) + (Time.now.utc - start).to_f.round(3) + end + def elapsed(start) (current_time - start).round(3) end diff --git a/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb b/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb new file mode 100644 index 00000000000..979a3fce7e6 --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + class InstrumentationLogger + def call(worker, job, queue) + yield + + # The Sidekiq logger is called outside the middleware block, so + # we need to modify the job hash to pass along this information + # since RequestStore is only active in the Sidekiq middleware. + # + # Modifying the job hash in a middleware is permitted by Sidekiq + # because Sidekiq keeps a pristine copy of the original hash + # before sending it to the middleware: + # https://github.com/mperham/sidekiq/blob/53bd529a0c3f901879925b8390353129c465b1f2/lib/sidekiq/processor.rb#L115-L118 + ::Gitlab::InstrumentationHelper.add_instrumentation_data(job) + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb index 671d795ec33..49c4fdc3033 100644 --- a/lib/gitlab/sidekiq_middleware/memory_killer.rb +++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb @@ -14,9 +14,12 @@ module Gitlab # shut Sidekiq down MUTEX = Mutex.new + attr_reader :worker + def call(worker, job, queue) yield + @worker = worker current_rss = get_rss return unless MAX_RSS > 0 && current_rss > MAX_RSS @@ -25,9 +28,11 @@ module Gitlab # Return if another thread is already waiting to shut Sidekiq down next unless MUTEX.try_lock - Sidekiq.logger.warn "Sidekiq worker PID-#{pid} current RSS #{current_rss}"\ - " exceeds maximum RSS #{MAX_RSS} after finishing job #{worker.class} JID-#{job['jid']}" - Sidekiq.logger.warn "Sidekiq worker PID-#{pid} will stop fetching new jobs in #{GRACE_TIME} seconds, and will be shut down #{SHUTDOWN_WAIT} seconds later" + warn("Sidekiq worker PID-#{pid} current RSS #{current_rss}"\ + " exceeds maximum RSS #{MAX_RSS} after finishing job #{worker.class} JID-#{job['jid']}") + + warn("Sidekiq worker PID-#{pid} will stop fetching new jobs"\ + " in #{GRACE_TIME} seconds, and will be shut down #{SHUTDOWN_WAIT} seconds later") # Wait `GRACE_TIME` to give the memory intensive job time to finish. # Then, tell Sidekiq to stop fetching new jobs. @@ -59,24 +64,28 @@ module Gitlab def wait_and_signal_pgroup(time, signal, explanation) return wait_and_signal(time, signal, explanation) unless Process.getpgrp == pid - Sidekiq.logger.warn "waiting #{time} seconds before sending Sidekiq worker PGRP-#{pid} #{signal} (#{explanation})" + warn("waiting #{time} seconds before sending Sidekiq worker PGRP-#{pid} #{signal} (#{explanation})", signal: signal) sleep(time) - Sidekiq.logger.warn "sending Sidekiq worker PGRP-#{pid} #{signal} (#{explanation})" + warn("sending Sidekiq worker PGRP-#{pid} #{signal} (#{explanation})", signal: signal) Process.kill(signal, 0) end def wait_and_signal(time, signal, explanation) - Sidekiq.logger.warn "waiting #{time} seconds before sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" + warn("waiting #{time} seconds before sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})", signal: signal) sleep(time) - Sidekiq.logger.warn "sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" + warn("sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})", signal: signal) Process.kill(signal, pid) end def pid Process.pid end + + def warn(message, signal: nil) + Sidekiq.logger.warn(class: worker.class.name, pid: pid, signal: signal, message: message) + end end end end diff --git a/lib/gitlab/sidekiq_middleware/metrics.rb b/lib/gitlab/sidekiq_middleware/metrics.rb new file mode 100644 index 00000000000..3dc9521ee8b --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/metrics.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + class Metrics + # SIDEKIQ_LATENCY_BUCKETS are latency histogram buckets better suited to Sidekiq + # timeframes than the DEFAULT_BUCKET definition. Defined in seconds. + SIDEKIQ_LATENCY_BUCKETS = [0.1, 0.25, 0.5, 1, 2.5, 5, 10, 60, 300, 600].freeze + + def initialize + @metrics = init_metrics + end + + def call(_worker, job, queue) + labels = create_labels(queue) + @metrics[:sidekiq_running_jobs].increment(labels, 1) + + if job['retry_count'].present? + @metrics[:sidekiq_jobs_retried_total].increment(labels, 1) + end + + realtime = Benchmark.realtime do + yield + end + + @metrics[:sidekiq_jobs_completion_seconds].observe(labels, realtime) + rescue Exception # rubocop: disable Lint/RescueException + @metrics[:sidekiq_jobs_failed_total].increment(labels, 1) + raise + ensure + @metrics[:sidekiq_running_jobs].increment(labels, -1) + end + + private + + def init_metrics + { + sidekiq_jobs_completion_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete sidekiq job', buckets: SIDEKIQ_LATENCY_BUCKETS), + sidekiq_jobs_failed_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_failed_total, 'Sidekiq jobs failed'), + sidekiq_jobs_retried_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_retried_total, 'Sidekiq jobs retried'), + sidekiq_running_jobs: ::Gitlab::Metrics.gauge(:sidekiq_running_jobs, 'Number of Sidekiq jobs running', {}, :livesum) + } + end + + def create_labels(queue) + { + queue: queue + } + end + end + end +end diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb index 583a970bf4e..0f890a12134 100644 --- a/lib/gitlab/sidekiq_status.rb +++ b/lib/gitlab/sidekiq_status.rb @@ -53,14 +53,14 @@ module Gitlab self.num_running(job_ids).zero? end - # Returns true if the given job is running + # Returns true if the given job is running or enqueued. # # job_id - The Sidekiq job ID to check. def self.running?(job_id) num_running([job_id]) > 0 end - # Returns the number of jobs that are running. + # Returns the number of jobs that are running or enqueued. # # job_ids - The Sidekiq job IDs to check. def self.num_running(job_ids) @@ -81,7 +81,7 @@ module Gitlab # job_ids - The Sidekiq job IDs to check. # # Returns an array of true or false indicating job completion. - # true = job is still running + # true = job is still running or enqueued # false = job completed def self.job_status(job_ids) keys = job_ids.map { |jid| key_for(jid) } diff --git a/lib/gitlab/slug/environment.rb b/lib/gitlab/slug/environment.rb new file mode 100644 index 00000000000..1b87d3bb626 --- /dev/null +++ b/lib/gitlab/slug/environment.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +# An environment name is not necessarily suitable for use in URLs, DNS +# or other third-party contexts, so provide a slugified version. A slug has +# the following properties: +# * contains only lowercase letters (a-z), numbers (0-9), and '-' +# * begins with a letter +# * has a maximum length of 24 bytes (OpenShift limitation) +# * cannot end with `-` +module Gitlab + module Slug + class Environment + attr_reader :name + + def initialize(name) + @name = name + end + + def generate + # Lowercase letters and numbers only + slugified = name.to_s.downcase.gsub(/[^a-z0-9]/, '-') + + # Must start with a letter + slugified = 'env-' + slugified unless slugified.match?(/^[a-z]/) + + # Repeated dashes are invalid (OpenShift limitation) + slugified.squeeze!('-') + + slugified = + if slugified.size > 24 || slugified != name + # Maximum length: 24 characters (OpenShift limitation) + shorten_and_add_suffix(slugified) + else + # Cannot end with a dash (Kubernetes label limitation) + slugified.chomp('-') + end + + slugified + end + + private + + def shorten_and_add_suffix(slug) + slug = slug[0..16] + slug << '-' unless slug.ends_with?('-') + slug << suffix + end + + # Slugifying a name may remove the uniqueness guarantee afforded by it being + # based on name (which must be unique). To compensate, we add a predictable + # 6-byte suffix in those circumstances. This is not *guaranteed* uniqueness, + # but the chance of collisions is vanishingly small + def suffix + Digest::SHA2.hexdigest(name.to_s).to_i(16).to_s(36).last(6) + end + end + end +end diff --git a/lib/gitlab/snippet_search_results.rb b/lib/gitlab/snippet_search_results.rb index e360b552f89..ac3b219e0c7 100644 --- a/lib/gitlab/snippet_search_results.rb +++ b/lib/gitlab/snippet_search_results.rb @@ -22,6 +22,17 @@ module Gitlab end end + def formatted_count(scope) + case scope + when 'snippet_titles' + snippet_titles_count.to_s + when 'snippet_blobs' + snippet_blobs_count.to_s + else + super + end + end + def snippet_titles_count @snippet_titles_count ||= snippet_titles.count end diff --git a/lib/gitlab/snowplow_tracker.rb b/lib/gitlab/snowplow_tracker.rb new file mode 100644 index 00000000000..9f12513e09e --- /dev/null +++ b/lib/gitlab/snowplow_tracker.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require 'snowplow-tracker' + +module Gitlab + module SnowplowTracker + NAMESPACE = 'cf' + + class << self + def track_event(category, action, label: nil, property: nil, value: nil, context: nil) + tracker&.track_struct_event(category, action, label, property, value, context, Time.now.to_i) + end + + private + + def tracker + return unless enabled? + + @tracker ||= ::SnowplowTracker::Tracker.new(emitter, subject, NAMESPACE, Gitlab::CurrentSettings.snowplow_site_id) + end + + def subject + ::SnowplowTracker::Subject.new + end + + def emitter + ::SnowplowTracker::Emitter.new(Gitlab::CurrentSettings.snowplow_collector_hostname) + end + + def enabled? + Gitlab::CurrentSettings.snowplow_enabled? + end + end + end +end diff --git a/lib/gitlab/sql/pattern.rb b/lib/gitlab/sql/pattern.rb index fd108b4c124..f6edbfced7f 100644 --- a/lib/gitlab/sql/pattern.rb +++ b/lib/gitlab/sql/pattern.rb @@ -9,14 +9,16 @@ module Gitlab REGEX_QUOTED_WORD = /(?<=\A| )"[^"]+"(?= |\z)/.freeze class_methods do - def fuzzy_search(query, columns) - matches = columns.map { |col| fuzzy_arel_match(col, query) }.compact.reduce(:or) + def fuzzy_search(query, columns, use_minimum_char_limit: true) + matches = columns.map do |col| + fuzzy_arel_match(col, query, use_minimum_char_limit: use_minimum_char_limit) + end.compact.reduce(:or) where(matches) end - def to_pattern(query) - if partial_matching?(query) + def to_pattern(query, use_minimum_char_limit: true) + if partial_matching?(query, use_minimum_char_limit: use_minimum_char_limit) "%#{sanitize_sql_like(query)}%" else sanitize_sql_like(query) @@ -27,7 +29,9 @@ module Gitlab MIN_CHARS_FOR_PARTIAL_MATCHING end - def partial_matching?(query) + def partial_matching?(query, use_minimum_char_limit: true) + return true unless use_minimum_char_limit + query.length >= min_chars_for_partial_matching end @@ -35,14 +39,14 @@ module Gitlab # query - The text to search for. # lower_exact_match - When set to `true` we'll fall back to using # `LOWER(column) = query` instead of using `ILIKE`. - def fuzzy_arel_match(column, query, lower_exact_match: false) + def fuzzy_arel_match(column, query, lower_exact_match: false, use_minimum_char_limit: true) query = query.squish return unless query.present? - words = select_fuzzy_words(query) + words = select_fuzzy_words(query, use_minimum_char_limit: use_minimum_char_limit) if words.any? - words.map { |word| arel_table[column].matches(to_pattern(word)) }.reduce(:and) + words.map { |word| arel_table[column].matches(to_pattern(word, use_minimum_char_limit: use_minimum_char_limit)) }.reduce(:and) else # No words of at least 3 chars, but we can search for an exact # case insensitive match with the query as a whole @@ -56,7 +60,7 @@ module Gitlab end end - def select_fuzzy_words(query) + def select_fuzzy_words(query, use_minimum_char_limit: true) quoted_words = query.scan(REGEX_QUOTED_WORD) query = quoted_words.reduce(query) { |q, quoted_word| q.sub(quoted_word, '') } @@ -67,7 +71,7 @@ module Gitlab words.concat(quoted_words) - words.select { |word| partial_matching?(word) } + words.select { |word| partial_matching?(word, use_minimum_char_limit: use_minimum_char_limit) } end end end diff --git a/lib/gitlab/submodule_links.rb b/lib/gitlab/submodule_links.rb new file mode 100644 index 00000000000..18fd604a3b0 --- /dev/null +++ b/lib/gitlab/submodule_links.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + class SubmoduleLinks + include Gitlab::Utils::StrongMemoize + + def initialize(repository) + @repository = repository + end + + def for(submodule, sha) + submodule_url = submodule_url_for(sha, submodule.path) + SubmoduleHelper.submodule_links_for_url(submodule.id, submodule_url, repository) + end + + private + + attr_reader :repository + + def submodule_urls_for(sha) + strong_memoize(:"submodule_urls_for_#{sha}") do + repository.submodule_urls_for(sha) + end + end + + def submodule_url_for(sha, path) + urls = submodule_urls_for(sha) + urls && urls[path] + end + end +end diff --git a/lib/gitlab/thread_memory_cache.rb b/lib/gitlab/thread_memory_cache.rb new file mode 100644 index 00000000000..7f363dc7feb --- /dev/null +++ b/lib/gitlab/thread_memory_cache.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + class ThreadMemoryCache + THREAD_KEY = :thread_memory_cache + + def self.cache_backend + # Note ActiveSupport::Cache::MemoryStore is thread-safe. Since + # each backend is local per thread we probably don't need to worry + # about synchronizing access, but this is a drop-in replacement + # for ActiveSupport::Cache::RedisStore. + Thread.current[THREAD_KEY] ||= ActiveSupport::Cache::MemoryStore.new + end + end +end diff --git a/lib/gitlab/time_tracking_formatter.rb b/lib/gitlab/time_tracking_formatter.rb index cc206010e74..302da91328a 100644 --- a/lib/gitlab/time_tracking_formatter.rb +++ b/lib/gitlab/time_tracking_formatter.rb @@ -6,7 +6,7 @@ module Gitlab def parse(string) with_custom_config do - string.sub!(/\A-/, '') + string = string.sub(/\A-/, '') seconds = ChronicDuration.parse(string, default_unit: 'hours') rescue nil seconds *= -1 if seconds && Regexp.last_match @@ -16,10 +16,12 @@ module Gitlab def output(seconds) with_custom_config do - ChronicDuration.output(seconds, format: :short, limit_to_hours: false, weeks: true) rescue nil + ChronicDuration.output(seconds, format: :short, limit_to_hours: limit_to_hours_setting, weeks: true) rescue nil end end + private + def with_custom_config # We may want to configure it through project settings in a future version. ChronicDuration.hours_per_day = 8 @@ -32,5 +34,9 @@ module Gitlab result end + + def limit_to_hours_setting + Gitlab::CurrentSettings.time_tracking_limit_to_hours + end end end diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index 9a8df719827..9c35d200dcb 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -18,7 +18,6 @@ module Gitlab # enforce_sanitization - Raises error if URL includes any HTML/CSS/JS tags and argument is true. # # Returns an array with [<uri>, <original-hostname>]. - # rubocop:disable Metrics/CyclomaticComplexity # rubocop:disable Metrics/ParameterLists def validate!( url, @@ -30,7 +29,6 @@ module Gitlab enforce_user: false, enforce_sanitization: false, dns_rebind_protection: true) - # rubocop:enable Metrics/CyclomaticComplexity # rubocop:enable Metrics/ParameterLists return [nil, nil] if url.nil? @@ -38,36 +36,34 @@ module Gitlab # Param url can be a string, URI or Addressable::URI uri = parse_url(url) - validate_html_tags!(uri) if enforce_sanitization + validate_uri( + uri: uri, + schemes: schemes, + ports: ports, + enforce_sanitization: enforce_sanitization, + enforce_user: enforce_user, + ascii_only: ascii_only + ) + normalized_hostname = uri.normalized_host hostname = uri.hostname port = get_port(uri) - unless internal?(uri) - validate_scheme!(uri.scheme, schemes) - validate_port!(port, ports) if ports.any? - validate_user!(uri.user) if enforce_user - validate_hostname!(hostname) - validate_unicode_restriction!(uri) if ascii_only - end - - begin - addrs_info = Addrinfo.getaddrinfo(hostname, port, nil, :STREAM).map do |addr| - addr.ipv6_v4mapped? ? addr.ipv6_to_ipv4 : addr - end - rescue SocketError - return [uri, nil] - end + address_info = get_address_info(hostname, port) + return [uri, nil] unless address_info - protected_uri_with_hostname = enforce_uri_hostname(addrs_info, uri, hostname, dns_rebind_protection) + ip_address = ip_address(address_info) + protected_uri_with_hostname = enforce_uri_hostname(ip_address, uri, hostname, dns_rebind_protection) # Allow url from the GitLab instance itself but only for the configured hostname and ports return protected_uri_with_hostname if internal?(uri) - validate_localhost!(addrs_info) unless allow_localhost - validate_loopback!(addrs_info) unless allow_localhost - validate_local_network!(addrs_info) unless allow_local_network - validate_link_local!(addrs_info) unless allow_local_network + validate_local_request( + normalized_hostname: normalized_hostname, + address_info: address_info, + allow_localhost: allow_localhost, + allow_local_network: allow_local_network + ) protected_uri_with_hostname end @@ -90,10 +86,7 @@ module Gitlab # # The original hostname is used to validate the SSL, given in that scenario # we'll be making the request to the IP address, instead of using the hostname. - def enforce_uri_hostname(addrs_info, uri, hostname, dns_rebind_protection) - address = addrs_info.first - ip_address = address&.ip_address - + def enforce_uri_hostname(ip_address, uri, hostname, dns_rebind_protection) return [uri, nil] unless dns_rebind_protection && ip_address && ip_address != hostname uri = uri.dup @@ -101,11 +94,67 @@ module Gitlab [uri, hostname] end + def ip_address(address_info) + address_info.first&.ip_address + end + + def validate_uri(uri:, schemes:, ports:, enforce_sanitization:, enforce_user:, ascii_only:) + validate_html_tags(uri) if enforce_sanitization + + return if internal?(uri) + + validate_scheme(uri.scheme, schemes) + validate_port(get_port(uri), ports) if ports.any? + validate_user(uri.user) if enforce_user + validate_hostname(uri.hostname) + validate_unicode_restriction(uri) if ascii_only + end + + def get_address_info(hostname, port) + Addrinfo.getaddrinfo(hostname, port, nil, :STREAM).map do |addr| + addr.ipv6_v4mapped? ? addr.ipv6_to_ipv4 : addr + end + rescue SocketError + # In the test suite we use a lot of mocked urls that are either invalid or + # don't exist. In order to avoid modifying a ton of tests and factories + # we allow invalid urls unless the environment variable RSPEC_ALLOW_INVALID_URLS + # is not true + return if Rails.env.test? && ENV['RSPEC_ALLOW_INVALID_URLS'] == 'true' + + # If the addr can't be resolved or the url is invalid (i.e http://1.1.1.1.1) + # we block the url + raise BlockedUrlError, "Host cannot be resolved or invalid" + end + + def validate_local_request( + normalized_hostname:, + address_info:, + allow_localhost:, + allow_local_network:) + return if allow_local_network && allow_localhost + + ip_whitelist, domain_whitelist = + Gitlab::CurrentSettings.outbound_local_requests_whitelist_arrays + + return if local_domain_whitelisted?(domain_whitelist, normalized_hostname) || + local_ip_whitelisted?(ip_whitelist, ip_address(address_info)) + + unless allow_localhost + validate_localhost(address_info) + validate_loopback(address_info) + end + + unless allow_local_network + validate_local_network(address_info) + validate_link_local(address_info) + end + end + def get_port(uri) uri.port || uri.default_port end - def validate_html_tags!(uri) + def validate_html_tags(uri) uri_str = uri.to_s sanitized_uri = ActionController::Base.helpers.sanitize(uri_str, tags: []) if sanitized_uri != uri_str @@ -125,7 +174,7 @@ module Gitlab CGI.unescape(url.to_s) =~ /\n|\r/ end - def validate_port!(port, ports) + def validate_port(port, ports) return if port.blank? # Only ports under 1024 are restricted return if port >= 1024 @@ -134,20 +183,20 @@ module Gitlab raise BlockedUrlError, "Only allowed ports are #{ports.join(', ')}, and any over 1024" end - def validate_scheme!(scheme, schemes) + def validate_scheme(scheme, schemes) if scheme.blank? || (schemes.any? && !schemes.include?(scheme)) raise BlockedUrlError, "Only allowed schemes are #{schemes.join(', ')}" end end - def validate_user!(value) + def validate_user(value) return if value.blank? return if value =~ /\A\p{Alnum}/ raise BlockedUrlError, "Username needs to start with an alphanumeric character" end - def validate_hostname!(value) + def validate_hostname(value) return if value.blank? return if IPAddress.valid?(value) return if value =~ /\A\p{Alnum}/ @@ -155,13 +204,13 @@ module Gitlab raise BlockedUrlError, "Hostname or IP address invalid" end - def validate_unicode_restriction!(uri) + def validate_unicode_restriction(uri) return if uri.to_s.ascii_only? raise BlockedUrlError, "URI must be ascii only #{uri.to_s.dump}" end - def validate_localhost!(addrs_info) + def validate_localhost(addrs_info) local_ips = ["::", "0.0.0.0"] local_ips.concat(Socket.ip_address_list.map(&:ip_address)) @@ -170,19 +219,19 @@ module Gitlab raise BlockedUrlError, "Requests to localhost are not allowed" end - def validate_loopback!(addrs_info) + def validate_loopback(addrs_info) return unless addrs_info.any? { |addr| addr.ipv4_loopback? || addr.ipv6_loopback? } raise BlockedUrlError, "Requests to loopback addresses are not allowed" end - def validate_local_network!(addrs_info) + def validate_local_network(addrs_info) return unless addrs_info.any? { |addr| addr.ipv4_private? || addr.ipv6_sitelocal? || addr.ipv6_unique_local? } raise BlockedUrlError, "Requests to the local network are not allowed" end - def validate_link_local!(addrs_info) + def validate_link_local(addrs_info) netmask = IPAddr.new('169.254.0.0/16') return unless addrs_info.any? { |addr| addr.ipv6_linklocal? || netmask.include?(addr.ip_address) } @@ -205,6 +254,16 @@ module Gitlab (uri.port.blank? || uri.port == config.gitlab_shell.ssh_port) end + def local_ip_whitelisted?(ip_whitelist, ip_string) + ip_obj = Gitlab::Utils.string_to_ip_object(ip_string) + + ip_whitelist.any? { |ip| ip.include?(ip_obj) } + end + + def local_domain_whitelisted?(domain_whitelist, domain_string) + domain_whitelist.include?(domain_string) + end + def config Gitlab.config end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 9aa2e972adf..353298e67b3 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -6,7 +6,9 @@ module Gitlab class << self def data(force_refresh: false) - Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) { uncached_data } + Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) do + uncached_data + end end def uncached_data @@ -98,9 +100,7 @@ module Gitlab .merge(services_usage) .merge(approximate_counts) }.tap do |data| - if Feature.enabled?(:group_overview_security_dashboard) - data[:counts][:user_preferences] = user_preferences_usage - end + data[:counts][:user_preferences] = user_preferences_usage end end # rubocop: enable CodeReuse/ActiveRecord @@ -128,16 +128,29 @@ module Gitlab } end + # @return [Hash<Symbol, Integer>] def usage_counters - { - web_ide_commits: Gitlab::WebIdeCommitsCounter.total_count - } + usage_data_counters.map(&:totals).reduce({}) { |a, b| a.merge(b) } + end + + # @return [Array<#totals>] An array of objects that respond to `#totals` + def usage_data_counters + [ + Gitlab::UsageDataCounters::WikiPageCounter, + Gitlab::UsageDataCounters::WebIdeCounter, + Gitlab::UsageDataCounters::NoteCounter, + Gitlab::UsageDataCounters::SnippetCounter, + Gitlab::UsageDataCounters::SearchCounter, + Gitlab::UsageDataCounters::CycleAnalyticsCounter, + Gitlab::UsageDataCounters::SourceCodeCounter + ] end def components_usage_data { - gitlab_pages: { enabled: Gitlab.config.pages.enabled, version: Gitlab::Pages::VERSION }, git: { version: Gitlab::Git.version }, + gitaly: { version: Gitaly::Server.all.first.server_version, servers: Gitaly::Server.count, filesystems: Gitaly::Server.filesystems }, + gitlab_pages: { enabled: Gitlab.config.pages.enabled, version: Gitlab::Pages::VERSION }, database: { adapter: Gitlab::Database.adapter_name, version: Gitlab::Database.version } } end @@ -175,8 +188,8 @@ module Gitlab {} # augmented in EE end - def count(relation, fallback: -1) - relation.count + def count(relation, count_by: nil, fallback: -1) + count_by ? relation.count(count_by) : relation.count rescue ActiveRecord::StatementInvalid fallback end diff --git a/lib/gitlab/usage_data_counters/base_counter.rb b/lib/gitlab/usage_data_counters/base_counter.rb new file mode 100644 index 00000000000..2b52571c3cc --- /dev/null +++ b/lib/gitlab/usage_data_counters/base_counter.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab::UsageDataCounters + class BaseCounter + extend RedisCounter + + UnknownEvent = Class.new(StandardError) + + class << self + def redis_key(event) + Gitlab::Sentry.track_exception(UnknownEvent, extra: { event: event }) unless known_events.include?(event.to_s) + + "USAGE_#{prefix}_#{event}".upcase + end + + def count(event) + increment(redis_key event) + end + + def read(event) + total_count(redis_key event) + end + + def totals + known_events.map { |e| ["#{prefix}_#{e}".to_sym, read(e)] }.to_h + end + + private + + def known_events + self::KNOWN_EVENTS + end + + def prefix + self::PREFIX + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/cycle_analytics_counter.rb b/lib/gitlab/usage_data_counters/cycle_analytics_counter.rb new file mode 100644 index 00000000000..1ff4296ef65 --- /dev/null +++ b/lib/gitlab/usage_data_counters/cycle_analytics_counter.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +module Gitlab::UsageDataCounters + class CycleAnalyticsCounter < BaseCounter + KNOWN_EVENTS = %w[views].freeze + PREFIX = 'cycle_analytics' + end +end diff --git a/lib/gitlab/usage_data_counters/note_counter.rb b/lib/gitlab/usage_data_counters/note_counter.rb new file mode 100644 index 00000000000..672450ec82b --- /dev/null +++ b/lib/gitlab/usage_data_counters/note_counter.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab::UsageDataCounters + class NoteCounter < BaseCounter + KNOWN_EVENTS = %w[create].freeze + PREFIX = 'note' + COUNTABLE_TYPES = %w[Snippet Commit MergeRequest].freeze + + class << self + def redis_key(event, noteable_type) + "#{super(event)}_#{noteable_type}".upcase + end + + def count(event, noteable_type) + return unless countable?(noteable_type) + + increment(redis_key(event, noteable_type)) + end + + def read(event, noteable_type) + return 0 unless countable?(noteable_type) + + total_count(redis_key(event, noteable_type)) + end + + def totals + COUNTABLE_TYPES.map do |countable_type| + [:"#{countable_type.underscore}_comment", read(:create, countable_type)] + end.to_h + end + + private + + def countable?(noteable_type) + COUNTABLE_TYPES.include?(noteable_type.to_s) + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/redis_counter.rb b/lib/gitlab/usage_data_counters/redis_counter.rb new file mode 100644 index 00000000000..75d5a75e3a4 --- /dev/null +++ b/lib/gitlab/usage_data_counters/redis_counter.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + module RedisCounter + def increment(redis_counter_key) + return unless Gitlab::CurrentSettings.usage_ping_enabled + + Gitlab::Redis::SharedState.with { |redis| redis.incr(redis_counter_key) } + end + + def total_count(redis_counter_key) + Gitlab::Redis::SharedState.with { |redis| redis.get(redis_counter_key).to_i } + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/search_counter.rb b/lib/gitlab/usage_data_counters/search_counter.rb new file mode 100644 index 00000000000..5f0735347e1 --- /dev/null +++ b/lib/gitlab/usage_data_counters/search_counter.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + class SearchCounter + extend RedisCounter + + NAVBAR_SEARCHES_COUNT_KEY = 'NAVBAR_SEARCHES_COUNT' + + class << self + def increment_navbar_searches_count + increment(NAVBAR_SEARCHES_COUNT_KEY) + end + + def total_navbar_searches_count + total_count(NAVBAR_SEARCHES_COUNT_KEY) + end + + def totals + { + navbar_searches: total_navbar_searches_count + } + end + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/snippet_counter.rb b/lib/gitlab/usage_data_counters/snippet_counter.rb new file mode 100644 index 00000000000..e4d234ce4d9 --- /dev/null +++ b/lib/gitlab/usage_data_counters/snippet_counter.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +module Gitlab::UsageDataCounters + class SnippetCounter < BaseCounter + KNOWN_EVENTS = %w[create update].freeze + PREFIX = 'snippet' + end +end diff --git a/lib/gitlab/usage_data_counters/source_code_counter.rb b/lib/gitlab/usage_data_counters/source_code_counter.rb new file mode 100644 index 00000000000..8a1771a7bd1 --- /dev/null +++ b/lib/gitlab/usage_data_counters/source_code_counter.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +module Gitlab::UsageDataCounters + class SourceCodeCounter < BaseCounter + KNOWN_EVENTS = %w[pushes].freeze + PREFIX = 'source_code' + end +end diff --git a/lib/gitlab/usage_data_counters/web_ide_counter.rb b/lib/gitlab/usage_data_counters/web_ide_counter.rb new file mode 100644 index 00000000000..0718c1dd761 --- /dev/null +++ b/lib/gitlab/usage_data_counters/web_ide_counter.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + class WebIdeCounter + extend RedisCounter + + COMMITS_COUNT_KEY = 'WEB_IDE_COMMITS_COUNT' + MERGE_REQUEST_COUNT_KEY = 'WEB_IDE_MERGE_REQUESTS_COUNT' + VIEWS_COUNT_KEY = 'WEB_IDE_VIEWS_COUNT' + + class << self + def increment_commits_count + increment(COMMITS_COUNT_KEY) + end + + def total_commits_count + total_count(COMMITS_COUNT_KEY) + end + + def increment_merge_requests_count + increment(MERGE_REQUEST_COUNT_KEY) + end + + def total_merge_requests_count + total_count(MERGE_REQUEST_COUNT_KEY) + end + + def increment_views_count + increment(VIEWS_COUNT_KEY) + end + + def total_views_count + total_count(VIEWS_COUNT_KEY) + end + + def totals + { + web_ide_commits: total_commits_count, + web_ide_views: total_views_count, + web_ide_merge_requests: total_merge_requests_count + } + end + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/wiki_page_counter.rb b/lib/gitlab/usage_data_counters/wiki_page_counter.rb new file mode 100644 index 00000000000..9cfe0be5bab --- /dev/null +++ b/lib/gitlab/usage_data_counters/wiki_page_counter.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +module Gitlab::UsageDataCounters + class WikiPageCounter < BaseCounter + KNOWN_EVENTS = %w[create update delete].freeze + PREFIX = 'wiki_pages' + end +end diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index 9ef23cf849f..097b502316e 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -45,7 +45,7 @@ module Gitlab if protected?(ProtectedTag, project, ref) protected_tag_accessible_to?(ref, action: :create) else - user.can?(:push_code, project) + user.can?(:admin_tag, project) end end diff --git a/lib/gitlab/user_extractor.rb b/lib/gitlab/user_extractor.rb deleted file mode 100644 index ede60c9ab1d..00000000000 --- a/lib/gitlab/user_extractor.rb +++ /dev/null @@ -1,56 +0,0 @@ -# frozen_string_literal: true - -# This class extracts all users found in a piece of text by the username or the -# email address - -module Gitlab - class UserExtractor - # Not using `Devise.email_regexp` to filter out any chars that an email - # does not end with and not pinning the email to a start of end of a string. - EMAIL_REGEXP = /(?<email>([^@\s]+@[^@\s]+(?<!\W)))/.freeze - USERNAME_REGEXP = User.reference_pattern - - def initialize(text) - # EE passes an Array to `text` in a few places, so we want to support both - # here. - @text = Array(text).join(' ') - end - - def users - return User.none unless @text.present? - return User.none if references.empty? - - @users ||= User.from_union(union_relations) - end - - def usernames - matches[:usernames] - end - - def emails - matches[:emails] - end - - def references - @references ||= matches.values.flatten - end - - def matches - @matches ||= { - emails: @text.scan(EMAIL_REGEXP).flatten.uniq, - usernames: @text.scan(USERNAME_REGEXP).flatten.uniq - } - end - - private - - def union_relations - relations = [] - - relations << User.by_any_email(emails) if emails.any? - relations << User.by_username(usernames) if usernames.any? - - relations - end - end -end diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index 16ec8a8bb28..c66ce0434a4 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -22,7 +22,7 @@ module Gitlab end def force_utf8(str) - str.force_encoding(Encoding::UTF_8) + str.dup.force_encoding(Encoding::UTF_8) end def ensure_utf8_size(str, bytes:) @@ -131,5 +131,12 @@ module Gitlab data end end + + def string_to_ip_object(str) + return unless str + + IPAddr.new(str) + rescue IPAddr::InvalidAddressError + end end end diff --git a/lib/gitlab/utils/deep_size.rb b/lib/gitlab/utils/deep_size.rb new file mode 100644 index 00000000000..562cf09e249 --- /dev/null +++ b/lib/gitlab/utils/deep_size.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +require 'objspace' + +module Gitlab + module Utils + class DeepSize + Error = Class.new(StandardError) + TooMuchDataError = Class.new(Error) + + DEFAULT_MAX_SIZE = 1.megabyte + DEFAULT_MAX_DEPTH = 100 + + def initialize(root, max_size: DEFAULT_MAX_SIZE, max_depth: DEFAULT_MAX_DEPTH) + @root = root + @max_size = max_size + @max_depth = max_depth + @size = 0 + @depth = 0 + + evaluate + end + + def valid? + !too_big? && !too_deep? + end + + private + + def evaluate + add_object(@root) + rescue Error + # NOOP + end + + def too_big? + @size > @max_size + end + + def too_deep? + @depth > @max_depth + end + + def add_object(object) + @size += ObjectSpace.memsize_of(object) + raise TooMuchDataError if @size > @max_size + + add_array(object) if object.is_a?(Array) + add_hash(object) if object.is_a?(Hash) + end + + def add_array(object) + with_nesting do + object.each do |n| + add_object(n) + end + end + end + + def add_hash(object) + with_nesting do + object.each do |key, value| + add_object(key) + add_object(value) + end + end + end + + def with_nesting + @depth += 1 + raise TooMuchDataError if too_deep? + + yield + + @depth -= 1 + end + end + end +end diff --git a/lib/gitlab/utils/sanitize_node_link.rb b/lib/gitlab/utils/sanitize_node_link.rb new file mode 100644 index 00000000000..620d71a7814 --- /dev/null +++ b/lib/gitlab/utils/sanitize_node_link.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +require_dependency 'gitlab/utils' + +module Gitlab + module Utils + module SanitizeNodeLink + UNSAFE_PROTOCOLS = %w(data javascript vbscript).freeze + ATTRS_TO_SANITIZE = %w(href src data-src).freeze + + def remove_unsafe_links(env, remove_invalid_links: true) + node = env[:node] + + sanitize_node(node: node, remove_invalid_links: remove_invalid_links) + + # HTML entities such as <video></video> have scannable attrs in + # children elements, which also need to be sanitized. + # + node.children.each do |child_node| + sanitize_node(node: child_node, remove_invalid_links: remove_invalid_links) + end + end + + # Remove all invalid scheme characters before checking against the + # list of unsafe protocols. + # + # See https://tools.ietf.org/html/rfc3986#section-3.1 + # + def safe_protocol?(scheme) + return false unless scheme + + scheme = scheme + .strip + .downcase + .gsub(/[^A-Za-z\+\.\-]+/, '') + + UNSAFE_PROTOCOLS.none?(scheme) + end + + private + + def sanitize_node(node:, remove_invalid_links: true) + ATTRS_TO_SANITIZE.each do |attr| + next unless node.has_attribute?(attr) + + begin + node[attr] = node[attr].strip + uri = Addressable::URI.parse(node[attr]) + + next unless uri.scheme + next if safe_protocol?(uri.scheme) + + node.remove_attribute(attr) + rescue Addressable::URI::InvalidURIError + node.remove_attribute(attr) if remove_invalid_links + end + end + end + end + end +end diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index 8f9d5cf1e63..e2787744f09 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -138,5 +138,18 @@ module Gitlab def visibility=(level) self[visibility_level_field] = Gitlab::VisibilityLevel.level_value(level) end + + def visibility_attribute_present?(attributes) + visibility_level_attributes.each do |attr| + return true if attributes[attr].present? + end + + false + end + + def visibility_level_attributes + [visibility_level_field, visibility_level_field.to_s, + :visibility, 'visibility'] + end end end diff --git a/lib/gitlab/web_ide_commits_counter.rb b/lib/gitlab/web_ide_commits_counter.rb deleted file mode 100644 index 1cd9b5295b9..00000000000 --- a/lib/gitlab/web_ide_commits_counter.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module WebIdeCommitsCounter - WEB_IDE_COMMITS_KEY = "WEB_IDE_COMMITS_COUNT".freeze - - class << self - def increment - Gitlab::Redis::SharedState.with { |redis| redis.incr(WEB_IDE_COMMITS_KEY) } - end - - def total_count - Gitlab::Redis::SharedState.with { |redis| redis.get(WEB_IDE_COMMITS_KEY).to_i } - end - end - end -end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 46a7b5b982a..3b77fe838ae 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -221,7 +221,7 @@ module Gitlab end def set_key_and_notify(key, value, expire: nil, overwrite: true) - Gitlab::Redis::Queues.with do |redis| + Gitlab::Redis::SharedState.with do |redis| result = redis.set(key, value, ex: expire, nx: !overwrite) if result redis.publish(NOTIFICATION_CHANNEL, "#{key}=#{value}") diff --git a/lib/gitlab/zoom_link_extractor.rb b/lib/gitlab/zoom_link_extractor.rb new file mode 100644 index 00000000000..7ac14eb2d4f --- /dev/null +++ b/lib/gitlab/zoom_link_extractor.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +# Detect links matching the following formats: +# Zoom Start links: https://zoom.us/s/<meeting-id> +# Zoom Join links: https://zoom.us/j/<meeting-id> +# Personal Zoom links: https://zoom.us/my/<meeting-id> +# Vanity Zoom links: https://gitlab.zoom.us/j/<meeting-id> (also /s and /my) + +module Gitlab + class ZoomLinkExtractor + ZOOM_REGEXP = %r{https://(?:[\w-]+\.)?zoom\.us/(?:s|j|my)/\S+}.freeze + + def initialize(text) + @text = text.to_s + end + + def links + @text.scan(ZOOM_REGEXP) + end + + def match? + ZOOM_REGEXP.match?(@text) + end + end +end diff --git a/lib/gt_one_coercion.rb b/lib/gt_one_coercion.rb deleted file mode 100644 index 99be51bc8c6..00000000000 --- a/lib/gt_one_coercion.rb +++ /dev/null @@ -1,7 +0,0 @@ -# frozen_string_literal: true - -class GtOneCoercion < Virtus::Attribute - def coerce(value) - [1, value.to_i].max - end -end diff --git a/lib/mattermost/session.rb b/lib/mattermost/session.rb index 722e3e04d1c..eea7daa3d8e 100644 --- a/lib/mattermost/session.rb +++ b/lib/mattermost/session.rb @@ -41,7 +41,7 @@ module Mattermost begin yield self rescue Errno::ECONNREFUSED => e - Rails.logger.error(e.message + "\n" + e.backtrace.join("\n")) + Rails.logger.error(e.message + "\n" + e.backtrace.join("\n")) # rubocop:disable Gitlab/RailsLogger raise Mattermost::NoSessionError ensure destroy diff --git a/lib/microsoft_teams/notifier.rb b/lib/microsoft_teams/notifier.rb index c7dec09ba6b..a7dcd322e27 100644 --- a/lib/microsoft_teams/notifier.rb +++ b/lib/microsoft_teams/notifier.rb @@ -20,7 +20,7 @@ module MicrosoftTeams result = true if response rescue Gitlab::HTTP::Error, StandardError => error - Rails.logger.info("#{self.class.name}: Error while connecting to #{@webhook}: #{error.message}") + Rails.logger.info("#{self.class.name}: Error while connecting to #{@webhook}: #{error.message}") # rubocop:disable Gitlab/RailsLogger end result diff --git a/lib/mysql_zero_date.rb b/lib/mysql_zero_date.rb deleted file mode 100644 index f36610abf8f..00000000000 --- a/lib/mysql_zero_date.rb +++ /dev/null @@ -1,20 +0,0 @@ -# frozen_string_literal: true - -# Disable NO_ZERO_DATE mode for mysql in rails 5. -# We use zero date as a default value -# (config/initializers/active_record_mysql_timestamp.rb), in -# Rails 5 using zero date fails by default (https://gitlab.com/gitlab-org/gitlab-ce/-/jobs/75450216) -# and NO_ZERO_DATE has to be explicitly disabled. Disabling strict mode -# is not sufficient. - -require 'active_record/connection_adapters/abstract_mysql_adapter' - -module MysqlZeroDate - def configure_connection - super - - @connection.query "SET @@SESSION.sql_mode = REPLACE(@@SESSION.sql_mode, 'NO_ZERO_DATE', '');" # rubocop:disable Gitlab/ModuleWithInstanceVariables - end -end - -ActiveRecord::ConnectionAdapters::AbstractMysqlAdapter.prepend(MysqlZeroDate) diff --git a/lib/peek/rblineprof/custom_controller_helpers.rb b/lib/peek/rblineprof/custom_controller_helpers.rb deleted file mode 100644 index 581cc6a37b4..00000000000 --- a/lib/peek/rblineprof/custom_controller_helpers.rb +++ /dev/null @@ -1,124 +0,0 @@ -# frozen_string_literal: true - -module Peek - module Rblineprof - module CustomControllerHelpers - extend ActiveSupport::Concern - - # This will become useless once https://github.com/peek/peek-rblineprof/pull/5 - # is merged - def pygmentize(file_name, code, lexer = nil) - if lexer.present? - Gitlab::Highlight.highlight(file_name, code) - else - "<pre>#{Rack::Utils.escape_html(code)}</pre>" - end - end - - # rubocop:disable all - def inject_rblineprof - ret = nil - profile = lineprof(rblineprof_profiler_regex) do - ret = yield - end - - if response.content_type =~ %r|text/html| - sort = params[:lineprofiler_sort] - mode = params[:lineprofiler_mode] || 'cpu' - min = (params[:lineprofiler_min] || 5).to_i * 1000 - summary = params[:lineprofiler_summary] - - # Sort each file by the longest calculated time - per_file = profile.map do |file, lines| - total, child, excl, total_cpu, child_cpu, excl_cpu = lines[0] - - wall = summary == 'exclusive' ? excl : total - cpu = summary == 'exclusive' ? excl_cpu : total_cpu - idle = summary == 'exclusive' ? (excl - excl_cpu) : (total - total_cpu) - - [ - file, lines, - wall, cpu, idle, - sort == 'idle' ? idle : sort == 'cpu' ? cpu : wall - ] - end.sort_by{ |a,b,c,d,e,f| -f } - - output = ["<div class='modal-dialog modal-xl'><div class='modal-content'>"] - output << "<div class='modal-header'>" - output << "<h4>Line profiling: #{human_description(params[:lineprofiler])}</h4>" - output << "<button class='close' type='button' data-dismiss='modal' aria-label='close'><span aria-hidden='true'>×</span></button>" - output << "</div>" - output << "<div class='modal-body'>" - - per_file.each do |file_name, lines, file_wall, file_cpu, file_idle, file_sort| - output << "<div class='peek-rblineprof-file'><div class='heading'>" - - show_src = file_sort > min - tmpl = show_src ? "<a href='#' class='js-lineprof-file'>%s</a>" : "%s" - - if mode == 'cpu' - output << sprintf("<span class='duration'>% 8.1fms + % 8.1fms</span> #{tmpl}", file_cpu / 1000.0, file_idle / 1000.0, file_name.sub(Rails.root.to_s + '/', '')) - else - output << sprintf("<span class='duration'>% 8.1fms</span> #{tmpl}", file_wall/1000.0, file_name.sub(Rails.root.to_s + '/', '')) - end - - output << "</div>" # .heading - - next unless show_src - - output << "<div class='data'>" - code = [] - times = [] - File.readlines(file_name).each_with_index do |line, i| - code << line - wall, cpu, calls = lines[i + 1] - - if calls && calls > 0 - if mode == 'cpu' - idle = wall - cpu - times << sprintf("% 8.1fms + % 8.1fms (% 5d)", cpu / 1000.0, idle / 1000.0, calls) - else - times << sprintf("% 8.1fms (% 5d)", wall / 1000.0, calls) - end - else - times << ' ' - end - end - output << "<pre class='duration'>#{times.join("\n")}</pre>" - # The following line was changed from - # https://github.com/peek/peek-rblineprof/blob/8d3b7a283a27de2f40abda45974516693d882258/lib/peek/rblineprof/controller_helpers.rb#L125 - # This will become useless once https://github.com/peek/peek-rblineprof/pull/16 - # is merged and is implemented. - output << "<pre class='code highlight white'>#{pygmentize(file_name, code.join, 'ruby')}</pre>" - output << "</div></div>" # .data then .peek-rblineprof-file - end - - output << "</div></div></div>" - - response.body += "<div class='modal' id='modal-peek-line-profile' tabindex=-1>#{output.join}</div>".html_safe - end - - ret - end - - private - - def human_description(lineprofiler_param) - case lineprofiler_param - when 'app' - 'app/ & lib/' - when 'views' - 'app/view/' - when 'gems' - 'vendor/gems' - when 'all' - 'everything in Rails.root' - when 'stdlib' - 'everything in the Ruby standard library' - else - 'app/, config/, lib/, vendor/ & plugin/' - end - end - end - end -end diff --git a/lib/peek/views/active_record.rb b/lib/peek/views/active_record.rb new file mode 100644 index 00000000000..2d78818630d --- /dev/null +++ b/lib/peek/views/active_record.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Peek + module Views + class ActiveRecord < DetailedView + private + + def setup_subscribers + super + + subscribe('sql.active_record') do |_, start, finish, _, data| + if Gitlab::SafeRequestStore.store[:peek_enabled] + unless data[:cached] + detail_store << { + duration: finish - start, + sql: data[:sql].strip, + backtrace: Gitlab::Profiler.clean_backtrace(caller) + } + end + end + end + end + end + end +end diff --git a/lib/peek/views/detailed_view.rb b/lib/peek/views/detailed_view.rb new file mode 100644 index 00000000000..f4ca1cb5075 --- /dev/null +++ b/lib/peek/views/detailed_view.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module Peek + module Views + class DetailedView < View + def results + { + duration: formatted_duration, + calls: calls, + details: details + } + end + + def detail_store + ::Gitlab::SafeRequestStore["#{key}_call_details"] ||= [] + end + + private + + def duration + detail_store.map { |entry| entry[:duration] }.sum # rubocop:disable CodeReuse/ActiveRecord + end + + def calls + detail_store.count + end + + def call_details + detail_store + end + + def format_call_details(call) + call.merge(duration: (call[:duration] * 1000).round(3)) + end + + def details + call_details + .sort { |a, b| b[:duration] <=> a[:duration] } + .map(&method(:format_call_details)) + end + + def formatted_duration + ms = duration * 1000 + + if ms >= 1000 + "%.2fms" % ms + else + "%.0fms" % ms + end + end + end + end +end diff --git a/lib/peek/views/gitaly.rb b/lib/peek/views/gitaly.rb index 30f95a10024..6ad6ddfd89d 100644 --- a/lib/peek/views/gitaly.rb +++ b/lib/peek/views/gitaly.rb @@ -2,7 +2,9 @@ module Peek module Views - class Gitaly < View + class Gitaly < DetailedView + private + def duration ::Gitlab::GitalyClient.query_time end @@ -11,36 +13,14 @@ module Peek ::Gitlab::GitalyClient.get_request_count end - def results - { - duration: formatted_duration, - calls: calls, - details: details - } - end - - private - - def details + def call_details ::Gitlab::GitalyClient.list_call_details - .sort { |a, b| b[:duration] <=> a[:duration] } - .map(&method(:format_call_details)) end def format_call_details(call) pretty_request = call[:request]&.reject { |k, v| v.blank? }.to_h.pretty_inspect - call.merge(duration: (call[:duration] * 1000).round(3), - request: pretty_request || {}) - end - - def formatted_duration - ms = duration * 1000 - if ms >= 1000 - "%.2fms" % ms - else - "%.0fms" % ms - end + super.merge(request: pretty_request || {}) end def setup_subscribers diff --git a/lib/peek/views/redis_detailed.rb b/lib/peek/views/redis_detailed.rb new file mode 100644 index 00000000000..f36f581d5e9 --- /dev/null +++ b/lib/peek/views/redis_detailed.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +require 'redis' + +module Gitlab + module Peek + module RedisInstrumented + def call(*args, &block) + start = Time.now + super(*args, &block) + ensure + duration = (Time.now - start) + add_call_details(duration, args) + end + + private + + def add_call_details(duration, args) + return unless peek_enabled? + # redis-rb passes an array (e.g. [:get, key]) + return unless args.length == 1 + + detail_store << { + cmd: args.first, + duration: duration, + backtrace: ::Gitlab::Profiler.clean_backtrace(caller) + } + end + + def peek_enabled? + Gitlab::SafeRequestStore.store[:peek_enabled] + end + + def detail_store + ::Gitlab::SafeRequestStore['redis_call_details'] ||= [] + end + end + end +end + +module Peek + module Views + class RedisDetailed < DetailedView + REDACTED_MARKER = "<redacted>" + + def key + 'redis' + end + + private + + def format_call_details(call) + super.merge(cmd: format_command(call[:cmd])) + end + + def format_command(cmd) + if cmd.length >= 2 && cmd.first =~ /^auth$/i + cmd[-1] = REDACTED_MARKER + # Scrub out the value of the SET calls to avoid binary + # data or large data from spilling into the view + elsif cmd.length >= 3 && cmd.first =~ /set/i + cmd[2..-1] = REDACTED_MARKER + end + + cmd.join(' ') + end + end + end +end + +class Redis::Client + prepend Gitlab::Peek::RedisInstrumented +end diff --git a/lib/peek/views/rugged.rb b/lib/peek/views/rugged.rb new file mode 100644 index 00000000000..18b3f422852 --- /dev/null +++ b/lib/peek/views/rugged.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Peek + module Views + class Rugged < DetailedView + def results + return {} unless calls > 0 + + super + end + + private + + def duration + ::Gitlab::RuggedInstrumentation.query_time + end + + def calls + ::Gitlab::RuggedInstrumentation.query_count + end + + def call_details + ::Gitlab::RuggedInstrumentation.list_call_details + end + + def format_call_details(call) + super.merge(args: format_args(call[:args])) + end + + def format_args(args) + args.map do |arg| + # ActiveSupport::JSON recursively calls as_json on all + # instance variables, and if that instance variable points to + # something that refers back to the same instance, we can wind + # up in an infinite loop. Currently this only seems to happen with + # Gitlab::Git::Repository and ::Repository. + if arg.instance_variables.present? + arg.to_s + else + arg + end + end + end + end + end +end diff --git a/lib/prometheus/cleanup_multiproc_dir_service.rb b/lib/prometheus/cleanup_multiproc_dir_service.rb new file mode 100644 index 00000000000..6418b4de166 --- /dev/null +++ b/lib/prometheus/cleanup_multiproc_dir_service.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Prometheus + class CleanupMultiprocDirService + include Gitlab::Utils::StrongMemoize + + def execute + FileUtils.rm_rf(old_metrics) if old_metrics + end + + private + + def old_metrics + strong_memoize(:old_metrics) do + Dir[File.join(multiprocess_files_dir, '*.db')] if multiprocess_files_dir + end + end + + def multiprocess_files_dir + ::Prometheus::Client.configuration.multiprocess_files_dir + end + end +end diff --git a/lib/prometheus/pid_provider.rb b/lib/prometheus/pid_provider.rb new file mode 100644 index 00000000000..e0f7e7e0a9e --- /dev/null +++ b/lib/prometheus/pid_provider.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Prometheus + module PidProvider + extend self + + def worker_id + if Sidekiq.server? + 'sidekiq' + elsif defined?(Unicorn::Worker) + unicorn_worker_id + elsif defined?(::Puma) + puma_worker_id + else + unknown_process_id + end + end + + private + + def unicorn_worker_id + if matches = process_name.match(/unicorn.*worker\[([0-9]+)\]/) + "unicorn_#{matches[1]}" + elsif process_name =~ /unicorn/ + "unicorn_master" + else + unknown_process_id + end + end + + def puma_worker_id + if matches = process_name.match(/puma.*cluster worker ([0-9]+):/) + "puma_#{matches[1]}" + elsif process_name =~ /puma/ + "puma_master" + else + unknown_process_id + end + end + + def unknown_process_id + "process_#{Process.pid}" + end + + def process_name + $0 + end + end +end diff --git a/lib/quality/test_level.rb b/lib/quality/test_level.rb index 24d8eac200c..60d79b52680 100644 --- a/lib/quality/test_level.rb +++ b/lib/quality/test_level.rb @@ -14,6 +14,7 @@ module Quality finders frontend graphql + haml_lint helpers initializers javascripts diff --git a/lib/rspec_flaky/listener.rb b/lib/rspec_flaky/listener.rb index 19cc0baa2d3..bf15130d17e 100644 --- a/lib/rspec_flaky/listener.rb +++ b/lib/rspec_flaky/listener.rb @@ -32,6 +32,7 @@ module RspecFlaky flaky_examples[current_example.uid] = flaky_example end + # rubocop:disable Gitlab/RailsLogger def dump_summary(_) RspecFlaky::Report.new(flaky_examples).write(RspecFlaky::Config.flaky_examples_report_path) # write_report_file(flaky_examples, RspecFlaky::Config.flaky_examples_report_path) @@ -45,6 +46,7 @@ module RspecFlaky # write_report_file(new_flaky_examples, RspecFlaky::Config.new_flaky_examples_report_path) end end + # rubocop:enable Gitlab/RailsLogger private diff --git a/lib/sentry/client.rb b/lib/sentry/client.rb index 4022e8ff946..07cca1c8d1e 100644 --- a/lib/sentry/client.rb +++ b/lib/sentry/client.rb @@ -67,7 +67,7 @@ module Sentry def handle_request_exceptions yield - rescue HTTParty::Error => e + rescue Gitlab::HTTP::Error => e Gitlab::Sentry.track_acceptable_exception(e) raise_error 'Error when connecting to Sentry' rescue Net::OpenTimeout diff --git a/lib/serializers/json.rb b/lib/serializers/json.rb index 93cb192087a..1ed5d5dc3f5 100644 --- a/lib/serializers/json.rb +++ b/lib/serializers/json.rb @@ -1,32 +1,16 @@ # frozen_string_literal: true module Serializers - # This serializer exports data as JSON, - # it is designed to be used with interwork compatibility between MySQL and PostgreSQL - # implementations, as used version of MySQL does not support native json type - # - # Secondly, the loader makes the resulting hash to have deep indifferent access + # Make the resulting hash have deep indifferent access class JSON class << self def dump(obj) - # MySQL stores data as text - # look at ./config/initializers/ar_mysql_jsonb_support.rb - if Gitlab::Database.mysql? - obj = ActiveSupport::JSON.encode(obj) - end - obj end def load(data) return if data.nil? - # On MySQL we store data as text - # look at ./config/initializers/ar_mysql_jsonb_support.rb - if Gitlab::Database.mysql? - data = ActiveSupport::JSON.decode(data) - end - Gitlab::Utils.deep_indifferent_access(data) end end diff --git a/lib/support/deploy/deploy.sh b/lib/support/deploy/deploy.sh index ab46c47d8f5..1f8c915140c 100755 --- a/lib/support/deploy/deploy.sh +++ b/lib/support/deploy/deploy.sh @@ -28,7 +28,7 @@ sudo -u git -H git pull origin master echo 'Deploy: Bundle and migrate' # change it to your needs -sudo -u git -H bundle --without aws development test mysql --deployment +sudo -u git -H bundle --without aws development test --deployment sudo -u git -H bundle exec rake db:migrate RAILS_ENV=production sudo -u git -H bundle exec rake gitlab:assets:clean RAILS_ENV=production diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab index 32df74f104a..bccb94ff0bf 100755 --- a/lib/support/init.d/gitlab +++ b/lib/support/init.d/gitlab @@ -67,6 +67,13 @@ if ! cd "$app_root" ; then echo "Failed to cd into $app_root, exiting!"; exit 1 fi +# Select the web server to use +if [ -z "$EXPERIMENTAL_PUMA" ]; then + use_web_server="unicorn" +else + use_web_server="puma" +fi + ### Init Script functions @@ -256,7 +263,7 @@ start_gitlab() { check_stale_pids if [ "$web_status" != "0" ]; then - echo "Starting GitLab web server" + echo "Starting GitLab web server ($use_web_server)" fi if [ "$sidekiq_status" != "0" ]; then echo "Starting GitLab Sidekiq" @@ -281,7 +288,7 @@ start_gitlab() { # Remove old socket if it exists rm -f "$rails_socket" 2>/dev/null # Start the web server - RAILS_ENV=$RAILS_ENV EXPERIMENTAL_PUMA=$EXPERIMENTAL_PUMA bin/web start + RAILS_ENV=$RAILS_ENV USE_WEB_SERVER=$use_web_server bin/web start fi # If sidekiq is already running, don't start it again. @@ -343,7 +350,7 @@ stop_gitlab() { if [ "$web_status" = "0" ]; then echo "Shutting down GitLab web server" - RAILS_ENV=$RAILS_ENV EXPERIMENTAL_PUMA=$EXPERIMENTAL_PUMA bin/web stop + RAILS_ENV=$RAILS_ENV USE_WEB_SERVER=$use_web_server bin/web stop fi if [ "$sidekiq_status" = "0" ]; then echo "Shutting down GitLab Sidekiq" @@ -447,7 +454,7 @@ reload_gitlab(){ exit 1 fi printf "Reloading GitLab web server configuration... " - RAILS_ENV=$RAILS_ENV EXPERIMENTAL_PUMA=$EXPERIMENTAL_PUMA bin/web reload + RAILS_ENV=$RAILS_ENV USE_WEB_SERVER=$use_web_server bin/web reload echo "Done." echo "Restarting GitLab Sidekiq since it isn't capable of reloading its config..." diff --git a/lib/system_check/app/git_version_check.rb b/lib/system_check/app/git_version_check.rb index 467711fb74e..08c8df9b044 100644 --- a/lib/system_check/app/git_version_check.rb +++ b/lib/system_check/app/git_version_check.rb @@ -7,7 +7,7 @@ module SystemCheck set_check_pass -> { "yes (#{self.current_version})" } def self.required_version - @required_version ||= Gitlab::VersionInfo.parse('2.21.0') + @required_version ||= Gitlab::VersionInfo.parse('2.22.0') end def self.current_version diff --git a/lib/system_check/ldap_check.rb b/lib/system_check/ldap_check.rb index 619fb3cccb8..938026424ed 100644 --- a/lib/system_check/ldap_check.rb +++ b/lib/system_check/ldap_check.rb @@ -33,8 +33,13 @@ module SystemCheck $stdout.puts "LDAP users with access to your GitLab server (only showing the first #{limit} results)" users = adapter.users(adapter.config.uid, '*', limit) - users.each do |user| - $stdout.puts "\tDN: #{user.dn}\t #{adapter.config.uid}: #{user.uid}" + + if should_sanitize? + $stdout.puts "\tUser output sanitized. Found #{users.length} users of #{limit} limit." + else + users.each do |user| + $stdout.puts "\tDN: #{user.dn}\t #{adapter.config.uid}: #{user.uid}" + end end end rescue Net::LDAP::ConnectionRefusedError, Errno::ECONNREFUSED => e diff --git a/lib/tasks/frontend.rake b/lib/tasks/frontend.rake new file mode 100644 index 00000000000..1cac7520227 --- /dev/null +++ b/lib/tasks/frontend.rake @@ -0,0 +1,21 @@ +unless Rails.env.production? + namespace :frontend do + desc 'GitLab | Frontend | Generate fixtures for JavaScript tests' + RSpec::Core::RakeTask.new(:fixtures, [:pattern]) do |t, args| + args.with_defaults(pattern: '{spec,ee/spec}/frontend/fixtures/*.rb') + ENV['NO_KNAPSACK'] = 'true' + t.pattern = args[:pattern] + t.rspec_opts = '--format documentation' + end + + desc 'GitLab | Frontend | Run JavaScript tests' + task tests: ['yarn:check'] do + sh "yarn test" do |ok, res| + abort('rake frontend:tests failed') unless ok + end + end + end + + desc 'GitLab | Frontend | Shortcut for frontend:fixtures and frontend:tests' + task frontend: ['frontend:fixtures', 'frontend:tests'] +end diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index c531eb1d216..2bf71701b57 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -21,10 +21,10 @@ namespace :gitlab do backup.cleanup backup.remove_old - puts "Warning: Your gitlab.rb and gitlab-secrets.json files contain sensitive data \n" \ + progress.puts "Warning: Your gitlab.rb and gitlab-secrets.json files contain sensitive data \n" \ "and are not included in this backup. You will need these files to restore a backup.\n" \ "Please back them up manually.".color(:red) - puts "Backup task is done." + progress.puts "Backup task is done." end # Restore backup of GitLab system diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index 760331620ef..4d854cd178d 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -115,6 +115,70 @@ namespace :gitlab do end end + desc 'GitLab | Cleanup | Clean orphan job artifact files' + task orphan_job_artifact_files: :gitlab_environment do + warn_user_is_not_gitlab + + cleaner = Gitlab::Cleanup::OrphanJobArtifactFiles.new(limit: limit, dry_run: dry_run?, niceness: niceness, logger: logger) + cleaner.run! + + if dry_run? + logger.info "To clean up these files run this command with DRY_RUN=false".color(:yellow) + end + end + + namespace :sessions do + desc "GitLab | Cleanup | Sessions | Clean ActiveSession lookup keys" + task active_sessions_lookup_keys: :gitlab_environment do + session_key_pattern = "#{Gitlab::Redis::SharedState::USER_SESSIONS_LOOKUP_NAMESPACE}:*" + last_save_check = Time.at(0) + wait_time = 10.seconds + cursor = 0 + total_users_scanned = 0 + + Gitlab::Redis::SharedState.with do |redis| + begin + cursor, keys = redis.scan(cursor, match: session_key_pattern) + total_users_scanned += keys.count + + if last_save_check < Time.now - 1.second + while redis.info('persistence')['rdb_bgsave_in_progress'] == '1' + puts "BGSAVE in progress, waiting #{wait_time} seconds" + sleep(wait_time) + end + last_save_check = Time.now + end + + keys.each do |key| + user_id = key.split(':').last + + lookup_key_count = redis.scard(key) + + session_ids = ActiveSession.session_ids_for_user(user_id) + entries = ActiveSession.raw_active_session_entries(session_ids, user_id) + session_ids_and_entries = session_ids.zip(entries) + + inactive_session_ids = session_ids_and_entries.map do |session_id, session| + session_id if session.nil? + end.compact + + redis.pipelined do |conn| + inactive_session_ids.each do |session_id| + conn.srem(key, session_id) + end + end + + if inactive_session_ids + puts "deleted #{inactive_session_ids.count} out of #{lookup_key_count} lookup keys for User ##{user_id}" + end + end + end while cursor.to_i != 0 + + puts "--- All done! Total number of scanned users: #{total_users_scanned}" + end + end + end + def remove? ENV['REMOVE'] == 'true' end @@ -123,16 +187,31 @@ namespace :gitlab do ENV['DRY_RUN'] != 'false' end + def debug? + ENV['DEBUG'].present? + end + + def limit + ENV['LIMIT']&.to_i + end + + def niceness + ENV['NICENESS'].presence + end + + # rubocop:disable Gitlab/RailsLogger def logger return @logger if defined?(@logger) @logger = if Rails.env.development? || Rails.env.production? Logger.new(STDOUT).tap do |stdout_logger| stdout_logger.extend(ActiveSupport::Logger.broadcast(Rails.logger)) + stdout_logger.level = debug? ? Logger::DEBUG : Logger::INFO end else Rails.logger end end + # rubocop:enable Gitlab/RailsLogger end end diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 4e7a8adbef6..1961f64659c 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -26,26 +26,19 @@ namespace :gitlab do task drop_tables: :environment do connection = ActiveRecord::Base.connection - # If MySQL, turn off foreign key checks - connection.execute('SET FOREIGN_KEY_CHECKS=0') if Gitlab::Database.mysql? - - # connection.tables is deprecated in MySQLAdapter, but in PostgreSQLAdapter - # data_sources returns both views and tables, so use #tables instead - tables = Gitlab::Database.mysql? ? connection.data_sources : connection.tables + # In PostgreSQLAdapter, data_sources returns both views and tables, so use + # #tables instead + tables = connection.tables # Removes the entry from the array tables.delete 'schema_migrations' # Truncate schema_migrations to ensure migrations re-run - connection.execute('TRUNCATE schema_migrations') if connection.data_source_exists? 'schema_migrations' + connection.execute('TRUNCATE schema_migrations') if connection.table_exists? 'schema_migrations' # Drop tables with cascade to avoid dependent table errors # PG: http://www.postgresql.org/docs/current/static/ddl-depend.html - # MySQL: http://dev.mysql.com/doc/refman/5.7/en/drop-table.html # Add `IF EXISTS` because cascade could have already deleted a table. tables.each { |t| connection.execute("DROP TABLE IF EXISTS #{connection.quote_table_name(t)} CASCADE") } - - # If MySQL, re-enable foreign key checks - connection.execute('SET FOREIGN_KEY_CHECKS=1') if Gitlab::Database.mysql? end desc 'Configures the database by running migrate, or by loading the schema and seeding if needed' @@ -77,5 +70,13 @@ namespace :gitlab do Gitlab::DowntimeCheck.new.check_and_print(migrations) end + + desc 'Sets up EE specific database functionality' + + if Gitlab.ee? + task setup_ee: %w[geo:db:drop geo:db:create geo:db:schema:load geo:db:migrate] + else + task :setup_ee + end end end diff --git a/lib/tasks/gitlab/features.rake b/lib/tasks/gitlab/features.rake index d88bcca0819..9cf568c07fe 100644 --- a/lib/tasks/gitlab/features.rake +++ b/lib/tasks/gitlab/features.rake @@ -10,14 +10,22 @@ namespace :gitlab do set_rugged_feature_flags(false) puts 'All Rugged feature flags were disabled.' end + + task unset_rugged: :environment do + set_rugged_feature_flags(nil) + puts 'All Rugged feature flags were unset.' + end end def set_rugged_feature_flags(status) Gitlab::Git::RuggedImpl::Repository::FEATURE_FLAGS.each do |flag| - if status - Feature.enable(flag) - else + case status + when nil Feature.get(flag).remove + when true + Feature.enable(flag) + when false + Feature.disable(flag) end end end diff --git a/lib/tasks/gitlab/graphql.rake b/lib/tasks/gitlab/graphql.rake new file mode 100644 index 00000000000..c53d55ceea2 --- /dev/null +++ b/lib/tasks/gitlab/graphql.rake @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +return if Rails.env.production? + +namespace :gitlab do + OUTPUT_DIR = Rails.root.join("doc/api/graphql/reference").freeze + TEMPLATES_DIR = 'lib/gitlab/graphql/docs/templates/'.freeze + + namespace :graphql do + desc 'GitLab | Generate GraphQL docs' + task compile_docs: :environment do + renderer = Gitlab::Graphql::Docs::Renderer.new(GitlabSchema.graphql_definition, render_options) + + renderer.render + + puts "Documentation compiled." + end + end +end + +def render_options + { + output_dir: OUTPUT_DIR, + template: Rails.root.join(TEMPLATES_DIR, 'default.md.haml') + } +end diff --git a/lib/tasks/gitlab/import_export.rake b/lib/tasks/gitlab/import_export.rake index 900dbf7be24..5365bd3920f 100644 --- a/lib/tasks/gitlab/import_export.rake +++ b/lib/tasks/gitlab/import_export.rake @@ -7,7 +7,7 @@ namespace :gitlab do desc "GitLab | Display exported DB structure" task data: :environment do - puts YAML.load_file(Gitlab::ImportExport.config_file)['project_tree'].to_yaml(SortKeys: true) + puts Gitlab::ImportExport::Config.new.to_h['project_tree'].to_yaml(SortKeys: true) end desc 'GitLab | Bumps the Import/Export version in fixtures and project templates' diff --git a/lib/tasks/gitlab/seed.rake b/lib/tasks/gitlab/seed.rake index 155ba979b36..d76e38b73b5 100644 --- a/lib/tasks/gitlab/seed.rake +++ b/lib/tasks/gitlab/seed.rake @@ -1,7 +1,9 @@ namespace :gitlab do namespace :seed do desc "GitLab | Seed | Seeds issues" - task :issues, [:project_full_path] => :environment do |t, args| + task :issues, [:project_full_path, :backfill_weeks, :average_issues_per_week] => :environment do |t, args| + args.with_defaults(backfill_weeks: 5, average_issues_per_week: 2) + projects = if args.project_full_path project = Project.find_by_full_path(args.project_full_path) @@ -26,7 +28,8 @@ namespace :gitlab do projects.each do |project| puts "\nSeeding issues for the '#{project.full_path}' project" seeder = Quality::Seeders::Issues.new(project: project) - issues_created = seeder.seed(backfill_weeks: 5, average_issues_per_week: 2) + issues_created = seeder.seed(backfill_weeks: args.backfill_weeks.to_i, + average_issues_per_week: args.average_issues_per_week.to_i) puts "\n#{issues_created} issues created!" end end diff --git a/lib/tasks/gitlab/setup.rake b/lib/tasks/gitlab/setup.rake index e876b23d43f..5d86d6e466c 100644 --- a/lib/tasks/gitlab/setup.rake +++ b/lib/tasks/gitlab/setup.rake @@ -31,7 +31,6 @@ namespace :gitlab do terminate_all_connections unless Rails.env.production? Rake::Task["db:reset"].invoke - Rake::Task["add_limits_mysql"].invoke Rake::Task["setup_postgresql"].invoke Rake::Task["db:seed_fu"].invoke rescue Gitlab::TaskAbortedByUserError @@ -46,8 +45,6 @@ namespace :gitlab do # method terminates all the connections so that a subsequent DROP # will work. def self.terminate_all_connections - return false unless Gitlab::Database.postgresql? - cmd = <<~SQL SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity diff --git a/lib/tasks/gitlab/storage.rake b/lib/tasks/gitlab/storage.rake index 954f827f716..ccc96b7edfb 100644 --- a/lib/tasks/gitlab/storage.rake +++ b/lib/tasks/gitlab/storage.rake @@ -3,50 +3,44 @@ namespace :gitlab do desc 'GitLab | Storage | Migrate existing projects to Hashed Storage' task migrate_to_hashed: :environment do if Gitlab::Database.read_only? - warn 'This task requires database write access. Exiting.' - - next + abort 'This task requires database write access. Exiting.' end storage_migrator = Gitlab::HashedStorage::Migrator.new helper = Gitlab::HashedStorage::RakeHelper if storage_migrator.rollback_pending? - warn "There is already a rollback operation in progress, " \ + abort "There is already a rollback operation in progress, " \ "running a migration at the same time may have unexpected consequences." - - next end if helper.range_single_item? project = Project.with_unmigrated_storage.find_by(id: helper.range_from) unless project - warn "There are no projects requiring storage migration with ID=#{helper.range_from}" - - next + abort "There are no projects requiring storage migration with ID=#{helper.range_from}" end puts "Enqueueing storage migration of #{project.full_path} (ID=#{project.id})..." storage_migrator.migrate(project) + else + legacy_projects_count = if helper.using_ranges? + Project.with_unmigrated_storage.id_in(helper.range_from..helper.range_to).count + else + Project.with_unmigrated_storage.count + end + + if legacy_projects_count == 0 + abort 'There are no projects requiring storage migration. Nothing to do!' + end - next - end - - legacy_projects_count = Project.with_unmigrated_storage.count - - if legacy_projects_count == 0 - warn 'There are no projects requiring storage migration. Nothing to do!' - - next - end - - print "Enqueuing migration of #{legacy_projects_count} projects in batches of #{helper.batch_size}" + print "Enqueuing migration of #{legacy_projects_count} projects in batches of #{helper.batch_size}" - helper.project_id_batches_migration do |start, finish| - storage_migrator.bulk_schedule_migration(start: start, finish: finish) + helper.project_id_batches_migration do |start, finish| + storage_migrator.bulk_schedule_migration(start: start, finish: finish) - print '.' + print '.' + end end puts ' Done!' @@ -55,50 +49,44 @@ namespace :gitlab do desc 'GitLab | Storage | Rollback existing projects to Legacy Storage' task rollback_to_legacy: :environment do if Gitlab::Database.read_only? - warn 'This task requires database write access. Exiting.' - - next + abort 'This task requires database write access. Exiting.' end storage_migrator = Gitlab::HashedStorage::Migrator.new helper = Gitlab::HashedStorage::RakeHelper if storage_migrator.migration_pending? - warn "There is already a migration operation in progress, " \ + abort "There is already a migration operation in progress, " \ "running a rollback at the same time may have unexpected consequences." - - next end if helper.range_single_item? project = Project.with_storage_feature(:repository).find_by(id: helper.range_from) unless project - warn "There are no projects that can be rolledback with ID=#{helper.range_from}" - - next + abort "There are no projects that can be rolledback with ID=#{helper.range_from}" end puts "Enqueueing storage rollback of #{project.full_path} (ID=#{project.id})..." storage_migrator.rollback(project) + else + hashed_projects_count = if helper.using_ranges? + Project.with_storage_feature(:repository).id_in(helper.range_from..helper.range_to).count + else + Project.with_storage_feature(:repository).count + end + + if hashed_projects_count == 0 + abort 'There are no projects that can have storage rolledback. Nothing to do!' + end - next - end - - hashed_projects_count = Project.with_storage_feature(:repository).count - - if hashed_projects_count == 0 - warn 'There are no projects that can have storage rolledback. Nothing to do!' - - next - end - - print "Enqueuing rollback of #{hashed_projects_count} projects in batches of #{helper.batch_size}" + print "Enqueuing rollback of #{hashed_projects_count} projects in batches of #{helper.batch_size}" - helper.project_id_batches_rollback do |start, finish| - storage_migrator.bulk_schedule_rollback(start: start, finish: finish) + helper.project_id_batches_rollback do |start, finish| + storage_migrator.bulk_schedule_rollback(start: start, finish: finish) - print '.' + print '.' + end end puts ' Done!' diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake index e058e9fe069..fdcd34320b1 100644 --- a/lib/tasks/gitlab/update_templates.rake +++ b/lib/tasks/gitlab/update_templates.rake @@ -5,25 +5,42 @@ namespace :gitlab do end desc "GitLab | Update project templates" - task :update_project_templates do - include Gitlab::ImportExport::CommandLineUtil + task :update_project_templates, [] => :environment do |_task, args| + # we need an instance method from Gitlab::ImportExport::CommandLineUtil and don't + # want to include it in the task, as this would affect subsequent tasks as well + downloader = Class.new do + extend Gitlab::ImportExport::CommandLineUtil + + def self.call(uploader, upload_path) + download_or_copy_upload(uploader, upload_path) + end + end + + template_names = args.extras.to_set if Rails.env.production? - puts "This rake task is not meant fo production instances".red - exit(1) + raise "This rake task is not meant for production instances" end admin = User.find_by(admin: true) unless admin - puts "No admin user could be found".red - exit(1) + raise "No admin user could be found" end - Gitlab::ProjectTemplate.all.each do |template| + tmp_namespace_path = "tmp-project-import-#{Time.now.to_i}" + puts "Creating temporary namespace #{tmp_namespace_path}" + tmp_namespace = Namespace.create!(owner: admin, name: tmp_namespace_path, path: tmp_namespace_path) + + templates = if template_names.empty? + Gitlab::ProjectTemplate.all + else + Gitlab::ProjectTemplate.all.select { |template| template_names.include?(template.name) } + end + + templates.each do |template| params = { - import_url: template.clone_url, - namespace_id: admin.namespace.id, + namespace_id: tmp_namespace.id, path: template.name, skip_wiki: true } @@ -32,33 +49,67 @@ namespace :gitlab do project = Projects::CreateService.new(admin, params).execute unless project.persisted? - puts project.errors.messages - exit(1) + raise "Failed to create project: #{project.errors.messages}" end - loop do - if project.finished? - puts "Import finished for #{template.name}" - break + uri_encoded_project_path = template.uri_encoded_project_path + + # extract a concrete commit for signing off what we actually downloaded + # this way we do the right thing even if the repository gets updated in the meantime + get_commits_response = Gitlab::HTTP.get("https://gitlab.com/api/v4/projects/#{uri_encoded_project_path}/repository/commits", + query: { page: 1, per_page: 1 } + ) + raise "Failed to retrieve latest commit for template '#{template.name}'" unless get_commits_response.success? + + commit_sha = get_commits_response.parsed_response.dig(0, 'id') + + project_archive_uri = "https://gitlab.com/api/v4/projects/#{uri_encoded_project_path}/repository/archive.tar.gz?sha=#{commit_sha}" + commit_message = <<~MSG + Initialized from '#{template.title}' project template + + Template repository: #{template.preview} + Commit SHA: #{commit_sha} + MSG + + Dir.mktmpdir do |tmpdir| + Dir.chdir(tmpdir) do + Gitlab::TaskHelpers.run_command!(['wget', project_archive_uri, '-O', 'archive.tar.gz']) + Gitlab::TaskHelpers.run_command!(['tar', 'xf', 'archive.tar.gz']) + extracted_project_basename = Dir['*/'].first + Dir.chdir(extracted_project_basename) do + Gitlab::TaskHelpers.run_command!(%w(git init)) + Gitlab::TaskHelpers.run_command!(%w(git add .)) + Gitlab::TaskHelpers.run_command!(['git', 'commit', '--author', 'GitLab <root@localhost>', '--message', commit_message]) + + # Hacky workaround to push to the project in a way that works with both GDK and the test environment + Gitlab::GitalyClient::StorageSettings.allow_disk_access do + Gitlab::TaskHelpers.run_command!(['git', 'remote', 'add', 'origin', "file://#{project.repository.raw.path}"]) + end + Gitlab::TaskHelpers.run_command!(['git', 'push', '-u', 'origin', 'master']) + end end + end - if project.failed? - puts "Failed to import from #{project_params[:import_url]}".red - exit(1) - end + project.reset - puts "Waiting for the import to finish" + Projects::ImportExport::ExportService.new(project, admin).execute + downloader.call(project.export_file, template.archive_path) - sleep(5) - project.reset + unless Projects::DestroyService.new(project, admin).execute + puts "Failed to destroy project #{template.name} (but namespace will be cleaned up later)" end - Projects::ImportExport::ExportService.new(project, admin).execute - download_or_copy_upload(project.export_file, template.archive_path) - Projects::DestroyService.new(admin, project).execute puts "Exported #{template.name}".green end - puts "Done".green + + success = true + ensure + if tmp_namespace + puts "Destroying temporary namespace #{tmp_namespace_path}" + tmp_namespace.destroy + end + + puts "Done".green if success end def update(template) diff --git a/lib/tasks/gitlab/uploads/legacy.rake b/lib/tasks/gitlab/uploads/legacy.rake new file mode 100644 index 00000000000..18fb8afe455 --- /dev/null +++ b/lib/tasks/gitlab/uploads/legacy.rake @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +namespace :gitlab do + namespace :uploads do + namespace :legacy do + desc "GitLab | Uploads | Migrate all legacy attachments" + task migrate: :environment do + class Upload < ApplicationRecord + self.table_name = 'uploads' + + include ::EachBatch + end + + migration = 'LegacyUploadsMigrator'.freeze + batch_size = 5000 + delay_interval = 5.minutes.to_i + + Upload.where(uploader: 'AttachmentUploader').each_batch(of: batch_size) do |relation, index| + start_id, end_id = relation.pluck('MIN(id), MAX(id)').first + delay = index * delay_interval + + BackgroundMigrationWorker.perform_in(delay, migration, [start_id, end_id]) + end + end + end + end +end diff --git a/lib/tasks/karma.rake b/lib/tasks/karma.rake index 2dc14183fa3..36590010406 100644 --- a/lib/tasks/karma.rake +++ b/lib/tasks/karma.rake @@ -1,15 +1,8 @@ unless Rails.env.production? namespace :karma do + # alias exists for legacy reasons desc 'GitLab | Karma | Generate fixtures for JavaScript tests' - task fixtures: ['karma:rspec_fixtures'] - - desc 'GitLab | Karma | Generate fixtures using RSpec' - RSpec::Core::RakeTask.new(:rspec_fixtures, [:pattern]) do |t, args| - args.with_defaults(pattern: '{spec,ee/spec}/javascripts/fixtures/*.rb') - ENV['NO_KNAPSACK'] = 'true' - t.pattern = args[:pattern] - t.rspec_opts = '--format documentation' - end + task fixtures: ['frontend:fixtures'] desc 'GitLab | Karma | Run JavaScript tests' task tests: ['yarn:check'] do diff --git a/lib/tasks/migrate/add_limits_mysql.rake b/lib/tasks/migrate/add_limits_mysql.rake deleted file mode 100644 index c77fa49d586..00000000000 --- a/lib/tasks/migrate/add_limits_mysql.rake +++ /dev/null @@ -1,17 +0,0 @@ -require Rails.root.join('db/migrate/limits_to_mysql') -require Rails.root.join('db/migrate/markdown_cache_limits_to_mysql') -require Rails.root.join('db/migrate/merge_request_diff_file_limits_to_mysql') -require Rails.root.join('db/migrate/limits_ci_build_trace_chunks_raw_data_for_mysql') -require Rails.root.join('db/migrate/gpg_keys_limits_to_mysql') -require Rails.root.join('db/migrate/prometheus_metrics_limits_to_mysql') - -desc "GitLab | Add limits to strings in mysql database" -task add_limits_mysql: :environment do - puts "Adding limits to schema.rb for mysql" - LimitsToMysql.new.up - MarkdownCacheLimitsToMysql.new.up - MergeRequestDiffFileLimitsToMysql.new.up - LimitsCiBuildTraceChunksRawDataForMysql.new.up - IncreaseMysqlTextLimitForGpgKeys.new.up - PrometheusMetricsLimitsToMysql.new.up -end diff --git a/lib/tasks/migrate/schema_check.rake b/lib/tasks/migrate/schema_check.rake new file mode 100644 index 00000000000..76f1f23c7bd --- /dev/null +++ b/lib/tasks/migrate/schema_check.rake @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +# Configures the database by running migrate, or by loading the schema and seeding if needed +task schema_version_check: :environment do + next if ENV['SKIP_SCHEMA_VERSION_CHECK'] + + schema_version = ActiveRecord::Migrator.current_version + + # Ensure migrations are being run from a supported schema version + # A schema verison of 0 is a fresh db, and should be safe to run migrations + # But a database with existing migrations less than our min version is not + if schema_version > 0 && schema_version < Gitlab::Database::MIN_SCHEMA_VERSION + raise "Your current database version is too old to be migrated. " \ + "You should upgrade to GitLab #{Gitlab::Database::MIN_SCHEMA_GITLAB_VERSION} before moving to this version. " \ + "Please see https://docs.gitlab.com/ee/policy/maintenance.html#upgrade-recommendations" + end +end + +# Ensure the check is a pre-requisite when running db:migrate +Rake::Task["db:migrate"].enhance [:schema_version_check] diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake index f69d204c579..cda88c130bb 100644 --- a/lib/tasks/migrate/setup_postgresql.rake +++ b/lib/tasks/migrate/setup_postgresql.rake @@ -1,23 +1,9 @@ desc 'GitLab | Sets up PostgreSQL' task setup_postgresql: :environment do - require Rails.root.join('db/migrate/20151007120511_namespaces_projects_path_lower_indexes') - require Rails.root.join('db/migrate/20151008110232_add_users_lower_username_email_indexes') - require Rails.root.join('db/migrate/20161212142807_add_lower_path_index_to_routes') - require Rails.root.join('db/migrate/20170317203554_index_routes_path_for_like') - require Rails.root.join('db/migrate/20170724214302_add_lower_path_index_to_redirect_routes') - require Rails.root.join('db/migrate/20170503185032_index_redirect_routes_path_for_like') - require Rails.root.join('db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb') require Rails.root.join('db/migrate/20180215181245_users_name_lower_index.rb') require Rails.root.join('db/migrate/20180504195842_project_name_lower_index.rb') require Rails.root.join('db/post_migrate/20180306164012_add_path_index_to_redirect_routes.rb') - NamespacesProjectsPathLowerIndexes.new.up - AddUsersLowerUsernameEmailIndexes.new.up - AddLowerPathIndexToRoutes.new.up - IndexRoutesPathForLike.new.up - AddLowerPathIndexToRedirectRoutes.new.up - IndexRedirectRoutesPathForLike.new.up - AddIndexOnNamespacesLowerName.new.up UsersNameLowerIndex.new.up ProjectNameLowerIndex.new.up AddPathIndexToRedirectRoutes.new.up diff --git a/lib/tasks/services.rake b/lib/tasks/services.rake index 56b81106c5f..4ec4fdd281f 100644 --- a/lib/tasks/services.rake +++ b/lib/tasks/services.rake @@ -86,7 +86,7 @@ namespace :services do doc_start = Time.now doc_path = File.join(Rails.root, 'doc', 'api', 'services.md') - result = ERB.new(services_template, 0, '>') + result = ERB.new(services_template, trim_mode: '>') .result(OpenStruct.new(services: services).instance_eval { binding }) File.open(doc_path, 'w') do |f| diff --git a/lib/tasks/spec.rake b/lib/tasks/spec.rake index c881ad4cf12..bf18332a8eb 100644 --- a/lib/tasks/spec.rake +++ b/lib/tasks/spec.rake @@ -2,8 +2,6 @@ return if Rails.env.production? -Rake::Task["spec"].clear if Rake::Task.task_defined?('spec') - namespace :spec do desc 'GitLab | RSpec | Run unit tests' RSpec::Core::RakeTask.new(:unit, :rspec_opts) do |t, args| @@ -26,63 +24,8 @@ namespace :spec do t.rspec_opts = args[:rspec_opts] end - desc '[Deprecated] Use the "bin/rspec --tag api" instead' - task :api do - cmds = [ - %w(rake gitlab:setup), - %w(rspec spec --tag @api) - ] - run_commands(cmds) - end - - desc '[Deprecated] Use the "spec:system" task instead' - task :feature do - cmds = [ - %w(rake gitlab:setup), - %w(rspec spec --tag @feature) - ] - run_commands(cmds) - end - - desc '[Deprecated] Use "bin/rspec spec/models" instead' - task :models do - cmds = [ - %w(rake gitlab:setup), - %w(rspec spec --tag @models) - ] - run_commands(cmds) - end - - desc '[Deprecated] Use "bin/rspec spec/services" instead' - task :services do - cmds = [ - %w(rake gitlab:setup), - %w(rspec spec --tag @services) - ] - run_commands(cmds) - end - - desc '[Deprecated] Use "bin/rspec spec/lib" instead' - task :lib do - cmds = [ - %w(rake gitlab:setup), - %w(rspec spec --tag @lib) - ] - run_commands(cmds) - end -end - -desc "GitLab | Run specs" -task :spec do - cmds = [ - %w(rake gitlab:setup), - %w(rspec spec) - ] - run_commands(cmds) -end - -def run_commands(cmds) - cmds.each do |cmd| - system({ 'RAILS_ENV' => 'test', 'force' => 'yes' }, *cmd) || raise("#{cmd} failed!") + desc 'Run the code examples in spec/requests/api' + RSpec::Core::RakeTask.new(:api) do |t| + t.pattern = 'spec/requests/api/**/*_spec.rb' end end diff --git a/lib/tasks/yarn.rake b/lib/tasks/yarn.rake index 2ac88a039e7..32061ad4a57 100644 --- a/lib/tasks/yarn.rake +++ b/lib/tasks/yarn.rake @@ -24,7 +24,7 @@ namespace :yarn do desc 'Install Node dependencies with Yarn' task install: ['yarn:available'] do - unless system('yarn install --pure-lockfile --ignore-engines') + unless system('yarn install --pure-lockfile --ignore-engines --prefer-offline') abort 'Error: Unable to install node modules.'.color(:red) end end |