Merge branch 'master' of gitlab.com:gitlab-org/gitlab-ci

23 files changed, 296 insertions, 85 deletions

diff --git a/.gitignore b/.gitignore
index 794c1ec..6c806ae 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,3 +17,4 @@ tmp/*
 /db/*.sqlite3
 /log/*.log
 /.idea
+/builds/*
diff --git a/CHANGELOG b/CHANGELOG
index e64281d..a1a24e0 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -9,9 +9,11 @@ v7.13.0
   - Redirect back after authorization
   - Change favicon
   - Refactoring: Get rid of private_token usage in the frontend.
+  - Allow to specify allow_failure for job
 
 v7.12.2
   - Revert: Runner without tag should pick builds without tag only
+  - Build traces is stored in the file instead of database
 
 v7.12.1
   - Runner without tag should pick builds without tag only
diff --git a/app/models/build.rb b/app/models/build.rb
index 725da75..5e65e26 100644
--- a/app/models/build.rb
+++ b/app/models/build.rb
@@ -2,22 +2,23 @@
 #
 # Table name: builds
 #
-#  id          :integer          not null, primary key
-#  project_id  :integer
-#  status      :string(255)
-#  finished_at :datetime
-#  trace       :text
-#  created_at  :datetime
-#  updated_at  :datetime
-#  started_at  :datetime
-#  runner_id   :integer
-#  commit_id   :integer
-#  coverage    :float
-#  commands    :text
-#  options     :text
-#  job_id      :integer
-#  name        :string(255)
-#  deploy      :boolean          default(FALSE)
+#  id            :integer          not null, primary key
+#  project_id    :integer
+#  status        :string(255)
+#  finished_at   :datetime
+#  trace         :text
+#  created_at    :datetime
+#  updated_at    :datetime
+#  started_at    :datetime
+#  runner_id     :integer
+#  commit_id     :integer
+#  coverage      :float
+#  commands      :text
+#  job_id        :integer
+#  name          :string(255)
+#  deploy        :boolean          default(FALSE)
+#  options       :text
+#  allow_failure :boolean          default(FALSE), not null
 #
 
 class Build < ActiveRecord::Base
@@ -75,6 +76,7 @@ class Build < ActiveRecord::Base
       new_build.commit_id = build.commit_id
       new_build.project_id = build.project_id
       new_build.name = build.name
+      new_build.allow_failure = build.allow_failure
       new_build.save
       new_build
     end
@@ -153,6 +155,10 @@ class Build < ActiveRecord::Base
     canceled? || success? || failed?
   end
 
+  def ignored?
+    failed? && allow_failure?
+  end
+
   def timeout
     project.timeout
   end
@@ -210,4 +216,37 @@ class Build < ActiveRecord::Base
       # so we just silentrly ignore error for now
     end
   end
+
+  def trace
+    if File.exist?(path_to_trace)
+      File.read(path_to_trace)
+    else
+      # backward compatibility
+      read_attribute :trace
+    end
+  end
+
+  def trace=(trace)
+    unless Dir.exists? dir_to_trace
+      FileUtils.mkdir_p dir_to_trace
+    end
+
+    File.write(path_to_trace, trace)
+  end
+
+  def dir_to_trace
+    Rails.root.join(
+      root_dir_to_trace,
+      created_at.utc.strftime("%Y_%m"),
+      project.id.to_s
+    )
+  end
+
+  def root_dir_to_trace
+    "builds"
+  end
+
+  def path_to_trace
+    "#{dir_to_trace}/#{id}.log"
+  end
 end
diff --git a/app/models/commit.rb b/app/models/commit.rb
index d9774d3..3680220 100644
--- a/app/models/commit.rb
+++ b/app/models/commit.rb
@@ -111,7 +111,8 @@ class Commit < ActiveRecord::Base
         name: build_attrs[:name],
         commands: build_attrs[:script],
         tag_list: build_attrs[:tags],
-        options: build_attrs[:options]
+        options: build_attrs[:options],
+        allow_failure: build_attrs[:allow_failure]
       })
     end
   end
@@ -149,6 +150,7 @@ class Commit < ActiveRecord::Base
         commands: build_attrs[:script],
         tag_list: build_attrs[:tags],
         options: build_attrs[:options],
+        allow_failure: build_attrs[:allow_failure],
         deploy: true
       })
     end
@@ -186,7 +188,7 @@ class Commit < ActiveRecord::Base
 
   def success?
     builds_without_retry.all? do |build|
-      build.success?
+      build.success? || build.ignored?
     end
   end
 
diff --git a/app/views/builds/_build.html.haml b/app/views/builds/_build.html.haml
index b71111b..f2a340d 100644
--- a/app/views/builds/_build.html.haml
+++ b/app/views/builds/_build.html.haml
@@ -17,11 +17,13 @@
       #{build.short_sha}
   %td
     = build.name
-    - if build.tags.any?
-      - build.tag_list.each do |tag|
-        %span.label.label-primary
-          = tag
-
+    .pull-right
+      - if build.tags.any?
+        - build.tag_list.each do |tag|
+          %span.label.label-primary
+            = tag
+      - if build.allow_failure
+        %span.label.label-danger allowed to fail
 
   %td.duration
     - if build.duration
diff --git a/builds/.gitkeep b/builds/.gitkeep
new file mode 100644
index 0000000..45adbb2
--- /dev/null
+++ b/builds/.gitkeep
@@ -0,0 +1 @@
+.gitkeep
\ No newline at end of file
diff --git a/db/migrate/20150707134456_add_allow_failure_to_builds.rb b/db/migrate/20150707134456_add_allow_failure_to_builds.rb
new file mode 100644
index 0000000..cc3da34
--- /dev/null
+++ b/db/migrate/20150707134456_add_allow_failure_to_builds.rb
@@ -0,0 +1,5 @@
+class AddAllowFailureToBuilds < ActiveRecord::Migration
+  def change
+    add_column :builds, :allow_failure, :boolean, default: false, null: false
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index c5e59b2..6b88c7f 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20150706103229) do
+ActiveRecord::Schema.define(version: 20150707134456) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -30,8 +30,9 @@ ActiveRecord::Schema.define(version: 20150706103229) do
     t.text     "commands"
     t.integer  "job_id"
     t.string   "name"
-    t.boolean  "deploy",      default: false
+    t.boolean  "deploy",        default: false
     t.text     "options"
+    t.boolean  "allow_failure", default: false, null: false
   end
 
   add_index "builds", ["commit_id"], name: "index_builds_on_commit_id", using: :btree
diff --git a/doc/install/installation.md b/doc/install/installation.md
index 4d85fb4..ba9c9ba 100644
--- a/doc/install/installation.md
+++ b/doc/install/installation.md
@@ -129,6 +129,9 @@ We recommend PostgreSQL but you can also use MySQL
     sudo -u gitlab_ci -H mkdir -p tmp/pids/
     sudo chmod -R u+rwX  tmp/pids/
 
+    # Make sure GitLab CI can write to the builds/ directory
+    sudo chmod -R u+rwX  builds
+
 ### Install gems
 
     # For MySQL (note, the option says "without ... postgres")
diff --git a/doc/raketasks/backup_restore.md b/doc/raketasks/backup_restore.md
index 73ca73e..3da3f26 100644
--- a/doc/raketasks/backup_restore.md
+++ b/doc/raketasks/backup_restore.md
@@ -2,7 +2,7 @@
 
 ## Create a backup of the GitLab CI
 
-A backup creates an archive file that contains the database.
+A backup creates an archive file that contains the database and builds files.
 This archive will be saved in backup_path (see `config/application.yml`).
 The filename will be `[TIMESTAMP]_gitlab_ci_backup.tar.gz`. This timestamp can be used to restore an specific backup.
 You can only restore a backup to exactly the same version of GitLab CI that you created it on, for example 7.10.1.
@@ -24,6 +24,8 @@ Example output:
 Dumping database ...
 Dumping PostgreSQL database gitlab_ci_development ... [DONE]
 done
+Dumping builds ...
+done
 Creating backup archive: 1430930060_gitlab_ci_backup.tar.gz ... done
 Uploading backup archive to remote storage  ... skipped
 Deleting tmp directories ... done
diff --git a/doc/update/7.12-to-7.13.md b/doc/update/7.12-to-7.13.md
new file mode 100644
index 0000000..e3600eb
--- /dev/null
+++ b/doc/update/7.12-to-7.13.md
@@ -0,0 +1,51 @@
+# Update from 7.12 to 7.13
+
+## Notice
+
+__GitLab CI 7.13 requires GitLab 7.12 or higher and GitLab Multi Runner 0.4.0 or higher
+
+### 1. Stop CI server
+
+    sudo service gitlab_ci stop
+
+### 2. Switch to your gitlab_ci user
+
+```
+sudo su gitlab_ci
+cd /home/gitlab_ci/gitlab-ci
+```
+
+### 3. Get latest code
+
+```
+git fetch
+git checkout 7-13-stable
+```
+
+### 4. Make sure GitLab CI can write to the builds/ directory
+
+```
+sudo chmod -R u+rwX  builds
+```
+
+### 5. Install libs, migrations etc
+
+
+```
+# Install nodejs dependency:
+sudo apt-get install nodejs
+
+# For MySQL users
+bundle install --without postgres development test --deployment
+
+# For Postgres users
+bundle install --without mysql development test --deployment
+
+# Run migrations
+bundle exec rake db:migrate RAILS_ENV=production
+```
+
+
+### 5. Start web application
+
+    sudo service gitlab_ci start
diff --git a/lib/backup/builds.rb b/lib/backup/builds.rb
new file mode 100644
index 0000000..71e9704
--- /dev/null
+++ b/lib/backup/builds.rb
@@ -0,0 +1,30 @@
+module Backup
+  class Builds
+    attr_reader :app_builds_dir, :backup_builds_dir, :backup_dir
+
+    def initialize
+      @app_builds_dir = File.realpath(Rails.root.join('builds'))
+      @backup_dir = GitlabCi.config.backup.path
+      @backup_builds_dir = File.join(GitlabCi.config.backup.path, 'builds')
+    end
+
+    # Copy builds from builds directory to backup/builds
+    def dump
+      FileUtils.mkdir_p(backup_builds_dir)
+      FileUtils.cp_r(app_builds_dir, backup_dir)
+    end
+
+    def restore
+      backup_existing_builds_dir
+
+      FileUtils.cp_r(backup_builds_dir, app_builds_dir)
+    end
+
+    def backup_existing_builds_dir
+      timestamped_builds_path = File.join(app_builds_dir, '..', "builds.#{Time.now.to_i}")
+      if File.exists?(app_builds_dir)
+        FileUtils.mv(app_builds_dir, File.expand_path(timestamped_builds_path))
+      end
+    end
+  end
+end
diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb
index fc847e2..43fb362 100644
--- a/lib/backup/manager.rb
+++ b/lib/backup/manager.rb
@@ -15,7 +15,7 @@ module Backup
           file << s.to_yaml.gsub(/^---\n/,'')
         end
 
-        FileUtils.chmod(0700, "db")
+        FileUtils.chmod(0700, ["db", "builds"])
 
         # create archive
         $progress.print "Creating backup archive: #{tar_file} ... "
@@ -146,7 +146,7 @@ module Backup
     private
 
     def backup_contents
-      ["db", "backup_information.yml"]
+      ["db", "builds", "backup_information.yml"]
     end
 
     def settings
diff --git a/lib/gitlab_ci_yaml_processor.rb b/lib/gitlab_ci_yaml_processor.rb
index 00fceda..b055d81 100644
--- a/lib/gitlab_ci_yaml_processor.rb
+++ b/lib/gitlab_ci_yaml_processor.rb
@@ -84,6 +84,7 @@ class GitlabCiYamlProcessor
       name: name,
       only: job[:only],
       except: job[:except],
+      allow_failure: job[:allow_failure] || false,
       options: {
         image: job[:image] || @image,
         services: job[:services] || @services
@@ -133,7 +134,7 @@ class GitlabCiYamlProcessor
 
   def validate_job!(name, job)
     job.keys.each do |key|
-      unless [:tags, :script, :only, :except, :type, :image, :services].include? key
+      unless [:tags, :script, :only, :except, :type, :image, :services, :allow_failure].include? key
         raise ValidationError, "#{name}: unknown parameter #{key}"
       end
     end
@@ -159,5 +160,9 @@ class GitlabCiYamlProcessor
     if job[:except] && !job[:except].is_a?(Array)
       raise ValidationError, "#{name}: except parameter should be an array"
     end
+
+    if job[:allow_failure] && !job[:allow_failure].in?([true, false])
+      raise ValidationError, "#{name}: allow_failure parameter should be an boolean"
+    end
   end
 end
diff --git a/lib/tasks/backup.rake b/lib/tasks/backup.rake
index 3431bd1..df20c40 100644
--- a/lib/tasks/backup.rake
+++ b/lib/tasks/backup.rake
@@ -5,10 +5,13 @@ namespace :backup do
     configure_cron_mode
 
     $progress.puts "Dumping database ... ".blue
-
     Backup::Database.new.dump
     $progress.puts "done".green
 
+    $progress.puts "Dumping builds ... ".blue
+    Backup::Builds.new.dump
+    $progress.puts "done".green
+
     backup = Backup::Manager.new
     backup.pack
     backup.cleanup
@@ -26,6 +29,10 @@ namespace :backup do
     Backup::Database.new.restore
     $progress.puts "done".green
 
+    $progress.puts "Restoring builds ... ".blue
+    Backup::Builds.new.restore
+    $progress.puts "done".green
+
     backup.cleanup
   end
 
diff --git a/spec/factories/builds.rb b/spec/factories/builds.rb
index bddf6eb..af63bbd 100644
--- a/spec/factories/builds.rb
+++ b/spec/factories/builds.rb
@@ -2,22 +2,23 @@
 #
 # Table name: builds
 #
-#  id          :integer          not null, primary key
-#  project_id  :integer
-#  status      :string(255)
-#  finished_at :datetime
-#  trace       :text
-#  created_at  :datetime
-#  updated_at  :datetime
-#  started_at  :datetime
-#  runner_id   :integer
-#  commit_id   :integer
-#  coverage    :float
-#  commands    :text
-#  options     :text
-#  job_id      :integer
-#  name        :string(255)
-#  deploy      :boolean          default(FALSE)
+#  id            :integer          not null, primary key
+#  project_id    :integer
+#  status        :string(255)
+#  finished_at   :datetime
+#  trace         :text
+#  created_at    :datetime
+#  updated_at    :datetime
+#  started_at    :datetime
+#  runner_id     :integer
+#  commit_id     :integer
+#  coverage      :float
+#  commands      :text
+#  job_id        :integer
+#  name          :string(255)
+#  deploy        :boolean          default(FALSE)
+#  options       :text
+#  allow_failure :boolean          default(FALSE), not null
 #
 
 # Read about factories at https://github.com/thoughtbot/factory_girl
diff --git a/spec/factories/commits.rb b/spec/factories/commits.rb
index 1f4a383..4a411ee 100644
--- a/spec/factories/commits.rb
+++ b/spec/factories/commits.rb
@@ -49,5 +49,17 @@ FactoryGirl.define do
         ci_yaml_file: File.read(Rails.root.join('spec/support/gitlab_stubs/gitlab_ci.yml'))
       }
     end
+
+    factory :commit_with_one_job do
+      after(:create) do |commit, evaluator|
+        commit.push_data[:ci_yaml_file] = YAML.dump({rspec: { script: "ls" }})
+      end
+    end
+
+    factory :commit_with_two_jobs do
+      after(:create) do |commit, evaluator|
+        commit.push_data[:ci_yaml_file] = YAML.dump({rspec: { script: "ls" }, spinach: { script: "ls" }})
+      end
+    end
   end
 end
diff --git a/spec/lib/gitlab_ci_yaml_processor_spec.rb b/spec/lib/gitlab_ci_yaml_processor_spec.rb
index 9c9fde9..5f202b2 100644
--- a/spec/lib/gitlab_ci_yaml_processor_spec.rb
+++ b/spec/lib/gitlab_ci_yaml_processor_spec.rb
@@ -18,7 +18,8 @@ describe GitlabCiYamlProcessor do
         only: nil,
         script: "pwd\nrspec",
         tags: [],
-        options: {}
+        options: {},
+        allow_failure: false
       }
     end
 
@@ -71,7 +72,7 @@ describe GitlabCiYamlProcessor do
     it "returns builds if no branch specified" do
       config = YAML.dump({
         before_script: ["pwd"],
-        rspec: {script: "rspec", type: "deploy"}
+        rspec: {script: "rspec", type: "deploy", allow_failure: true}
       })
 
       config_processor = GitlabCiYamlProcessor.new(config)
@@ -83,7 +84,8 @@ describe GitlabCiYamlProcessor do
         only: nil,
         script: "pwd\nrspec",
         tags: [],
-        options: {}
+        options: {},
+        allow_failure: true
       }
     end
 
@@ -153,7 +155,8 @@ describe GitlabCiYamlProcessor do
         options: {
           image: "ruby:2.1",
           services: ["mysql"]
-        }
+        },
+        allow_failure: false
       }
     end
 
@@ -177,7 +180,8 @@ describe GitlabCiYamlProcessor do
         options: {
           image: "ruby:2.5",
           services: ["postgresql"]
-        }
+        },
+        allow_failure: false
       }
     end
   end
@@ -256,5 +260,12 @@ describe GitlabCiYamlProcessor do
         GitlabCiYamlProcessor.new(config)
       end.to raise_error(GitlabCiYamlProcessor::ValidationError, "Please define at least one job")
     end
+
+    it "returns errors if job allow_failure parameter is not an boolean" do
+      config = YAML.dump({rspec: {script: "test", allow_failure: "string"}})
+      expect do
+        GitlabCiYamlProcessor.new(config)
+      end.to raise_error(GitlabCiYamlProcessor::ValidationError, "rspec job: allow_failure parameter should be an boolean")
+    end
   end
 end
\ No newline at end of file
diff --git a/spec/models/build_spec.rb b/spec/models/build_spec.rb
index 8bc9807..7e1c7e9 100644
--- a/spec/models/build_spec.rb
+++ b/spec/models/build_spec.rb
@@ -2,22 +2,23 @@
 #
 # Table name: builds
 #
-#  id          :integer          not null, primary key
-#  project_id  :integer
-#  status      :string(255)
-#  finished_at :datetime
-#  trace       :text
-#  created_at  :datetime
-#  updated_at  :datetime
-#  started_at  :datetime
-#  runner_id   :integer
-#  commit_id   :integer
-#  coverage    :float
-#  commands    :text
-#  options     :text
-#  job_id      :integer
-#  name        :string(255)
-#  deploy      :boolean          default(FALSE)
+#  id            :integer          not null, primary key
+#  project_id    :integer
+#  status        :string(255)
+#  finished_at   :datetime
+#  trace         :text
+#  created_at    :datetime
+#  updated_at    :datetime
+#  started_at    :datetime
+#  runner_id     :integer
+#  commit_id     :integer
+#  coverage      :float
+#  commands      :text
+#  job_id        :integer
+#  name          :string(255)
+#  deploy        :boolean          default(FALSE)
+#  options       :text
+#  allow_failure :boolean          default(FALSE), not null
 #
 
 require 'spec_helper'
@@ -126,6 +127,42 @@ describe Build do
     end
   end
 
+  describe :ignored? do
+    subject { build.ignored? }
+
+    context 'if build is not allowed to fail' do
+      before { build.allow_failure = false }
+
+      context 'and build.status is success' do
+        before { build.status = 'success' }
+
+        it { should be_false }
+      end
+
+      context 'and build.status is failed' do
+        before { build.status = 'failed' }
+
+        it { should be_false }
+      end
+    end
+
+    context 'if build is allowed to fail' do
+      before { build.allow_failure = true }
+
+      context 'and build.status is success' do
+        before { build.status = 'success' }
+
+        it { should be_false }
+      end
+
+      context 'and build.status is failed' do
+        before { build.status = 'failed' }
+
+        it { should be_true }
+      end
+    end
+  end
+
   describe :trace do
     subject { build.trace_html }
 
diff --git a/spec/models/project_services/hip_chat_message_spec.rb b/spec/models/project_services/hip_chat_message_spec.rb
index 1afe3ed..f1ad875 100644
--- a/spec/models/project_services/hip_chat_message_spec.rb
+++ b/spec/models/project_services/hip_chat_message_spec.rb
@@ -6,12 +6,7 @@ describe HipChatMessage do
   let(:project) { FactoryGirl.create(:project) }
 
   context "One build" do
-    let(:commit) do
-      commit = FactoryGirl.create(:commit, project: project)
-      commit.push_data[:ci_yaml_file] = YAML.dump({rspec: { script: 'pwd' }})
-      commit.save
-      commit
-    end
+    let(:commit) { FactoryGirl.create(:commit_with_one_job, project: project) }
 
     let(:build) do 
       commit.create_builds
@@ -42,7 +37,7 @@ describe HipChatMessage do
   end
 
   context "Several builds" do
-    let(:commit) {commit = FactoryGirl.create(:commit, project: project)}
+    let(:commit) { FactoryGirl.create(:commit_with_two_jobs, project: project) }
 
     let(:build) do 
       commit.builds.first
diff --git a/spec/models/project_services/slack_message_spec.rb b/spec/models/project_services/slack_message_spec.rb
index f60f89c..88e0f37 100644
--- a/spec/models/project_services/slack_message_spec.rb
+++ b/spec/models/project_services/slack_message_spec.rb
@@ -6,12 +6,7 @@ describe SlackMessage do
   let(:project) { FactoryGirl.create :project }
 
   context "One build" do
-    let(:commit) do
-      commit = FactoryGirl.create(:commit, project: project)
-      commit.push_data[:ci_yaml_file] = YAML.dump({rspec: { script: "ls" }})
-      commit.save
-      commit
-    end
+    let(:commit) { FactoryGirl.create(:commit_with_one_job, project: project) }
 
     let(:build) do 
       commit.create_builds
@@ -48,11 +43,7 @@ describe SlackMessage do
   end
 
   context "Several builds" do
-    let(:commit) {commit = FactoryGirl.create(:commit, project: project)}
-
-    let(:build) do 
-      commit.builds.first
-    end
+    let(:commit) { FactoryGirl.create(:commit_with_two_jobs, project: project) }
 
     context 'when all matrix builds succeeded' do
       let(:color) { 'good' }
diff --git a/spec/support/gitlab_stubs/gitlab_ci.yml b/spec/support/gitlab_stubs/gitlab_ci.yml
index 4f0f106..4095d6a 100644
--- a/spec/support/gitlab_stubs/gitlab_ci.yml
+++ b/spec/support/gitlab_stubs/gitlab_ci.yml
@@ -17,6 +17,7 @@ rspec:
 
 spinach:
   script: "rake spinach"
+  allow_failure: true
   tags: 
     - ruby
     - mysql
diff --git a/spec/support/setup_builds_storage.rb b/spec/support/setup_builds_storage.rb
new file mode 100644
index 0000000..ab93471
--- /dev/null
+++ b/spec/support/setup_builds_storage.rb
@@ -0,0 +1,12 @@
+RSpec.configure do |config|
+  config.before(:each) do
+    FileUtils.mkdir_p("tmp/builds_test")
+    Build.any_instance.stub(:root_dir_to_trace).and_return("tmp/builds_test")
+  end
+
+  config.after(:suite) do
+    Dir.chdir(Rails.root.join("tmp/builds_test")) do
+      `ls | grep -v .gitkeep | xargs rm -r`
+    end
+  end
+end