Merge branch 'master' of dev.gitlab.org:gitlab/gitlab-ci

11 files changed, 547 insertions, 1 deletions

diff --git a/CHANGELOG b/CHANGELOG
index cc933f2..42442a8 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -10,6 +10,7 @@ v7.11.0
   - Remove projects IDs from dashboard
   - UI fix: Remove page headers from the admin area
   - Improve Email templates
+  - Add backup/restore utility
 
 v7.10.1
   - Fix failing migration when update to 7.10 from 7.8 and older versions
diff --git a/Gemfile b/Gemfile
index 9950761..b70df69 100644
--- a/Gemfile
+++ b/Gemfile
@@ -85,6 +85,9 @@ gem 'nprogress-rails'
 # Soft deletion
 gem "paranoia", "~> 2.0"
 
+# Colored output to console
+gem "colored"
+
 
 group :development do
   gem 'brakeman', require: false
diff --git a/Gemfile.lock b/Gemfile.lock
index d2c9158..76bd8fd 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -81,6 +81,7 @@ GEM
       coffee-script-source
       execjs
     coffee-script-source (1.6.3)
+    colored (1.2)
     columnize (0.9.0)
     connection_pool (1.2.0)
     coveralls (0.7.0)
@@ -380,6 +381,7 @@ DEPENDENCIES
   byebug
   capybara
   coffee-rails (~> 4.0.0)
+  colored
   coveralls
   database_cleaner
   default_value_for (~> 3.0.0)
diff --git a/config/application.yml.example b/config/application.yml.example
index 124d147..8f64c33 100644
--- a/config/application.yml.example
+++ b/config/application.yml.example
@@ -31,6 +31,20 @@ defaults: &defaults
     plain_url: "http://www.gravatar.com/avatar/%{hash}?s=%{size}&d=mm"
     ssl_url:   "https://secure.gravatar.com/avatar/%{hash}?s=%{size}&d=mm"
 
+  ## Backup settings
+  backup:
+    path: "tmp/backups"   # Relative paths are relative to Rails.root (default: tmp/backups/)
+    # keep_time: 604800   # default: 0 (forever) (in seconds)
+    # upload:
+    #   # Fog storage connection settings, see http://fog.io/storage/ .
+    #   connection:
+    #     provider: AWS
+    #     region: eu-west-1
+    #     aws_access_key_id: AKIAKIAKI
+    #     aws_secret_access_key: 'secret123'
+    #   # The remote 'directory' to store your backups. For S3, this would be the bucket name.
+    #   remote_directory: 'my.s3.bucket'
+
 
 development:
   <<: *defaults
diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb
index 8e77da8..6021b46 100644
--- a/config/initializers/1_settings.rb
+++ b/config/initializers/1_settings.rb
@@ -51,3 +51,15 @@ Settings['gravatar'] ||= Settingslogic.new({})
 Settings.gravatar['enabled']     = true if Settings.gravatar['enabled'].nil?
 Settings.gravatar['plain_url'] ||= 'http://www.gravatar.com/avatar/%{hash}?s=%{size}&d=mm'
 Settings.gravatar['ssl_url']   ||= 'https://secure.gravatar.com/avatar/%{hash}?s=%{size}&d=mm'
+
+#
+# Backup
+#
+Settings['backup'] ||= Settingslogic.new({})
+Settings.backup['keep_time']  ||= 0
+Settings.backup['path']         = File.expand_path(Settings.backup['path'] || "tmp/backups/", Rails.root)
+Settings.backup['upload'] ||= Settingslogic.new({ 'remote_directory' => nil, 'connection' => nil })
+# Convert upload connection settings to use symbol keys, to make Fog happy
+if Settings.backup['upload']['connection']
+  Settings.backup['upload']['connection'] = Hash[Settings.backup['upload']['connection'].map { |k, v| [k.to_sym, v] }]
+end
diff --git a/doc/README.md b/doc/README.md
index 0f53296..a3e86c1 100644
--- a/doc/README.md
+++ b/doc/README.md
@@ -6,4 +6,5 @@
 + [Update](update/README.md)
 + [Jobs](jobs/README.md)
 + [Runners](runners/README.md)
-+ [Permissions](permissions/README.md)
++ [Permissions](permissions/README.md) User permissions
++ [Rake Tasks](raketasks/README.md) Backup and restore take tasks
diff --git a/doc/raketasks/README.md b/doc/raketasks/README.md
new file mode 100644
index 0000000..872be4d
--- /dev/null
+++ b/doc/raketasks/README.md
@@ -0,0 +1,3 @@
+# Rake Tasks
+
++ [Backup/Restore](backup_restore.md)
\ No newline at end of file
diff --git a/doc/raketasks/backup_restore.md b/doc/raketasks/backup_restore.md
new file mode 100644
index 0000000..efa95ef
--- /dev/null
+++ b/doc/raketasks/backup_restore.md
@@ -0,0 +1,220 @@
+# Backup restore
+
+## Create a backup of the GitLab CI
+
+A backup creates an archive file that contains the database.
+This archive will be saved in backup_path (see `config/application.yml`).
+The filename will be `[TIMESTAMP]_gitlab_ci_backup.tar.gz`. This timestamp can be used to restore an specific backup.
+You can only restore a backup to exactly the same version of GitLab CI that you created it on, for example 7.10.1.
+
+*If you are interested in the GitLab backup please follow to the [GitLab backup documentation](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/raketasks/backup_restore.md)*
+
+```
+# use this command if you've installed GitLab CI with the Omnibus package
+sudo gitlab-ci-rake backup:create
+
+# if you've installed GitLab from source
+sudo -u gitlab_ci -H bundle exec rake backup:create RAILS_ENV=production
+```
+
+
+Example output:
+
+```
+Dumping database ... 
+Dumping PostgreSQL database gitlab_ci_development ... [DONE]
+done
+Creating backup archive: 1430930060_gitlab_ci_backup.tar.gz ... done
+Uploading backup archive to remote storage  ... skipped
+Deleting tmp directories ... done
+done
+Deleting old backups ... skipping
+```
+
+## Upload backups to remote (cloud) storage
+
+You can let the backup script upload the '.tar.gz' file it creates.
+It uses the [Fog library](http://fog.io/) to perform the upload.
+In the example below we use Amazon S3 for storage.
+But Fog also lets you use [other storage providers](http://fog.io/storage/).
+
+For omnibus packages:
+
+```ruby
+gitlab_ci['backup_upload_connection'] = {
+  'provider' => 'AWS',
+  'region' => 'eu-west-1',
+  'aws_access_key_id' => 'AKIAKIAKI',
+  'aws_secret_access_key' => 'secret123'
+}
+gitlab_ci['backup_upload_remote_directory'] = 'my.s3.bucket'
+```
+
+For installations from source:
+
+```yaml
+  backup:
+    # snip
+    upload:
+      # Fog storage connection settings, see http://fog.io/storage/ .
+      connection:
+        provider: AWS
+        region: eu-west-1
+        aws_access_key_id: AKIAKIAKI
+        aws_secret_access_key: 'secret123'
+      # The remote 'directory' to store your backups. For S3, this would be the bucket name.
+      remote_directory: 'my.s3.bucket'
+```
+
+If you are uploading your backups to S3 you will probably want to create a new
+IAM user with restricted access rights. To give the upload user access only for
+uploading backups create the following IAM profile, replacing `my.s3.bucket`
+with the name of your bucket:
+
+```json
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Sid": "Stmt1412062044000",
+      "Effect": "Allow",
+      "Action": [
+        "s3:AbortMultipartUpload",
+        "s3:GetBucketAcl",
+        "s3:GetBucketLocation",
+        "s3:GetObject",
+        "s3:GetObjectAcl",
+        "s3:ListBucketMultipartUploads",
+        "s3:PutObject",
+        "s3:PutObjectAcl"
+      ],
+      "Resource": [
+        "arn:aws:s3:::my.s3.bucket/*"
+      ]
+    },
+    {
+      "Sid": "Stmt1412062097000",
+      "Effect": "Allow",
+      "Action": [
+        "s3:GetBucketLocation",
+        "s3:ListAllMyBuckets"
+      ],
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Sid": "Stmt1412062128000",
+      "Effect": "Allow",
+      "Action": [
+        "s3:ListBucket"
+      ],
+      "Resource": [
+        "arn:aws:s3:::my.s3.bucket"
+      ]
+    }
+  ]
+}
+```
+
+## Storing configuration files
+
+Please be informed that a backup does not store your configuration files.
+If you use an Omnibus package please see the [instructions in the readme to backup your configuration](https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/README.md#backup-and-restore-omnibus-gitlab-configuration).
+If you have a cookbook installation there should be a copy of your configuration in Chef.
+If you have an installation from source, please consider backing up your `application.yml` file, any SSL keys and certificates, and your [SSH host keys](https://superuser.com/questions/532040/copy-ssh-keys-from-one-server-to-another-server/532079#532079).
+
+
+## Restore a previously created backup
+
+You can only restore a backup to exactly the same version of GitLab CI that you created it on, for example 7.10.1.
+
+### Installation from source
+
+```
+sudo -u gitlab_ci -H bundle exec rake backup:restore RAILS_ENV=production
+```
+
+Options
+
+```
+BACKUP=timestamp_of_backup (required if more than one backup exists)
+```
+
+### Omnibus package installation
+
+We will assume that you have installed GitLab CI from an omnibus package and run
+`sudo gitlab-ctl reconfigure` at least once.
+
+First make sure your backup tar file is in `/var/opt/gitlab/backups`.
+
+```shell
+sudo cp 1393513186_gitlab_ci_backup.tar.gz /var/opt/gitlab/backups/
+```
+
+Next, restore the backup by running the restore command. You need to specify the
+timestamp of the backup you are restoring.
+
+```shell
+# Stop processes that are connected to the database
+sudo gitlab-ctl stop unicorn
+sudo gitlab-ctl stop sidekiq
+
+# This command will overwrite the contents of your GitLab CI database!
+sudo gitlab-ci-rake backup:restore BACKUP=1393513186
+
+# Start GitLab
+sudo gitlab-ctl start
+```
+
+If there is a GitLab version mismatch between your backup tar file and the installed
+version of GitLab, the restore command will abort with an error. Install a package for
+the [required version](https://www.gitlab.com/downloads/archives/) and try again.
+
+
+
+## Configure cron to make daily backups
+
+### For installation from source:
+```
+cd /home/git/gitlab
+sudo -u gitlab_ci -H editor config/application.yml # Enable keep_time in the backup section to automatically delete old backups
+sudo -u gitlab_ci crontab -e # Edit the crontab for the git user
+```
+
+Add the following lines at the bottom:
+
+```
+# Create a backup of the GitLab CI every day at 4am
+0 4 * * * cd /home/gitlab_ci/gitlab_ci && PATH=/usr/local/bin:/usr/bin:/bin bundle exec rake backup:create RAILS_ENV=production CRON=1
+```
+
+The `CRON=1` environment setting tells the backup script to suppress all progress output if there are no errors.
+This is recommended to reduce cron spam.
+
+### Omnibus package installation
+
+To schedule a cron job that backs up your GitLab CI, use the root user:
+
+```
+sudo su -
+crontab -e
+```
+
+There, add the following line to schedule the backup for everyday at 2 AM:
+
+```
+0 2 * * * /opt/gitlab/bin/gitlab-ci-rake backup:create CRON=1
+```
+
+You may also want to set a limited lifetime for backups to prevent regular
+backups using all your disk space.  To do this add the following lines to
+`/etc/gitlab/gitlab.rb` and reconfigure:
+
+```
+# limit backup lifetime to 7 days - 604800 seconds
+gitlab_ci['backup_keep_time'] = 604800
+```
+
+NOTE: This cron job does not [backup your omnibus-gitlab configuration](#backup-and-restore-omnibus-gitlab-configuration).
+
diff --git a/lib/backup/database.rb b/lib/backup/database.rb
new file mode 100644
index 0000000..3ef5f44
--- /dev/null
+++ b/lib/backup/database.rb
@@ -0,0 +1,92 @@
+require 'yaml'
+
+module Backup
+  class Database
+    attr_reader :config, :db_dir
+
+    def initialize
+      @config = YAML.load_file(File.join(Rails.root,'config','database.yml'))[Rails.env]
+      @db_dir = File.join(GitlabCi.config.backup.path, 'db')
+      FileUtils.mkdir_p(@db_dir) unless Dir.exists?(@db_dir)
+    end
+
+    def dump
+      success = case config["adapter"]
+      when /^mysql/ then
+        $progress.print "Dumping MySQL database #{config['database']} ... "
+        system('mysqldump', *mysql_args, config['database'], out: db_file_name)
+      when "postgresql" then
+        $progress.print "Dumping PostgreSQL database #{config['database']} ... "
+        pg_env
+        system('pg_dump', config['database'], out: db_file_name)
+      end
+      report_success(success)
+      abort 'Backup failed' unless success
+    end
+
+    def restore
+      success = case config["adapter"]
+      when /^mysql/ then
+        $progress.print "Restoring MySQL database #{config['database']} ... "
+        system('mysql', *mysql_args, config['database'], in: db_file_name)
+      when "postgresql" then
+        $progress.print "Restoring PostgreSQL database #{config['database']} ... "
+        # Drop all tables because PostgreSQL DB dumps do not contain DROP TABLE
+        # statements like MySQL.
+        drop_all_tables
+        drop_all_postgres_sequences
+        pg_env
+        system('psql', config['database'], '-f', db_file_name)
+      end
+      report_success(success)
+      abort 'Restore failed' unless success
+    end
+
+    protected
+
+    def db_file_name
+      File.join(db_dir, 'database.sql')
+    end
+
+    def mysql_args
+      args = {
+        'host'      => '--host',
+        'port'      => '--port',
+        'socket'    => '--socket',
+        'username'  => '--user',
+        'encoding'  => '--default-character-set',
+        'password'  => '--password'
+      }
+      args.map { |opt, arg| "#{arg}=#{config[opt]}" if config[opt] }.compact
+    end
+
+    def pg_env
+      ENV['PGUSER']     = config["username"] if config["username"]
+      ENV['PGHOST']     = config["host"] if config["host"]
+      ENV['PGPORT']     = config["port"].to_s if config["port"]
+      ENV['PGPASSWORD'] = config["password"].to_s if config["password"]
+    end
+
+    def report_success(success)
+      if success
+        $progress.puts '[DONE]'.green
+      else
+        $progress.puts '[FAILED]'.red
+      end
+    end
+
+    def drop_all_tables
+      connection = ActiveRecord::Base.connection
+      connection.tables.each do |table|
+        connection.drop_table(table)
+      end
+    end
+
+    def drop_all_postgres_sequences
+      connection = ActiveRecord::Base.connection
+      connection.execute("SELECT c.relname FROM pg_class c WHERE c.relkind = 'S';").each do |sequence|
+        connection.execute("DROP SEQUENCE #{sequence['relname']}")
+      end
+    end
+  end
+end
diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb
new file mode 100644
index 0000000..dc5ac75
--- /dev/null
+++ b/lib/backup/manager.rb
@@ -0,0 +1,155 @@
+module Backup
+  class Manager
+    def pack
+      # saving additional informations
+      s = {}
+      s[:db_version]         = "#{ActiveRecord::Migrator.current_version}"
+      s[:backup_created_at]  = Time.now
+      s[:gitlab_version]     = GitlabCi::VERSION
+      s[:tar_version]        = tar_version
+      tar_file = "#{s[:backup_created_at].to_i}_gitlab_ci_backup.tar.gz"
+
+      Dir.chdir(GitlabCi.config.backup.path) do
+        File.open("#{GitlabCi.config.backup.path}/backup_information.yml",
+                  "w+") do |file|
+          file << s.to_yaml.gsub(/^---\n/,'')
+        end
+
+        FileUtils.chmod(0700, "db")
+
+        # create archive
+        $progress.print "Creating backup archive: #{tar_file} ... "
+        orig_umask = File.umask(0077)
+        if Kernel.system('tar', '-czf', tar_file, *backup_contents)
+          $progress.puts "done".green
+        else
+          puts "creating archive #{tar_file} failed".red
+          abort 'Backup failed'
+        end
+        File.umask(orig_umask)
+
+        upload(tar_file)
+      end
+    end
+
+    def upload(tar_file)
+      remote_directory = GitlabCi.config.backup.upload.remote_directory
+      $progress.print "Uploading backup archive to remote storage #{remote_directory} ... "
+
+      connection_settings = GitlabCi.config.backup.upload.connection
+      if connection_settings.blank?
+        $progress.puts "skipped".yellow
+        return
+      end
+
+      connection = ::Fog::Storage.new(connection_settings)
+      directory = connection.directories.get(remote_directory)
+
+      if directory.files.create(key: tar_file, body: File.open(tar_file), public: false)
+        $progress.puts "done".green
+      else
+        puts "uploading backup to #{remote_directory} failed".red
+        abort 'Backup failed'
+      end
+    end
+
+    def cleanup
+      $progress.print "Deleting tmp directories ... "
+      
+      backup_contents.each do |dir|
+        next unless File.exist?(File.join(GitlabCi.config.backup.path, dir))
+
+        if FileUtils.rm_rf(File.join(GitlabCi.config.backup.path, dir))
+          $progress.puts "done".green
+        else
+          puts "deleting tmp directory '#{dir}' failed".red
+          abort 'Backup failed'
+        end
+      end
+    end
+
+    def remove_old
+      # delete backups
+      $progress.print "Deleting old backups ... "
+      keep_time = GitlabCi.config.backup.keep_time.to_i
+
+      if keep_time > 0
+        removed = 0
+        
+        Dir.chdir(GitlabCi.config.backup.path) do
+          file_list = Dir.glob('*_gitlab_ci_backup.tar.gz')
+          file_list.map! { |f| $1.to_i if f =~ /(\d+)_gitlab_ci_backup.tar.gz/ }
+          file_list.sort.each do |timestamp|
+            if Time.at(timestamp) < (Time.now - keep_time)
+              if Kernel.system(*%W(rm #{timestamp}_gitlab_ci_backup.tar.gz))
+                removed += 1
+              end
+            end
+          end
+        end
+
+        $progress.puts "done. (#{removed} removed)".green
+      else
+        $progress.puts "skipping".yellow
+      end
+    end
+
+    def unpack
+      Dir.chdir(GitlabCi.config.backup.path)
+
+      # check for existing backups in the backup dir
+      file_list = Dir.glob("*_gitlab_ci_backup.tar.gz").each.map { |f| f.split(/_/).first.to_i }
+      puts "no backups found" if file_list.count == 0
+
+      if file_list.count > 1 && ENV["BACKUP"].nil?
+        puts "Found more than one backup, please specify which one you want to restore:"
+        puts "rake gitlab:backup:restore BACKUP=timestamp_of_backup"
+        exit 1
+      end
+
+      tar_file = ENV["BACKUP"].nil? ? File.join("#{file_list.first}_gitlab_ci_backup.tar.gz") : File.join(ENV["BACKUP"] + "_gitlab_ci_backup.tar.gz")
+
+      unless File.exists?(tar_file)
+        puts "The specified backup doesn't exist!"
+        exit 1
+      end
+
+      $progress.print "Unpacking backup ... "
+
+      unless Kernel.system(*%W(tar -xzf #{tar_file}))
+        puts "unpacking backup failed".red
+        exit 1
+      else
+        $progress.puts "done".green
+      end
+
+      ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0
+
+      # restoring mismatching backups can lead to unexpected problems
+      if settings[:gitlab_version] != GitlabCi::VERSION
+        puts "GitLab CI version mismatch:".red
+        puts "  Your current GitLab CI version (#{GitlabCi::VERSION}) differs from the GitLab CI version in the backup!".red
+        puts "  Please switch to the following version and try again:".red
+        puts "  version: #{settings[:gitlab_version]}".red
+        puts
+        puts "Hint: git checkout v#{settings[:gitlab_version]}"
+        exit 1
+      end
+    end
+
+    def tar_version
+      tar_version = `tar --version`
+      tar_version.force_encoding('locale').split("\n").first
+    end
+
+    private
+
+    def backup_contents
+      ["db", "backup_information.yml"]
+    end
+
+    def settings
+      @settings ||= YAML.load_file("backup_information.yml")
+    end
+  end
+end
diff --git a/lib/tasks/backup.rake b/lib/tasks/backup.rake
new file mode 100644
index 0000000..84f9bcd
--- /dev/null
+++ b/lib/tasks/backup.rake
@@ -0,0 +1,43 @@
+namespace :backup do
+  
+  desc "GITLAB | Create a backup of the GitLab CI database"
+  task create: :environment do
+    configure_cron_mode
+
+    $progress.puts "Dumping database ... ".blue
+
+    Backup::Database.new.dump
+    $progress.puts "done".green
+
+    backup = Backup::Manager.new
+    backup.pack
+    backup.cleanup
+    backup.remove_old
+  end
+
+  desc "GITLAB | Restore a previously created backup"
+  task restore: :environment do
+    configure_cron_mode
+
+    backup = Backup::Manager.new
+    backup.unpack
+
+    $progress.puts "Restoring database ... ".blue
+    Backup::Database.new.restore
+    $progress.puts "done".green
+
+    backup.cleanup
+  end
+
+  def configure_cron_mode
+    if ENV['CRON']
+      # We need an object we can say 'puts' and 'print' to; let's use a
+      # StringIO.
+      require 'stringio'
+      $progress = StringIO.new
+    else
+      $progress = $stdout
+    end
+  end
+end
+