Dt e3 0 update

.travis.yml

@@ -11,10 +11,16 @@ env:
   matrix:
   - RAKE_TASK=spec
   global:
-    secure: BdVC3OHqYcgePLrkKIk28Ewn/dxCYFf3Cx+Q8P+BCDj6UPJyRSbKmILBzuX96H5xhKmUFo0A/upUhJI9UUP9aXHO7MzRe04/c88QdO4wGacVUaIyB20S0pr262zbc/nA50K9cVgpmWc64n6uQR1tgM6ZyyBnBeXkLzCAOHPq99I=
+    - secure: BdVC3OHqYcgePLrkKIk28Ewn/dxCYFf3Cx+Q8P+BCDj6UPJyRSbKmILBzuX96H5xhKmUFo0A/upUhJI9UUP9aXHO7MzRe04/c88QdO4wGacVUaIyB20S0pr262zbc/nA50K9cVgpmWc64n6uQR1tgM6ZyyBnBeXkLzCAOHPq99I=
+    - PATH=/snap/bin:$PATH
+
+before_install:
+  - gem uninstall -v '>=2' -i $(rvm gemdir)@global -ax bundler || true
+  - gem install bundler -v '2.3.7'
+  - gem install rubygems-update -v 3.4.22
 
 addons:
-  apt:
+  - apt:
     packages:
       - rabbitmq-server
 
@@ -28,11 +34,12 @@ jobs:
      script: bundle exec rspec spec
    - stage: ":ship: it to quay.io"
      dist: bionic
+     group: edge
      ruby:
      services:
      addons:
      before_install: skip
      before_script: skip
      install: skip
      script: make ship
-     if: commit_message =~ /ship:docker/ OR env(SHIP_DOCKER) = true
+     if: (branch = master and type = push ) OR commit_message =~ /ship:docker/ OR env(SHIP_DOCKER) = true

Dockerfile

@@ -2,11 +2,29 @@ FROM ruby:3.2.2-slim
 
 LABEL maintainer Travis CI GmbH <support+travis-scheduler-docker-images@travis-ci.com>
 
-# packages required for bundle install
+RUN ( \
+   bundle config set no-cache 'true'; \
+   bundle config --global frozen 1; \
+   bundle config set deployment 'true'; \
+   mkdir -p /app; \
+)
+WORKDIR /app
+COPY Gemfile*      /app/
+ARG bundle_gems__contribsys__com
 RUN ( \
    apt-get update ; \
+   apt-get upgrade -y ; \
    apt-get install -y --no-install-recommends git make gcc g++ libpq-dev libjemalloc-dev libcurl4 \
-   && rm -rf /var/lib/apt/lists/* \
+   && rm -rf /var/lib/apt/lists/*; \
+   gem update --system; \
+   gem install bundler -v '2.3.24'; \
+   bundle config https://gems.contribsys.com/ $bundle_gems__contribsys__com; \
+   bundle config set without 'development test'; \
+   bundle install; \
+   bundle config --delete https://gems.contribsys.com; \
+   apt-get remove -y gcc g++ make git perl && apt-get -y autoremove; \
+   bundle clean && rm -rf /app/vendor/bundle/ruby/2.7.0/cache/*; \
+   for i in `find /app/vendor/ -name \*.o -o -name \*.c -o -name \*.h`; do rm -f $i; done; \
 )
 
 ENV LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.2
@@ -31,4 +49,4 @@ RUN gem install --user-install executable-hooks
 
 COPY . /app
 
-CMD bundle exec bin/sidekiq-pgbouncer ${SIDEKIQ_CONCURRENCY:-5} ${SIDEKIQ_QUEUE:-scheduler}
+CMD ["bundle", "exec", "bin/sidekiq-pgbouncer", "${SIDEKIQ_CONCURRENCY:-5}", "${SIDEKIQ_QUEUE:-scheduler}"]

Makefile

@@ -31,7 +31,7 @@ DOCKER ?= docker
 
 .PHONY: docker-build
 docker-build:
-	$(DOCKER) build --build-arg bundle_gems__contribsys__com=$(BUNDLE_GEMS__CONTRIBSYS__COM) -t $(DOCKER_DEST) .
+	$(DOCKER) build --no-cache --pull --build-arg bundle_gems__contribsys__com=$(BUNDLE_GEMS__CONTRIBSYS__COM) -t $(DOCKER_DEST) .
 
 .PHONY: docker-login
 docker-login:
@@ -52,7 +52,7 @@ docker-push-branch:
 .PHONY: ship
 ship: docker-build docker-login
 
-ifeq ($(BRANCH),master)
+ifeq ($(TRAVIS_BRANCH),master)
 ifeq ($(TRAVIS_PULL_REQUEST),false)
 ship: docker-push-latest-master
 endif

bin/sidekiq

@@ -1,7 +1,6 @@
 #!/bin/bash
 
 threads=${SIDEKIQ_THREADS:-8}
-index=$(echo ${DYNO:-1} | sed 's/.*\.//')
 queues=$@
 
 export RUBYOPT="-Ilib $RUBYOPT"

lib/travis/amqp/publisher.rb

@@ -25,6 +25,7 @@ def initialize(routing_key, options = {})
       def publish(data, options = {})
         return unless data
 
+        Amqp.logger.warn "Queue #{routing_key} doesn't exist!" if ENV['AMQP_QUEUE_VALIDATION'] && !Amqp.connection.queue_exists?(routing_key)
         data = MultiJson.encode(data)
         exchange.publish(data, deep_merge(default_data, options))
         debug "Published AMQP message to #{routing_key}."

lib/travis/queue/matcher.rb

@@ -37,7 +37,7 @@ def paid
         job.paid?
       end
 
-      def repo_private
+        def repo_private
         job.private?
       end
 
@@ -90,7 +90,7 @@ def resources
       end
 
       def arch
-        return nil if job.private? && OSS_ONLY_ARCH.include?(job.config[:arch])
+        return nil if job.private? && OSS_ONLY_ARCH.include?(job.config[:arch]) && !enterprise?
 
         job.config[:arch]
       end
@@ -99,7 +99,7 @@ def virt
         job.config[:virt]
       end
 
-      def vm_size
+        def vm_size
         job.config[:vm][:size] if job.config[:vm]
       end
 
@@ -115,6 +115,10 @@ def check_unknown_matchers(used)
         unknown = used - KEYS
         logger.warn format(MSGS[:unknown_matchers], used, unknown, repo.slug) if logger && unknown.any?
       end
+
+        def enterprise?
+          !!Travis::Scheduler.context.config[:enterprise]
+        end
     end
   end
 end

lib/travis/scheduler/config.rb

@@ -13,7 +13,7 @@ class Config < Travis::Config
              enterprise: false,
              github: { api_url: 'https://api.github.com', source_host: 'github.com' },
              billing: { url: 'http://localhost:9292/', auth_key: 'auth_keys' },
-             host: 'https://travis-ci.com',
+             billing:       { url: 'http://localhost:9292/', auth_key: 'auth_keys' },host: 'https://travis-ci.com',
              interval: 2,
              limit: { public: 5, education: 1, default: 5, by_owner: {}, delegate: {} },
              lock: { strategy: :redis, ttl: 150 },

lib/travis/scheduler/jobs/capacity/plan.rb

@@ -12,7 +12,6 @@ def applicable?
           def report(status, job)
             super.merge(max:)
           end
-
           def accept?(job)
             super if !on_metered_plan? || billing_allowed?(job)
           end

lib/travis/scheduler/record/job.rb

@@ -4,6 +4,10 @@ class JobConfig < ActiveRecord::Base
   def config=(config)
     self.config_json = config if has_attribute?(:config_json)
     super
+  end
+
+  def save(arg)
+    super(arg)
   rescue Encoding::UndefinedConversionError
   end
 end
@@ -101,6 +105,7 @@ def public?
 
   def config
     record = super
+    record = JSON.parse(record) if record.is_a?(String)
     config = record&.config_json if record.respond_to?(:config_json) # TODO: remove once we've rolled over
     config ||= record&.config
     config ||= read_attribute(:config) if has_attribute?(:config)
@@ -112,4 +117,10 @@ def config
   def name
     config[:name]
   end
+
+  def new_queueable
+    return if repository_id.blank? # to avoid trying save objects without repository_id
+    saved = new_record? ? save : true # saves if it is a new record
+    create_queueable if saved # it is allowed to create queueable records only for jobs which are persisting in the database
+  end
 end

lib/travis/scheduler/record/repository/settings.rb

@@ -101,7 +101,7 @@ def custom_timeouts?(settings)
 
   validates_with TimeoutsValidator
 
-  def update_attributes!(attrs)
+  def update!(attrs)
     attrs.each { |key, value| send(:"#{key}=", value) }
     save
   end

lib/travis/scheduler/record/user.rb

@@ -40,18 +40,21 @@ def paid?
   end
 
   def paid_new_plan?
-    redis_key = "user:#{id}:plan"
+    redis_key = "user:#{self.id}:plan"
     plan = if redis.exists?(redis_key)
              JSON.parse(redis.get(redis_key))
            else
-             p = billing_client.get_plan(self)
-             p.length > 0 ? p.to_h : {}
+             billing_client.get_plan(self).to_h
            end
     return false if plan[:error] || plan['plan_name'].nil?
 
     plan['hybrid'] || !plan['plan_name'].include?('free')
   end
 
+  def enterprise?
+    !!context.config[:enterprise]
+  end
+
   def default_worker_timeout
     # When the user is a paid user ("subscribed") or has an active trial, they
     #   are granted a different default timeout on their jobs.
@@ -60,7 +63,7 @@ def default_worker_timeout
     #   those enforced by workers themselves, but we plan to sometime in the
     #   following weeks/months.
     #
-    if paid? || educational?
+    if enterprise? || paid? || educational?
       Travis.logger.info "Default Timeout: DEFAULT_SUBSCRIBED_TIMEOUT for owner=#{id}"
       DEFAULT_SUBSCRIBED_TIMEOUT
     else

lib/travis/scheduler/serialize/worker/config/decrypt.rb

@@ -19,7 +19,7 @@ def apply
 
             private
 
-            def force_vault_to_be_secure!(config)
+              def force_vault_to_be_secure!(config)
               config[:vault].delete(:token) if config.dig(:vault, :token).is_a?(String)
             end
 

lib/travis/scheduler/serialize/worker/job.rb

@@ -97,10 +97,12 @@ def vm_configs
           def job_repository
             return job.repository if job.source.event_type != 'pull_request' || job.source.request.pull_request.head_repo_slug == job.source.request.pull_request.base_repo_slug
 
+            return repository if repository.settings.share_encrypted_env_with_forks
+
             owner_name, repo_name = job.source.request.pull_request.head_repo_slug.split('/')
             return if owner_name.nil? || owner_name.empty? || repo_name.nil? || repo_name.empty?
 
-            ::Repository.find_by(owner_name:, name: repo_name)
+            ::Repository.find_by(owner_name: owner_name, name: repo_name)
           end
 
           def repository_key

lib/travis/scheduler/serialize/worker/repo.rb

@@ -79,7 +79,12 @@ def hard_limit_timeout
           end
 
           def timeout(type)
-            return unless timeout = repo.settings.send(:"timeout_#{type}")
+             timeout = repo.settings.send(:"timeout_#{type}")
+              if timeout == nil
+                config = Travis.config.settings&.timeouts&.defaults
+                timeout = config[type] if config
+              end
+              return unless timeout
 
             timeout = Integer(timeout)
             timeout * 60 # worker handles timeouts in seconds

lib/travis/scheduler/service/notify.rb

@@ -49,7 +49,8 @@ def notify_job_board
 
         def notify_rabbitmq
           info :publish, job.id, job.queue, 'rabbitmq'
-          amqp.publish(worker_payload, properties: { type: 'test', persistent: true })
+          w = worker_payload
+            amqp.publish(w, properties: { type: 'test', persistent: true })
         end
 
         def notify_live
@@ -101,7 +102,7 @@ def amqp
         def redirect_queue
           queue = redirections[job.queue] or return
           info format(MSGS[:redirect], job.queue, queue)
-          job.update_attributes!(queue:)
+          job.update!(queue:)
         end
 
         def redirections

lib/travis/scheduler/service/set_queue.rb

@@ -12,7 +12,8 @@ class SetQueue < Struct.new(:context, :job, :opts)
 
         MSGS = {
           redirect: 'Found job.queue: %s. Redirecting to: %s',
-          queue: 'Setting queue to %s for job=%s'
+          queue: 'Setting queue to %s for job=%s',
+          canceled: 'Build %s has been canceled, job %s being canceled'
         }.freeze
 
         def run
@@ -23,7 +24,17 @@ def run
         private
 
         def queue
-          @queue ||= redirect(Queue.new(job, config, logger).select)
+          if job.stage.present? && job.stage.state == "canceled"
+              info MSGS[:canceled] % [job.source.id, job.id]
+              payload = { id: job.id, source: 'scheduler' }
+              Hub.push('job:cancel', payload)
+            else
+              @queue ||= redirect(Queue.new(job, config, logger).select)
+            end
+          rescue => e
+            puts "ERROR while trying to queue: #{e.message}"
+            puts "Backtrace:"
+            puts e.backtrace.join("\n")@queue ||= redirect(Queue.new(job, config, logger).select)
         end
 
         # TODO: confirm we don't need queue redirection any more

lib/travis/service.rb

@@ -11,7 +11,7 @@ def self.push(*args)
         'queue' => ENV['SIDEKIQ_QUEUE'] || 'scheduler',
         'class' => 'Travis::Scheduler::Worker',
         'args' => args.map! { |arg| arg.to_json },
-        'at' => args.last.is_a?(Hash) ? args.last.delete(:at) : Time.now.to_f
+        'at'    => args.last.is_a?(Hash) ? (args.last.delete(:at) || Time.now.to_i) : Time.now.to_i
       )
     end
   end

lib/travis/service/job_board.rb

@@ -30,7 +30,7 @@ class JobBoard < Struct.new(:job_id, :data, :config, :logger)
       def post
         response = http.post(PATH, JSON.dump(payload))
         log response.status
-      rescue Faraday::ClientError => e
+      rescue Faraday::ClientError, Faraday::ServerError => e
         log e.response[:status], e.response[:body]
         raise
       rescue Faraday::ServerError => e

spec/travis/scheduler/jobs_spec.rb

@@ -503,7 +503,7 @@ def subscribe(plan, owner = user)
   describe 'with a boost of 5 and a repo settings limit 3' do
     before do
       redis.set("scheduler.owner.limit.#{user.login}", 5)
-      repo.settings.update_attributes!(maximum_number_of_builds: 3)
+      repo.settings.update!(maximum_number_of_builds: 3)
     end
 
     describe 'with private jobs only' do
@@ -569,7 +569,7 @@ def subscribe(plan, owner = user)
     before do
       subscribe(:two)
       redis.set("scheduler.owner.limit.#{user.login}", 4)
-      repo.settings.update_attributes!(maximum_number_of_builds: 3)
+      repo.settings.update!(maximum_number_of_builds: 3)
     end
 
     describe 'with private jobs only' do

spec/travis/scheduler/record/user_spec.rb

@@ -4,6 +4,8 @@
   let(:user) { FactoryBot.create(:user) }
   let(:repo) { FactoryBot.create(:repository) }
   let(:authorize_build_url) { "http://localhost:9292/users/#{user.id}/plan" }
+  let(:repo) { FactoryGirl.create(:repository) }
+  let(:authorize_build_url) { "http://localhost:9292/users/#{user.id}/plan" }
 
   describe 'constants' do
     # It isn't often that we see tests for constants, but these are special.
@@ -82,6 +84,19 @@
       end
     end
 
+    context "paid_new_plan? == true" do
+      before do
+        user.stubs(:paid_new_plan?).returns(true)
+        stub_request(:get, authorize_build_url).to_return(
+          body: MultiJson.dump(plan_name: 'two_concurrent_plan', hybrid: true, free: false, status: 'subscribed', metered: false)
+        )
+      end
+
+      it "returns the DEFAULT_SUBSCRIBED_TIMEOUT" do
+        expect(user.default_worker_timeout).to eq Organization::DEFAULT_SUBSCRIBED_TIMEOUT
+      end
+    end
+
     context 'paid_new_plan? == true' do
       before do
         user.stubs(:paid_new_plan?).returns(true)

spec/travis/scheduler/serialize/worker/config_spec.rb

@@ -96,6 +96,16 @@ def encrypt(string)
 
       it { is_expected.to eql vault: {} }
     end
+
+    describe 'decrypts vault secure token' do
+      let(:config) { { vault: { token: { secure: encrypt('my_key') } } } }
+      it { should eql vault: {token: 'my_key'} }
+    end
+
+    describe 'clears vault unsecure token' do
+      let(:config) { { vault: { token:  'my_key' } }  }
+      it { should eql vault: {} }
+    end
   end
 
   describe 'with secure env disabled' do