Support Ubuntu 18.04 for OnDemand 2.0 (#2160)

* Support Ubuntu 18.04
* Downgrade nokogiri to see if can get Ruby 2.5 support to work
* Avoid test Gems when deploying for production
* Downgrade dalli gem to support Ruby 2.5
* Try and build with python2.7
* Run unit tests against Ruby 2.5.1 which is what Ubuntu 18.04 can install
* Downgrade minitest to support Ruby 2.5
* Downgrade net-ssh for tests to support Ruby 2.5
* Downgrade capybara and selenium-webdriver to support Ruby 2.5
* Set GEM_HOME and PATH for deb SCL
* Avoid newer dotenv
* Update ood_packaging gem to support Ubuntu 18.04 builds
* Run e2e tests for Ubuntu 18.04
* OIDCStateMaxNumberOfCookies is not support by mod_auth_openidc on Ubuntu 18.04
* Long standing bug with lua-posix on Ubuntu 18.04
* Update ood_core to support Ruby 2.5

.github/workflows/tests.yml

@@ -11,7 +11,7 @@ jobs:
       fail-fast: false
       matrix:
         os: ["ubuntu-latest"]
-        ruby: ["2.7.1"]
+        ruby: ["2.5.1", "2.7.1"]
         bundler: ["2.1.4"]
     runs-on: ${{ matrix.os }}
     name: Unit tests
@@ -41,7 +41,9 @@ jobs:
         run: bundle exec rake test:shellcheck
 
       - name: Run unit tests
-        run: bundle exec rake test:unit
+        run: |
+          gem install --default -v 4.0.4 psych
+          bundle exec rake test:unit
 
       - name: Run lint tests
         run: bundle exec rake lint || true
@@ -78,7 +80,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        dist: ["el7", "el8", "ubuntu-20.04"]
+        dist: ["el7", "el8", "ubuntu-18.04", "ubuntu-20.04"]
         version: ["2.0"]
     runs-on: "ubuntu-latest"
     name: E2E test ${{ matrix.dist }} (ondemand-${{ matrix.version }})
@@ -107,3 +109,8 @@ jobs:
         env:
           BEAKER_set: ${{ matrix.dist }}
           OOD_BUILD_REPO: ${{ matrix.version }}
+
+      - name: Debug failure
+        if: failure()
+        run: |
+          find tmp/e2e_ctr/ -type f -name *.log -exec cat {} +

.gitlab-ci.yml

@@ -20,7 +20,7 @@ build:
     - bundle exec rake package:build[$OOD_PACKAGING_DIST]
   parallel:
     matrix:
-      - OOD_PACKAGING_DIST: [el7, el8, ubuntu-20.04]
+      - OOD_PACKAGING_DIST: [el7, el8, ubuntu-18.04, ubuntu-20.04]
   artifacts:
     paths:
       - dist

Gemfile

@@ -9,15 +9,17 @@ gem "bcrypt"
 gem 'dotenv', '~> 2.1'
 
 group :package do
-  gem 'ood_packaging', '0.0.2.r2.0'
+  gem 'ood_packaging', '0.0.3.r2.0'
 end
 
 group :test do
+  gem "minitest", '5.15.0'
   gem "rspec"
   gem "watir"
   gem "rubocop"
 
   gem "beaker"
   gem "beaker-rspec"
   gem "beaker-docker", '~> 1.1.1'
+  gem "net-ssh", '< 7.0.0'
 end

Gemfile.lock

@@ -93,17 +93,17 @@ GEM
     inifile (3.0.0)
     method_source (1.0.0)
     minitar (0.9)
-    minitest (5.16.2)
+    minitest (5.15.0)
     multi_json (1.15.0)
     multipart-post (2.1.1)
     net-scp (1.2.1)
       net-ssh (>= 2.6.5)
-    net-ssh (7.0.1)
+    net-ssh (6.1.0)
     net-telnet (0.1.1)
     oga (3.3)
       ast
       ruby-ll (~> 2.1)
-    ood_packaging (0.0.2.r2.0)
+    ood_packaging (0.0.3.r2.0)
       rake (~> 13.0.1)
     open_uri_redirections (0.2.1)
     parallel (1.20.1)
@@ -189,7 +189,9 @@ DEPENDENCIES
   beaker-docker (~> 1.1.1)
   beaker-rspec
   dotenv (~> 2.1)
-  ood_packaging (= 0.0.2.r2.0)
+  minitest (= 5.15.0)
+  net-ssh (< 7.0.0)
+  ood_packaging (= 0.0.3.r2.0)
   rake
   rspec
   rubocop

Rakefile

@@ -29,7 +29,7 @@ namespace :build do
       bundle_args << "--path vendor/bundle"
     end
     if PASSENGER_APP_ENV == "production"
-      bundle_args << "--without doc"
+      bundle_args << "--without doc test"
     end
     apps.each do |a|
       next unless a.ruby_app?

apps/dashboard/Gemfile

@@ -36,8 +36,8 @@ group :development, :test do
 end
 
 group :test do
-  gem "capybara"
-  gem "selenium-webdriver"
+  gem "capybara", '< 3.36.0'
+  gem "selenium-webdriver", '< 4.0.0'
 end
 
 group :development do
@@ -66,6 +66,7 @@ gem 'autoprefixer-rails', '~> 10.2.5'
 gem 'dotiw'
 gem 'local_time', '~> 1.0.3'
 gem 'turbolinks', '~> 5.2.0'
+gem 'nokogiri', '~> 1.12.5'
 
 # OOD specific gems
 gem 'ood_support', '~> 0.0.2'
@@ -77,7 +78,7 @@ gem 'pbs', '~> 2.2.1'
 gem "sinatra", require: false
 gem "sinatra-contrib", require: false
 gem "erubi", require: false
-gem "dalli", require: false
+gem "dalli", '<= 3.2.0', require: false
 
 gem 'webpacker', '~> 5.2', '>= 5.2.1'
 gem 'zip_tricks', '~> 5.5'

apps/dashboard/Gemfile.lock

@@ -53,16 +53,15 @@ GEM
     browser (2.7.1)
     builder (3.2.4)
     byebug (11.1.3)
-    capybara (3.37.1)
+    capybara (3.35.3)
       addressable
-      matrix
       mini_mime (>= 0.1.3)
       nokogiri (~> 1.8)
       rack (>= 1.6.0)
       rack-test (>= 0.6.3)
       regexp_parser (>= 1.5, < 3.0)
       xpath (~> 3.2)
-    childprocess (4.1.0)
+    childprocess (3.0.0)
     climate_control (0.2.0)
     coffee-rails (4.2.2)
       coffee-script (>= 2.2.0)
@@ -73,7 +72,7 @@ GEM
     coffee-script-source (1.12.2)
     concurrent-ruby (1.1.10)
     crass (1.0.6)
-    dalli (3.2.1)
+    dalli (3.2.0)
     data-confirm-modal (1.6.3)
       railties (>= 3.0)
     dotenv (2.7.6)
@@ -117,26 +116,25 @@ GEM
     mail (2.7.1)
       mini_mime (>= 0.1.1)
     marcel (1.0.2)
-    matrix (0.4.2)
     method_source (1.0.0)
     mini_mime (1.1.2)
-    mini_portile2 (2.8.0)
-    minitest (5.16.2)
+    mini_portile2 (2.6.1)
+    minitest (5.15.0)
     mocha (1.14.0)
     multi_json (1.15.0)
     mustermann (1.1.1)
       ruby2_keywords (~> 0.0.1)
     nio4r (2.5.8)
-    nokogiri (1.13.7)
-      mini_portile2 (~> 2.8.0)
+    nokogiri (1.12.5)
+      mini_portile2 (~> 2.6.1)
       racc (~> 1.4)
     ood_appkit (1.1.5)
       addressable (~> 2.4)
       lograge (~> 0.3)
       ood_core (~> 0.1)
       rails (> 4.0.7, < 6.0)
       redcarpet (~> 3.2)
-    ood_core (0.20.1)
+    ood_core (0.20.2)
       ffi (~> 1.9, >= 1.9.6)
       ood_support (~> 0.0.2)
       rexml (~> 3.2)
@@ -185,7 +183,7 @@ GEM
     rdoc (6.4.0)
       psych (>= 4.0.0)
     redcarpet (3.5.1)
-    regexp_parser (2.4.0)
+    regexp_parser (2.5.0)
     request_store (1.5.1)
       rack (>= 1.4)
     rexml (3.2.5)
@@ -206,9 +204,8 @@ GEM
       ffi (~> 1.9)
     sdoc (2.4.0)
       rdoc (>= 5.0)
-    selenium-webdriver (4.1.0)
-      childprocess (>= 0.5, < 5.0)
-      rexml (~> 3.2, >= 3.2.5)
+    selenium-webdriver (3.142.7)
+      childprocess (>= 0.5, < 4.0)
       rubyzip (>= 1.2.2)
     semantic_range (3.0.0)
     sinatra (2.2.0)
@@ -262,10 +259,10 @@ DEPENDENCIES
   bootstrap_form (~> 4.0)
   browser (~> 2.2)
   byebug
-  capybara
+  capybara (< 3.36.0)
   climate_control (~> 0.2)
   coffee-rails (~> 4.2)
-  dalli
+  dalli (<= 3.2.0)
   data-confirm-modal (~> 1.2)
   dotenv-rails (~> 2.1)
   dotiw
@@ -276,6 +273,7 @@ DEPENDENCIES
   jquery-rails
   local_time (~> 1.0.3)
   mocha (~> 1.1)
+  nokogiri (~> 1.12.5)
   ood_appkit (~> 1.1)
   ood_core (~> 0.11)
   ood_support (~> 0.0.2)
@@ -284,7 +282,7 @@ DEPENDENCIES
   redcarpet (~> 3.3)
   sass-rails (~> 5.0)
   sdoc
-  selenium-webdriver
+  selenium-webdriver (< 4.0.0)
   sinatra
   sinatra-contrib
   thor (= 0.19.1)

apps/dashboard/bin/setup

@@ -60,12 +60,12 @@ chdir APP_ROOT do
 
   if APP.production?
     puts "\n== Compiling assets =="
-    sh "bin/rake assets:clobber"
-    sh "bin/rake assets:precompile"
+    sh "bundle exec rake assets:clobber"
+    sh "bundle exec rake assets:precompile"
   end
 
   puts "\n== Removing old logs and tempfiles =="
-  sh "bin/rake log:clear tmp:clear"
+  sh "bundle exec rake log:clear tmp:clear"
 
   puts "\n== Restarting application server =="
   touch "tmp/restart.txt"

apps/myjobs/Gemfile

@@ -63,6 +63,7 @@ gem "js-routes", "~> 1.2.4"
 gem 'osc_machete_rails', '~> 1.3.0'
 gem 'pbs', '~> 2.2.1'
 gem 'ood_appkit', '~> 1.1.4'
+gem 'nokogiri', '~> 1.12.5'
 
 # thor > 0.19.1 prints some warnings to stderr which will require changes
 # https://github.com/erikhuda/thor/issues/538

apps/myjobs/Gemfile.lock

@@ -106,21 +106,21 @@ GEM
     marcel (1.0.2)
     method_source (1.0.0)
     mini_mime (1.1.2)
-    mini_portile2 (2.8.0)
-    minitest (5.16.2)
+    mini_portile2 (2.6.1)
+    minitest (5.15.0)
     mocha (1.14.0)
     mustache (1.1.1)
     nio4r (2.5.8)
-    nokogiri (1.13.7)
-      mini_portile2 (~> 2.8.0)
+    nokogiri (1.12.5)
+      mini_portile2 (~> 2.6.1)
       racc (~> 1.4)
     ood_appkit (1.1.5)
       addressable (~> 2.4)
       lograge (~> 0.3)
       ood_core (~> 0.1)
       rails (> 4.0.7, < 6.0)
       redcarpet (~> 3.2)
-    ood_core (0.20.1)
+    ood_core (0.20.2)
       ffi (~> 1.9, >= 1.9.6)
       ood_support (~> 0.0.2)
       rexml (~> 3.2)
@@ -228,6 +228,7 @@ DEPENDENCIES
   js-routes (~> 1.2.4)
   local_time (~> 1.0.3)
   mocha (~> 1.1)
+  nokogiri (~> 1.12.5)
   ood_appkit (~> 1.1.4)
   osc_machete_rails (~> 1.3.0)
   pbs (~> 2.2.1)

apps/myjobs/bin/setup

@@ -46,12 +46,12 @@ chdir APP_ROOT do
 
   if APP.production?
     puts "\n== Compiling assets =="
-    sh "bin/rake assets:clobber"
-    sh "bin/rake assets:precompile"
+    sh "bundle exec rake assets:clobber"
+    sh "bundle exec rake assets:precompile"
   end
 
   puts "\n== Removing old logs and tempfiles =="
-  sh "bin/rake log:clear tmp:clear"
+  sh "bundle exec rake log:clear tmp:clear"
 
   puts "\n== Restarting application server =="
   touch "tmp/restart.txt"

nginx_stage/nginx_stage.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "syslog", "~> 0.1.0"
-  spec.add_dependency 'dotenv', '~> 2.1'
+  spec.add_dependency 'dotenv', '< 2.8'
 
   spec.add_development_dependency "bundler", "~> 2.1.0"
   spec.add_development_dependency "rake", "~> 13.0.1"

ood-portal-generator/lib/ood_portal_generator.rb

@@ -37,6 +37,13 @@ def debian?
       false
     end
 
+    def debian_legacy?
+      return false unless debian?
+      env = Dotenv.parse(os_release_file)
+      return true if env['VERSION_ID'] == '18.04'
+      false
+    end
+
     def fqdn
       Socket.gethostbyname(Socket.gethostname).first
     end

ood-portal-generator/lib/ood_portal_generator/view.rb

@@ -102,7 +102,7 @@ def initialize(opts = {})
       @oidc_crypto_passphrase           = Digest::SHA1.hexdigest(servername)
       @oidc_session_inactivity_timeout  = opts.fetch(:oidc_session_inactivity_timeout, 28800)
       @oidc_session_max_duration        = opts.fetch(:oidc_session_max_duration, 28800)
-      @oidc_state_max_number_of_cookies = opts.fetch(:oidc_state_max_number_of_cookies, '10 true')
+      @oidc_state_max_number_of_cookies = OodPortalGenerator.debian_legacy? ? nil : opts.fetch(:oidc_state_max_number_of_cookies, '10 true')
       @oidc_cookie_same_site            = opts.fetch(:oidc_cookie_same_site, @ssl ? 'Off' : 'On')
       @oidc_settings                    = opts.fetch(:oidc_settings, {})
     end

ood-portal-generator/templates/ood-portal.conf.erb

@@ -123,7 +123,9 @@ Listen <%= addr_port %>
   OIDCCryptoPassphrase <%= @oidc_crypto_passphrase %>
   OIDCSessionInactivityTimeout <%= @oidc_session_inactivity_timeout %>
   OIDCSessionMaxDuration <%= @oidc_session_max_duration %>
+  <%- if @oidc_state_max_number_of_cookies -%>
   OIDCStateMaxNumberOfCookies <%= @oidc_state_max_number_of_cookies %>
+  <%- end -%>
   OIDCCookieSameSite <%= @oidc_cookie_same_site %>
   <%- @oidc_settings.keys.sort.each do |oidc_setting| -%>
   <%= oidc_setting %> <%= @oidc_settings[oidc_setting] %>

packaging/deb/control

@@ -5,7 +5,7 @@ Maintainer: Ohio Supercomputer Center <oschelp@osc.edu>
 Build-Depends: debhelper (>=11~), curl, build-essential,
   ruby, ruby-dev, tzdata,
   nodejs (>= 12.0), nodejs (<< 13.0),
-  sqlite, libsqlite3-dev, python3
+  sqlite, libsqlite3-dev, python2.7
 Standards-Version: 4.1.4
 Homepage: https://openondemand.org
 

packaging/deb/postinst

@@ -33,4 +33,11 @@ ln -sf ../sites-available/ood-portal.conf . || true
 
 [ -e "/bin/systemctl" ] && /bin/systemctl try-restart apache2.service
 
+# Address bug in lua-posix package
+# https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
+. /etc/os-release
+if [ "${VERSION_ID}" = "18.04" ]; then
+  ln -snf posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so
+fi
+
 exit 0