Merge branch 'wireapp:develop' into develop

cabal.project

@@ -47,6 +47,7 @@ packages:
   , tools/db/find-undead/
   , tools/db/inconsistencies/
   , tools/db/migrate-sso-feature-flag/
+  , tools/db/migrate-features/
   , tools/db/move-team/
   , tools/db/phone-users/
   , tools/db/repair-handles/

changelog.d/0-release-notes/simplify-feature-table

@@ -1 +1,30 @@
-This release introduces a new data storage format for team features and a corresponding migration. While the migration is running, team features are going to operate in read-only mode for the team that is currently being migrated. After migration, the new storage is going to be used. No special action should be required on the part of instance operators.
+This release introduces a new data storage format for team features and a corresponding migration. To migrate to the new format, a new tool called `migrate-features` has been added. This tool needs to be run after deployment of this release, and before deploying the next release.
+
+While the migration tool is running, team features are going to operate in read-only mode for the team that is currently being migrated. After migration, the new storage is going to be used. No other action should be required on the part of instance operators besides running the migration tool.
+
+This tool can be run in kubernetes using a job like this:
+
+```yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: migrate-features
+  namespace: <namespace>
+spec:
+  template:
+    spec:
+      containers:
+        - name: migrate-features
+          image: quay.io/wire/migrate-features:5.12.0
+          args:
+            [
+              --cassandra-host-galley,
+              <galley-host>,
+              --cassandra-port-galley,
+              "9042",
+              --cassandra-keyspace-galley,
+              galley,
+            ]
+      restartPolicy: Never
+  backoffLimit: 4
+```

changelog.d/2-features/WPB-16142

@@ -0,0 +1 @@
+For SAML authenticated users: Do not require email verification for registered email domains.

changelog.d/3-bug-fixes/WPB-16333

@@ -0,0 +1 @@
+Prevent guest users from migrating to teams

hack/bin/cassandra_dump_schema

@@ -18,7 +18,7 @@ def main():
 
     ks = []
     for line in s.splitlines():
-        ks.append(re.split('\s+', line))
+        ks.append(re.split(r'\s+', line))
 
     keyspaces = transpose(ks)
     print("-- automatically generated with `make git-add-cassandra-schema`\n")

integration/test/API/BrigInternal.hs

@@ -59,6 +59,11 @@ getUsersId domain ids = do
   req <- baseRequest domain Brig Unversioned "/i/users"
   submit "GET" $ req & addQueryParams [("ids", intercalate "," ids)]
 
+getUsersByEmail :: (HasCallStack, MakesValue domain) => domain -> [String] -> App Response
+getUsersByEmail domain emails = do
+  req <- baseRequest domain Brig Unversioned "/i/users"
+  submit "GET" $ req & addQueryParams [("email", intercalate "," emails)]
+
 data FedConn = FedConn
   { domain :: String,
     searchStrategy :: String,

integration/test/SetupHelpers.hs

@@ -36,6 +36,7 @@ import Test.DNSMock
 import Testlib.JSON
 import Testlib.Prelude
 import Testlib.Printing (indent)
+import Text.Regex.TDFA ((=~))
 import qualified Text.XML as XML
 import qualified Text.XML.Cursor as XML
 import qualified Text.XML.DSig as SAML
@@ -46,6 +47,14 @@ randomUser domain cu = bindResponse (createUser domain cu) $ \resp -> do
   resp.status `shouldMatchInt` 201
   resp.json
 
+ephemeralUser :: (HasCallStack, MakesValue domain) => domain -> App Value
+ephemeralUser domain = do
+  name <- randomName
+  req <- baseRequest domain Brig Versioned "/register"
+  bindResponse (submit "POST" $ req & addJSONObject ["name" .= name] & addHeader "X-Forwarded-For" "127.0.0.42") $ \resp -> do
+    resp.status `shouldMatchInt` 201
+    resp.json
+
 deleteUser :: (HasCallStack, MakesValue user) => user -> App ()
 deleteUser user = bindResponse (API.Brig.deleteUser user) $ \resp -> do
   resp.status `shouldMatchInt` 200
@@ -430,19 +439,20 @@ registerTestIdPWithMetaWithPrivateCreds owner = do
 -- | Given a team configured with saml sso, attempt a login with valid credentials.  This
 -- function simulates client *and* IdP (instead of talking to an IdP).  It can be used to test
 -- scim-provisioned users as well as saml auto-provisioning without scim.
-loginWithSaml :: (HasCallStack) => Bool -> String -> String -> (String, (SAML.IdPMetadata, SAML.SignPrivCreds)) -> App ()
+loginWithSaml :: (HasCallStack) => Bool -> String -> String -> (String, (SAML.IdPMetadata, SAML.SignPrivCreds)) -> App (Maybe String, SAML.SignedAuthnResponse)
 loginWithSaml expectSuccess tid email (iid, (meta, privcreds)) = do
   let idpConfig = SAML.IdPConfig (SAML.IdPId (fromMaybe (error "invalid idp id") (UUID.fromString iid))) meta ()
   spmeta <- getSPMetadata OwnDomain tid
   authnreq <- initiateSamlLogin OwnDomain iid
   let nameId = fromRight (error "could not create name id") $ SAML.emailNameID (cs email)
   authnResp <- runSimpleSP $ SAML.mkAuthnResponseWithSubj nameId privcreds idpConfig (toSPMetaData spmeta.body) (parseAuthnReqResp authnreq.body) True
-  finalizeSamlLogin OwnDomain tid authnResp `bindResponse` validateLoginResp
+  mUid <- finalizeSamlLogin OwnDomain tid authnResp `bindResponse` validateLoginResp
+  pure (mUid, authnResp)
   where
     toSPMetaData :: ByteString -> SAML.SPMetadata
     toSPMetaData bs = fromRight (error "could not decode spmetatdata") $ SAML.decode $ cs bs
 
-    validateLoginResp :: (HasCallStack) => Response -> App ()
+    validateLoginResp :: (HasCallStack) => Response -> App (Maybe String)
     validateLoginResp resp =
       if expectSuccess
         then do
@@ -467,12 +477,23 @@ loginWithSaml expectSuccess tid email (iid, (meta, privcreds)) = do
           bdy `shouldContain` "}, receiverOrigin)"
           hasPersistentCookieHeader resp
 
-    hasPersistentCookieHeader :: Response -> App ()
+    hasPersistentCookieHeader :: Response -> App (Maybe String)
     hasPersistentCookieHeader rsp = do
-      let cookie = getCookie "zuid" rsp
-      case cookie of
-        Nothing -> expectSuccess `shouldMatch` False
-        Just _ -> expectSuccess `shouldMatch` True
+      let mCookie = getCookie "zuid" rsp
+      case mCookie of
+        Nothing -> do
+          expectSuccess `shouldMatch` False
+          pure Nothing
+        Just cookie -> do
+          expectSuccess `shouldMatch` True
+          pure $ getUserIdFromCookie cookie
+
+    getUserIdFromCookie :: String -> Maybe String
+    getUserIdFromCookie cookie = do
+      let regex = "u=([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})"
+      case cookie =~ regex :: [[String]] of
+        [[_, uuid]] -> Just uuid
+        _ -> Nothing
 
     runSimpleSP :: SAML.SimpleSP a -> App a
     runSimpleSP action = liftIO $ do
@@ -549,3 +570,22 @@ setupOwnershipToken domain registrationDomain = do
     resp.json %. "domain_ownership_token" & asString
 
   pure $ DomainRegistrationSetup challenge.dnsToken challenge.technitiumToken ownershipToken
+
+activateEmail :: (HasCallStack, MakesValue domain) => domain -> String -> App ()
+activateEmail domain email = do
+  (actkey, code) <- bindResponse (getActivationCode domain email) $ \res -> do
+    (,)
+      <$> (res.json %. "key" >>= asString)
+      <*> (res.json %. "code" >>= asString)
+  API.Brig.activate domain actkey code >>= assertSuccess
+
+registerInvitedUser :: (HasCallStack, MakesValue domain) => domain -> String -> String -> App ()
+registerInvitedUser domain tid email = do
+  getInvitationByEmail domain email
+    >>= getJSON 200
+    >>= getInvitationCodeForTeam domain tid
+    >>= getJSON 200
+    >>= (%. "code")
+    >>= asString
+    >>= registerUser domain email
+    >>= assertSuccess

integration/test/Test/DomainVerification.hs

@@ -7,6 +7,7 @@ import API.Brig
 import API.BrigInternal
 import API.Common
 import API.GalleyInternal (setTeamFeatureLockStatus, setTeamFeatureStatus)
+import API.Spar
 import Control.Concurrent (threadDelay)
 import SetupHelpers
 import Test.DNSMock
@@ -504,3 +505,172 @@ testGetDomainRegistrationUserExists = do
     resp.json %. "domain_redirect" `shouldMatch` "none"
     lookupField resp.json "backend_url" `shouldMatch` (Nothing :: Maybe String)
     resp.json %. "due_to_existing_account" `shouldMatch` True
+
+testSsoLoginNoEmailVerification :: (HasCallStack) => App ()
+testSsoLoginNoEmailVerification = do
+  (owner, tid, _) <- createTeam OwnDomain 1
+  emailDomain <- randomDomain
+
+  assertSuccess =<< do
+    setTeamFeatureLockStatus owner tid "domainRegistration" "unlocked"
+    setTeamFeatureStatus owner tid "domainRegistration" "enabled"
+
+  setup <- setupOwnershipToken OwnDomain emailDomain
+  authorizeTeam owner emailDomain setup.ownershipToken >>= assertSuccess
+
+  void $ setTeamFeatureStatus owner tid "sso" "enabled"
+  (idp, idpMeta) <- registerTestIdPWithMetaWithPrivateCreds owner
+  idpId <- asString $ idp.json %. "id"
+
+  updateTeamInvite owner emailDomain (object ["team_invite" .= "not-allowed", "sso" .= idpId]) >>= assertSuccess
+
+  let email = "user@" <> emailDomain
+  (Just uid, _) <- loginWithSaml True tid email (idpId, idpMeta)
+  getUsersId OwnDomain [uid] `bindResponse` \res -> do
+    res.status `shouldMatchInt` 200
+    user <- res.json >>= asList >>= assertOne
+    user %. "status" `shouldMatch` "active"
+    user %. "email" `shouldMatch` email
+
+  getUsersByEmail OwnDomain [email] `bindResponse` \res -> do
+    res.status `shouldMatchInt` 200
+    user <- res.json >>= asList >>= assertOne
+    user %. "status" `shouldMatch` "active"
+    user %. "email" `shouldMatch` email
+
+  otherEmailDomain <- randomDomain
+  let otherEmail = "otherUser@" <> otherEmailDomain
+  (Just otherUid, _) <- loginWithSaml True tid otherEmail (idpId, idpMeta)
+
+  getUsersId OwnDomain [otherUid] `bindResponse` \res -> do
+    res.status `shouldMatchInt` 200
+    user <- res.json >>= asList >>= assertOne
+    user %. "status" `shouldMatch` "active"
+    lookupField user "email" `shouldMatch` (Nothing :: Maybe String)
+
+  getUsersByEmail OwnDomain [otherEmail] `bindResponse` \res -> do
+    res.status `shouldMatchInt` 200
+    res.json >>= asList >>= shouldBeEmpty
+
+  activateEmail OwnDomain otherEmail
+  getUsersId OwnDomain [otherUid] `bindResponse` \res -> do
+    res.status `shouldMatchInt` 200
+    user <- res.json >>= asList >>= assertOne
+    user %. "status" `shouldMatch` "active"
+    user %. "email" `shouldMatch` otherEmail
+
+testScimOnlyWithRegisteredEmailDomain :: (HasCallStack) => App ()
+testScimOnlyWithRegisteredEmailDomain = do
+  (owner, tid, _) <- createTeam OwnDomain 1
+  emailDomain <- randomDomain
+
+  assertSuccess =<< do
+    setTeamFeatureLockStatus owner tid "domainRegistration" "unlocked"
+    setTeamFeatureStatus owner tid "domainRegistration" "enabled"
+
+  setup <- setupOwnershipToken OwnDomain emailDomain
+  authorizeTeam owner emailDomain setup.ownershipToken >>= assertSuccess
+
+  updateTeamInvite owner emailDomain (object ["team_invite" .= "team", "team" .= tid]) >>= assertSuccess
+
+  tok <- createScimToken owner def >>= getJSON 200 >>= (%. "token") >>= asString
+  let email = "user@" <> emailDomain
+      extId = email
+  scimUser <- randomScimUserWith extId email
+  uid <- bindResponse (createScimUser owner tok scimUser) $ \resp -> do
+    resp.status `shouldMatchInt` 201
+    resp.json %. "id" >>= asString
+  registerInvitedUser OwnDomain tid email
+  bindResponse (login OwnDomain email defPassword) $ \resp -> do
+    resp.status `shouldMatchInt` 200
+  getUsersId OwnDomain [uid] `bindResponse` \res -> do
+    res.status `shouldMatchInt` 200
+    user <- res.json >>= asList >>= assertOne
+    user %. "status" `shouldMatch` "active"
+    user %. "email" `shouldMatch` email
+  getUsersByEmail OwnDomain [email] `bindResponse` \res -> do
+    res.status `shouldMatchInt` 200
+    user <- res.json >>= asList >>= assertOne
+    user %. "status" `shouldMatch` "active"
+    user %. "email" `shouldMatch` email
+
+testScimAndSamlWithRegisteredEmailDomain :: (HasCallStack) => App ()
+testScimAndSamlWithRegisteredEmailDomain = do
+  (owner, tid, _) <- createTeam OwnDomain 1
+  emailDomain <- randomDomain
+
+  assertSuccess =<< do
+    setTeamFeatureLockStatus owner tid "domainRegistration" "unlocked"
+    setTeamFeatureStatus owner tid "domainRegistration" "enabled"
+
+  setup <- setupOwnershipToken OwnDomain emailDomain
+  authorizeTeam owner emailDomain setup.ownershipToken >>= assertSuccess
+
+  void $ setTeamFeatureStatus owner tid "sso" "enabled"
+  (idp, idpMeta) <- registerTestIdPWithMetaWithPrivateCreds owner
+  idpId <- asString $ idp.json %. "id"
+
+  updateTeamInvite owner emailDomain (object ["team_invite" .= "not-allowed", "sso" .= idpId]) >>= assertSuccess
+
+  tok <-
+    createScimToken owner def {idp = Just idpId}
+      >>= getJSON 200
+      >>= (%. "token")
+      >>= asString
+  let email = "user@" <> emailDomain
+      extId = email
+  scimUser <- randomScimUserWith extId email
+  uid <- bindResponse (createScimUser owner tok scimUser) $ \resp -> do
+    resp.status `shouldMatchInt` 201
+    resp.json %. "id" >>= asString
+  void $ loginWithSaml True tid email (idpId, idpMeta)
+
+  getUsersId OwnDomain [uid] `bindResponse` \res -> do
+    res.status `shouldMatchInt` 200
+    user <- res.json >>= asList >>= assertOne
+    user %. "status" `shouldMatch` "active"
+    user %. "email" `shouldMatch` email
+
+  getUsersByEmail OwnDomain [email] `bindResponse` \res -> do
+    res.status `shouldMatchInt` 200
+    user <- res.json >>= asList >>= assertOne
+    user %. "status" `shouldMatch` "active"
+    user %. "email" `shouldMatch` email
+
+testVerificationRequiredIfEmailDomainRedirectNotSso :: (HasCallStack) => App ()
+testVerificationRequiredIfEmailDomainRedirectNotSso = do
+  (owner, tid, _) <- createTeam OwnDomain 1
+  emailDomain <- randomDomain
+
+  assertSuccess =<< do
+    setTeamFeatureLockStatus owner tid "domainRegistration" "unlocked"
+    setTeamFeatureStatus owner tid "domainRegistration" "enabled"
+
+  setup <- setupOwnershipToken OwnDomain emailDomain
+  authorizeTeam owner emailDomain setup.ownershipToken >>= assertSuccess
+
+  void $ setTeamFeatureStatus owner tid "sso" "enabled"
+  (idp, idpMeta) <- registerTestIdPWithMetaWithPrivateCreds owner
+  idpId <- asString $ idp.json %. "id"
+
+  updateTeamInvite owner emailDomain (object ["team_invite" .= "team", "team" .= tid]) >>= assertSuccess
+
+  let email = "user@" <> emailDomain
+  (Just uid, _) <- loginWithSaml True tid email (idpId, idpMeta)
+
+  getUsersId OwnDomain [uid] `bindResponse` \res -> do
+    res.status `shouldMatchInt` 200
+    user <- res.json >>= asList >>= assertOne
+    user %. "status" `shouldMatch` "active"
+    lookupField user "email" `shouldMatch` (Nothing :: Maybe String)
+
+  getUsersByEmail OwnDomain [email] `bindResponse` \res -> do
+    res.status `shouldMatchInt` 200
+    res.json >>= asList >>= shouldBeEmpty
+
+  activateEmail OwnDomain email
+  getUsersId OwnDomain [uid] `bindResponse` \res -> do
+    res.status `shouldMatchInt` 200
+    user <- res.json >>= asList >>= assertOne
+    user %. "status" `shouldMatch` "active"
+    user %. "email" `shouldMatch` email