Merge pull request #33 from Hub-of-all-Things/2.4.0

2.4.0 ready for use

Author: AndriusA

.gitmodules

@@ -1,12 +1,6 @@
 [submodule "hat-database-schema"]
 	path = hat/conf/evolutions/hat-database-schema
 	url = https://github.com/Hub-of-all-Things/hat-database-schema.git
-[submodule "hat-client-scala-play"]
-	path = hat-client-scala-play
-	url = git@github.com:Hub-of-all-Things/hat-client-scala-play.git
-[submodule "marketsquare-client-scala-play"]
-	path = marketsquare-client-scala-play
-	url = git@github.com:Hub-of-all-Things/marketsquare-client-scala-play.git
 [submodule "frontend"]
   	path = frontend
    	url = git@github.com:Hub-of-all-Things/Rumpel.git

.travis.yml

@@ -1,5 +1,7 @@
 language: scala
 
+dist: trusty
+
 scala:
    - 2.11.8
 jdk:
@@ -12,27 +14,32 @@ branches:
     - dev
 
 addons:
-  postgresql: "9.4"
+  postgresql: "9.5"
 
 cache:
   directories:
   - $HOME/.sbt
   - $HOME/.ivy2
 
+# Handle git submodules yourself
+git:
+    submodules: false
+
 before_script:
-  - cd ./hat-database-schema
-  - source ./env.sh
-  - bash ./setupDatabase.sh
-  - bash ./setupAccess.sh
-  - bash ./applyEvolutions.sh -c structuresonly,testdata
-  - cd ../
-  - sed -e "s;%DATABASE%;$DATABASE;g" -e "s;%DBUSER%;$DBUSER;g" -e "s;%DBPASS%;$DBPASS;g" deployment/database.conf.template > src/main/resources/database.conf
-  - cp ./src/main/resources/database.conf ./codegen/src/main/resources/database.conf
+  - psql -c "CREATE DATABASE testhatdb1;" -U postgres
+  - psql -c "CREATE USER testhatdb1 WITH PASSWORD 'testing';" -U postgres
+  - psql -c "GRANT CREATE ON DATABASE testhatdb1 TO testhatdb1" -U postgres
+
+# Use sed to replace the SSH URL with the public URL, then initialize submodules
+before_install:
+  - sed -i 's/git@github.com:/https:\/\/github.com\//' .gitmodules
+  - git submodule update --init --recursive
+
 
 script:
    - sbt clean
    - sbt ++$TRAVIS_SCALA_VERSION compile test:compile
-   - sbt coverage "testOnly hatdex.hat.api.*" -Dconfig.file=src/main/resources/application.test.conf
+   - sbt coverage "testOnly org.hatdex.hat.api.*" -Dconfig.file=src/main/resources/application.test.conf
 
 after_success:
   - sbt coveralls

README.md

@@ -79,7 +79,7 @@ To launch the HAT, follow these steps:
     ```
 4. Run the project:
     ```
-    sbt "project hat" run
+    sbt "project hat" -Dconfig.resource=dev.conf run
     ```
 5. Go to http://bobtheplumber.hat.org:9000
 

codegen/build.sbt

@@ -10,28 +10,6 @@ libraryDependencies ++= Seq(
   Library.Slick.slickPgPlayJson,
   Library.Slick.slickCodegen,
   Library.Akka.slf4j,
-  Library.Akka.httpCore,
-  Library.Akka.akkaStream,
-  Library.Akka.akkaHttpSprayJson,
-  Library.Utils.jodaTime,
-  Library.Utils.jodaConvert,
-  Library.Utils.jts,
-  Library.Utils.slf4j,
-  Library.Utils.logbackCore,
-  Library.Utils.logbackClassic,
-
-  Library.Slick.slickPgCore,
-  Library.Slick.slickPg,
-  Library.Slick.slickPgJoda,
-  Library.Slick.slickPgJts,
-  Library.Slick.slickPgSprayJson,
-  Library.Slick.slickCodegen,
-  Library.Akka.slf4j,
-  Library.Akka.httpCore,
-  Library.Akka.akkaStream,
-  Library.Akka.akkaHttpSprayJson,
-  Library.Akka.akkaActor,
-  Library.Akka.akkaTestkit,
   Library.Utils.jodaTime,
   Library.Utils.jodaConvert,
   Library.Utils.jts,

codegen/src/main/scala/org/hatdex/hat/dal/SlickPostgresDriver.scala

@@ -24,8 +24,12 @@
 
 package org.hatdex.hat.dal
 
+import java.sql.Timestamp
+
 import com.github.tminglei.slickpg._
-import play.api.libs.json.{ JsValue, Json }
+import org.joda.time.DateTime
+import play.api.libs.json.{ JsNull, JsValue, Json }
+import slick.jdbc.JdbcType
 
 trait SlickPostgresDriver extends ExPostgresDriver
     with PgArraySupport
@@ -38,6 +42,7 @@ trait SlickPostgresDriver extends ExPostgresDriver
 
   override val pgjson = "jsonb"
   override val api = MyAPI
+  override protected lazy val useTransactionForUpsert = false
 
   object MyAPI extends API with ArrayImplicits
       with DateTimeImplicits
@@ -53,6 +58,36 @@ trait SlickPostgresDriver extends ExPostgresDriver
         pgjson,
         (s) => utils.SimpleArrayUtils.fromString[JsValue](Json.parse(_))(s).orNull,
         (v) => utils.SimpleArrayUtils.mkString[JsValue](_.toString())(v)).to(_.toList)
+
+    import scala.language.implicitConversions
+
+    override implicit val playJsonTypeMapper: JdbcType[JsValue] =
+      new GenericJdbcType[JsValue](
+        pgjson,
+        (v) => Json.parse(v),
+        (v) => Json.stringify(v),
+        zero = JsNull,
+        hasLiteralForm = false)
+
+    override implicit def playJsonColumnExtensionMethods(c: Rep[JsValue]) = {
+      new FixedJsonColumnExtensionMethods[JsValue, JsValue](c)
+    }
+    override implicit def playJsonOptionColumnExtensionMethods(c: Rep[Option[JsValue]]) = {
+      new FixedJsonColumnExtensionMethods[JsValue, Option[JsValue]](c)
+    }
+
+    class FixedJsonColumnExtensionMethods[JSONType, P1](override val c: Rep[P1])(
+        implicit
+        tm: JdbcType[JSONType]) extends JsonColumnExtensionMethods[JSONType, P1](c) {
+      override def <@:[P2, R](c2: Rep[P2])(implicit om: o#arg[JSONType, P2]#to[Boolean, R]) = {
+        om.column(jsonLib.ContainsBy, n, c2.toNode)
+      }
+    }
+
+    val toJson: Rep[String] => Rep[JsValue] = SimpleFunction.unary[String, JsValue]("to_jsonb")
+    val toTimestamp: Rep[Double] => Rep[Timestamp] = SimpleFunction.unary[Double, Timestamp]("to_timestamp")
+    val datePart: (Rep[String], Rep[DateTime]) => Rep[String] = SimpleFunction.binary[String, DateTime, String]("date_part")
+    val datePartTimestamp: (Rep[String], Rep[Timestamp]) => Rep[String] = SimpleFunction.binary[String, Timestamp, String]("date_part")
   }
 
 }

deployment/docker/hat/Dockerfile

@@ -1,4 +1,4 @@
-FROM java:jre
+FROM openjdk:8-jre
 WORKDIR /opt/docker
 ADD opt /opt
 RUN ["chown", "-R", "daemon:daemon", "."]

deployment/docker/hat/docker-build.sh

@@ -6,12 +6,13 @@ set -e
 HAT_HOME=${HAT_HOME:-"$PWD/hat"} #if executing from deployment/  : "$PWD/../.."
 DOCKER=${DOCKER:-"$PWD/deployment/docker/hat"}
 DOCKER_DEPLOY=$DOCKER/docker-deploy
+APP=${APPLICATION_NAME:-hat-experimental}
 
 echo "Creating $DOCKER_DEPLOY"
 mkdir $DOCKER_DEPLOY
 
-echo "Building HAT : sbt docker:stage"
-sbt docker:stage
+echo "Building ${APP} : sbt docker:stage"
+sbt "project hat" docker:stage
 
 if [ ! -f "$HAT_HOME/target/docker/Dockerfile" ]; then
     echo "Missing $HAT_HOME/target/docker/Dockerfile"
@@ -24,8 +25,8 @@ cp -r $HAT_HOME/target/docker/stage/opt $DOCKER_DEPLOY/
 
 cp $DOCKER/Dockerfile $DOCKER_DEPLOY/Dockerfile
 
-echo "Building hat docker image: docker-hat"
-docker build -t hubofallthings/hat-experimental $DOCKER_DEPLOY
+echo "Building hat docker image: docker-hat hubofallthings/${APP}"
+docker build -t hubofallthings/${APP} $DOCKER_DEPLOY
 
 echo "Cleaning up"
 rm -r $DOCKER_DEPLOY

deployment/ecs/docker-aws-deploy.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -e
+
+VERSION=`git log --format="%H" -n 1`
+
+echo "Create package"
+export APPLICATION_NAME="hat"
+HAT_HOME=${PWD} #if executing from deployment/  : "$PWD/../.."
+DOCKER=${DOCKER:-"${HAT_HOME}/deployment/docker"}
+bash ${DOCKER}/hat/docker-build.sh
+
+echo "Publish to AWS"
+docker tag hubofallthings/${APPLICATION_NAME}:latest 717711705314.dkr.ecr.eu-west-1.amazonaws.com/hubofallthings:${VERSION}
+docker push 717711705314.dkr.ecr.eu-west-1.amazonaws.com/hubofallthings:${VERSION}
+
+echo "Built application version ${APPLICATION_NAME} ${VERSION}"

frontend

@@ -24,19 +24,19 @@
 
 package org.hatdex.hat.api.controllers
 
+import java.net.URLDecoder
 import javax.inject.Inject
 
 import com.mohiva.play.silhouette.api.repositories.AuthInfoRepository
 import com.mohiva.play.silhouette.api.util.{ Clock, Credentials, PasswordHasherRegistry }
 import com.mohiva.play.silhouette.api.{ LoginEvent, Silhouette }
-import com.mohiva.play.silhouette.impl.exceptions.InvalidPasswordException
+import com.mohiva.play.silhouette.impl.exceptions.{ IdentityNotFoundException, InvalidPasswordException }
 import com.mohiva.play.silhouette.impl.providers.CredentialsProvider
 import org.hatdex.hat.api.json.HatJsonFormats
-import org.hatdex.hat.api.models.{ ErrorMessage, SuccessResponse }
-import org.hatdex.hat.api.service.UsersService
+import org.hatdex.hat.api.models._
+import org.hatdex.hat.api.service.{ HatServicesService, MailTokenService, UsersService }
 import org.hatdex.hat.authentication._
 import org.hatdex.hat.phata.models.{ ApiPasswordChange, ApiPasswordResetRequest, MailTokenUser }
-import org.hatdex.hat.phata.service.{ HatServicesService, MailTokenService }
 import org.hatdex.hat.resourceManagement.{ HatServerProvider, _ }
 import org.hatdex.hat.utils.{ HatBodyParsers, HatMailer }
 import play.api.i18n.MessagesApi
@@ -64,7 +64,16 @@ class Authentication @Inject() (
 
   private val logger = Logger(this.getClass)
 
-  def hatLogin(name: String, redirectUrl: String): Action[AnyContent] = SecuredAction(WithRole("owner")).async { implicit request =>
+  def publicKey(): Action[AnyContent] = UserAwareAction.async { implicit request =>
+    val publicKey = hatServerProvider.toString(request.dynamicEnvironment.publicKey)
+    Future.successful(Ok(publicKey))
+  }
+
+  def validateToken(): Action[AnyContent] = SecuredAction.async { implicit request =>
+    Future.successful(Ok(Json.toJson(SuccessResponse("Authenticated"))))
+  }
+
+  def hatLogin(name: String, redirectUrl: String): Action[AnyContent] = SecuredAction(WithRole(Owner())).async { implicit request =>
     for {
       service <- hatServicesService.findOrCreateHatService(name, redirectUrl)
       linkedService <- hatServicesService.hatServiceLink(request.identity, service, Some(redirectUrl))
@@ -74,7 +83,55 @@ class Authentication @Inject() (
     }
   }
 
-  def passwordChangeProcess: Action[ApiPasswordChange] = SecuredAction(WithRole("owner")).async(parsers.json[ApiPasswordChange]) { implicit request =>
+  def applicationToken(name: String, resource: String): Action[AnyContent] = SecuredAction(WithRole(Owner())).async { implicit request =>
+    for {
+      service <- hatServicesService.findOrCreateHatService(name, resource)
+      token <- hatServicesService.hatServiceToken(request.identity, service)
+      result <- env.authenticatorService.embed(token.accessToken, Ok(Json.toJson(token)))
+    } yield {
+      result
+    }
+  }
+
+  private val hatService = HatService(
+    "hat", "hat", "HAT API",
+    "", "", "",
+    browser = true,
+    category = "api",
+    setup = true,
+    loginAvailable = true)
+  def accessToken(): Action[AnyContent] = UserAwareAction.async { implicit request =>
+    val eventuallyAuthenticatedUser = for {
+      usernameParam <- request.getQueryString("username").orElse(request.headers.get("username"))
+      passwordParam <- request.getQueryString("password").orElse(request.headers.get("password"))
+    } yield {
+      val username = usernameParam
+      val password = URLDecoder.decode(passwordParam, "UTF-8")
+      logger.info(s"Authenticating $username:$password")
+      credentialsProvider.authenticate(Credentials(username, password))
+        .flatMap { loginInfo =>
+          usersService.getUser(loginInfo.providerKey).flatMap {
+            case Some(user) =>
+              val customClaims = hatServicesService.generateUserTokenClaims(user, hatService)
+              for {
+                authenticator <- env.authenticatorService.create(loginInfo)
+                token <- env.authenticatorService.init(authenticator.copy(customClaims = Some(customClaims)))
+                _ <- usersService.logLogin(user, "api", user.roles.filter(_.extra.isEmpty).map(_.title).mkString(":"), None, None)
+                result <- env.authenticatorService.embed(token, Ok(Json.toJson(AccessToken(token, user.userId))))
+              } yield {
+                env.eventBus.publish(LoginEvent(user, request))
+                result
+              }
+            case None => Future.failed(new IdentityNotFoundException("Couldn't find user"))
+          }
+        }
+    }
+    eventuallyAuthenticatedUser getOrElse {
+      Future.successful(Unauthorized(Json.toJson(ErrorMessage("Credentials required", "No username or password provided to retrieve token"))))
+    }
+  }
+
+  def passwordChangeProcess: Action[ApiPasswordChange] = SecuredAction(WithRole(Owner())).async(parsers.json[ApiPasswordChange]) { implicit request =>
     request.body.password map { oldPassword =>
       val eventualResult = for {
         _ <- credentialsProvider.authenticate(Credentials(request.identity.email, oldPassword))
@@ -102,11 +159,10 @@ class Authentication @Inject() (
     val email = request.body.email
     val response = Ok(Json.toJson(SuccessResponse("If the email you have entered is correct, you will shortly receive an email with password reset instructions")))
     if (email == request.dynamicEnvironment.ownerEmail) {
-      usersService.listUsers.map(_.find(_.role == "owner")).flatMap {
+      usersService.listUsers.map(_.find(_.roles.contains(Owner()))).flatMap {
         case Some(user) =>
           val token = MailTokenUser(email, isSignUp = false)
           tokenService.create(token).map { _ =>
-            // TODO generate password reset link for frontend to handle
             val scheme = if (request.secure) {
               "https://"
             }
@@ -134,7 +190,7 @@ class Authentication @Inject() (
     tokenService.retrieve(tokenId).flatMap {
       case Some(token) if !token.isSignUp && !token.isExpired =>
         if (token.email == request.dynamicEnvironment.ownerEmail) {
-          usersService.listUsers.map(_.find(_.role == "owner")).flatMap {
+          usersService.listUsers.map(_.find(_.roles.contains(Owner()))).flatMap {
             case Some(user) =>
               for {
                 _ <- authInfoRepository.update(user.loginInfo, passwordHasherRegistry.current.hash(request.body.newPassword))
@@ -150,6 +206,7 @@ class Authentication @Inject() (
           }
         }
         else {
+          logger.info(s"Token email: ${token.email}, while owner email is ${request.dynamicEnvironment.ownerEmail}")
           Future.successful(Unauthorized(Json.toJson(ErrorMessage("Password reset unauthorized", "Only HAT owner can reset their password"))))
         }
       case Some(_) =>