Merge pull request #74 from Hub-of-all-Things/dev

Release v2.6.6

Author: augustinas

hat/app/org/hatdex/hat/api/controllers/SystemStatus.scala

@@ -33,16 +33,19 @@ import org.hatdex.hat.api.service.{ SystemStatusService, UsersService }
 import org.hatdex.hat.authentication.{ ContainsApplicationRole, HatApiAuthEnvironment, HatApiController, WithRole }
 import org.hatdex.hat.resourceManagement._
 import org.ocpsoft.prettytime.PrettyTime
-import play.api.Logger
-import play.api.cache.Cached
+import play.api.{ Configuration, Logger }
+import play.api.cache.{ AsyncCacheApi, Cached }
 import play.api.libs.json._
 import play.api.mvc._
 
-import scala.concurrent.{ ExecutionContext }
+import scala.concurrent.{ ExecutionContext, Future }
+import scala.util.{ Failure, Success }
 
 class SystemStatus @Inject() (
     components: ControllerComponents,
     cached: Cached,
+    cache: AsyncCacheApi,
+    configuration: Configuration,
     silhouette: Silhouette[HatApiAuthEnvironment],
     systemStatusService: SystemStatusService,
     usersService: UsersService,
@@ -51,6 +54,10 @@ class SystemStatus @Inject() (
     val ec: ExecutionContext,
     val applicationsService: ApplicationsService) extends HatApiController(components, silhouette) with HatJsonFormats {
 
+  private val dbStorageAllowance: Long = configuration.get[Long]("resourceManagement.hatDBStorageAllowance")
+  private val fileStorageAllowance: Long = configuration.get[Long]("resourceManagement.hatFileStorageAllowance")
+  private val hatSharedSecret: String = configuration.get[String]("resourceManagement.hatSharedSecret")
+
   private val logger = Logger(this.getClass)
 
   private val indefiniteSuccessCaching = cached
@@ -68,9 +75,6 @@ class SystemStatus @Inject() (
 
   def status(): Action[AnyContent] =
     SecuredAction(WithRole(Owner(), Platform()) || ContainsApplicationRole(Owner(), Platform())).async { implicit request =>
-      val dbStorageAllowance = Math.pow(1000, 3).toLong
-      val fileStorageAllowance = Math.pow(1000, 3).toLong
-
       val eventualStatus = for {
         dbsize <- systemStatusService.tableSizeTotal
         fileSize <- systemStatusService.fileStorageTotal
@@ -100,5 +104,28 @@ class SystemStatus @Inject() (
         Ok(Json.toJson(stats))
       }
     }
+
+  def destroyCache: Action[AnyContent] = UserAwareAction.async { implicit request =>
+    request.headers.get("X-Auth-Token") match {
+      case Some(authToken) if authToken == hatSharedSecret =>
+        val response = Ok(Json.toJson(SuccessResponse("beforeDestroy DONE")))
+
+        val hatAddress = request.dynamicEnvironment.domain
+        val clearApps = cache.remove(s"apps:$hatAddress")
+        val clearConfiguration = cache.remove(s"configuration:$hatAddress")
+        val clearServer = cache.remove(s"server:$hatAddress")
+
+        // We don't care if the cache is successfully cleared. We just go ahead and return successful.
+        Future.sequence(Seq(clearApps, clearConfiguration, clearServer)).onComplete {
+          case Success(_)         => logger.info(s"BEFORE DESTROY: $hatAddress DONE")
+          case Failure(exception) => logger.error(s"BEFORE DESTROY: Could not clear cache of $hatAddress. Reason: ${exception.getMessage}")
+        }
+        Future.successful(response)
+
+      case Some(_) => Future.successful(Forbidden(Json.toJson(ErrorMessage("Invalid token", s"Not a valid token provided"))))
+
+      case None    => Future.successful(Forbidden(Json.toJson(ErrorMessage("Credentials missing", s"Credentials required"))))
+    }
+  }
 }
 

hat/app/org/hatdex/hat/phata/assets/js/main.bundle.js

@@ -0,0 +1,56 @@
+/*
+ * Copyright (C) 2019 HAT Data Exchange Ltd
+ * SPDX-License-Identifier: AGPL-3.0
+ *
+ * This file is part of the Hub of All Things project (HAT).
+ *
+ * HAT is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License
+ * as published by the Free Software Foundation, version 3 of
+ * the License.
+ *
+ * HAT is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See
+ * the GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General
+ * Public License along with this program. If not, see
+ * <http://www.gnu.org/licenses/>.
+ *
+ * Written by Marios Tsekis <marios.tsekis@hatdex.org>
+ * 2 / 2019
+ */
+package org.hatdex.hat.she.controllers
+
+import com.mohiva.play.silhouette.api.Silhouette
+import javax.inject.Inject
+import org.hatdex.hat.api.json.RichDataJsonFormats
+import org.hatdex.hat.api.models.Owner
+import org.hatdex.hat.api.service.applications.ApplicationsService
+import org.hatdex.hat.authentication.{ ContainsApplicationRole, HatApiAuthEnvironment, HatApiController, WithRole }
+import org.hatdex.hat.she.models.FunctionConfigurationJsonProtocol
+import org.hatdex.hat.she.service.StaticDataGeneratorService
+import play.api.libs.json.Json
+import play.api.mvc._
+
+import scala.concurrent.ExecutionContext
+
+class StaticDataGenerator @Inject() (
+    components: ControllerComponents,
+    silhouette: Silhouette[HatApiAuthEnvironment],
+    feedGeneratorService: StaticDataGeneratorService)(
+    implicit
+    val ec: ExecutionContext,
+    applicationsService: ApplicationsService)
+  extends HatApiController(components, silhouette)
+  with RichDataJsonFormats
+  with FunctionConfigurationJsonProtocol {
+
+  def getStaticData(endpoint: String): Action[AnyContent] = {
+    SecuredAction(WithRole(Owner()) || ContainsApplicationRole(Owner())).async { implicit request ⇒
+      feedGeneratorService.getStaticData(endpoint)
+        .map(items ⇒ Ok(Json.toJson(items)))
+    }
+  }
+}

hat/app/org/hatdex/hat/phata/assets/js/main.bundle.js.gz

@@ -0,0 +1,34 @@
+/*
+ * Copyright (C) 2019 HAT Data Exchange Ltd
+ * SPDX-License-Identifier: AGPL-3.0
+ *
+ * This file is part of the Hub of All Things project (HAT).
+ *
+ * HAT is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License
+ * as published by the Free Software Foundation, version 3 of
+ * the License.
+ *
+ * HAT is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See
+ * the GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General
+ * Public License along with this program. If not, see
+ * <http://www.gnu.org/licenses/>.
+ *
+ * Written by Marios Tsekis <marios.tsekis@hatdex.org>
+ * 2 / 2019
+ */
+package org.hatdex.hat.she.models
+
+import play.api.libs.json.{ JsValue, Json, Writes }
+
+case class StaticDataValues(name: String, values: Map[String, JsValue]) {
+
+}
+
+object StaticDataValues {
+  implicit val facebookLocationReads: Writes[StaticDataValues] = Json.writes[StaticDataValues]
+}

hat/app/org/hatdex/hat/phata/assets/js/polyfills.bundle.js.gz

@@ -227,6 +227,10 @@ class InsightsMapper extends DataEndpointMapper {
     "facebook/feed" → "Posts composed",
     "notables/feed" → "Notes taken",
     "spotify/feed" → "Songs listened to",
+    "instagram/feed" → "Photos uploaded",
+    "fitbit/weight" -> "Fitbit weight records",
+    "fitbit/sleep" -> "Fitbit sleep records",
+    "fitbit/activity" -> "Fitbit activities logged",
     "calendar/google/events" → "Calendar events recorded",
     "monzo/transactions" → "Transactions performed",
     "she/insights/emotions" -> "Posts analysed for Sentiments",
@@ -239,6 +243,10 @@ class InsightsMapper extends DataEndpointMapper {
     "facebook/feed" → "facebook",
     "notables/feed" → "notables",
     "spotify/feed" → "spotify",
+    "instagram/feed" → "instagram",
+    "fitbit/weight" -> "fitbit-weight",
+    "fitbit/sleep" -> "fitbit-sleep",
+    "fitbit/activity" -> "fitbit-activity",
     "calendar/google/events" → "google",
     "monzo/transactions" → "monzo",
     "she/insights/emotions" -> "sentiment",
@@ -622,8 +630,8 @@ class FacebookProfileMapper extends DataEndpointMapper with FeedItemComparator {
   def dataQueries(fromDate: Option[DateTime], untilDate: Option[DateTime]): Seq[PropertyQuery] = {
     Seq(PropertyQuery(
       List(
-        EndpointQuery("facebook/profile", None, dateFilter(fromDate, untilDate).map(f ⇒ Seq(EndpointQueryFilter("updated_time", None, f))), None)),
-      Some("updated_time"), Some("descending"), None))
+        EndpointQuery("facebook/profile", None, dateFilter(fromDate, untilDate).map(f ⇒ Seq(EndpointQueryFilter("hat_updated_time", None, f))), None)),
+      Some("hat_updated_time"), Some("descending"), None))
   }
 
   def mapDataRecord(recordId: UUID, content: JsValue, tailRecordId: Option[UUID] = None, tailContent: Option[JsValue] = None): Try[DataFeedItem] = {
@@ -640,7 +648,7 @@ class FacebookProfileMapper extends DataEndpointMapper with FeedItemComparator {
             Some(contentText), None, None, None))
         }
       } yield {
-        DataFeedItem("facebook", (tailContent.getOrElse(content) \ "updated_time").as[DateTime], Seq("profile"),
+        DataFeedItem("facebook", (tailContent.getOrElse(content) \ "hat_updated_time").as[DateTime], Seq("profile"),
           Some(title), Some(itemContent), None)
       }
     }
@@ -760,6 +768,54 @@ class FacebookFeedMapper extends DataEndpointMapper {
   }
 }
 
+class FacebookPagesLikesMapper extends DataEndpointMapper {
+  def dataQueries(fromDate: Option[DateTime], untilDate: Option[DateTime]): Seq[PropertyQuery] = {
+    Seq(PropertyQuery(
+      List(
+        EndpointQuery("facebook/likes/pages", None, dateFilter(fromDate, untilDate).map(f ⇒ Seq(EndpointQueryFilter("created_time", None, f))), None)),
+      Some("created_time"), Some("descending"), None))
+  }
+
+  def mapDataRecord(recordId: UUID, content: JsValue, tailRecordId: Option[UUID] = None, tailContent: Option[JsValue] = None): Try[DataFeedItem] = {
+    for {
+      name <- Try((content \ "name").as[String])
+      title <- Try(DataFeedItemTitle(s"You liked $name", None, None))
+
+      itemContent ← Try(DataFeedItemContent(
+        Some(
+          s"""Page Name - ${name}
+             |
+             |Location - ${(content \ "location" \ "city").asOpt[String].getOrElse("")}
+             |Website - ${(content \ "website").asOpt[String].getOrElse("")}""".stripMargin.trim), None, None, None))
+      date ← Try((content \ "created_time").as[DateTime])
+      tags ← Try(Seq("page", name))
+    } yield {
+
+      val locationGeo = Try(LocationGeo(
+        (content \ "location" \ "longitude").as[Double],
+        (content \ "location" \ "latitude").as[Double])).toOption
+
+      val locationAddress = Try(LocationAddress(
+        (content \ "location" \ "country").asOpt[String],
+        (content \ "location" \ "city").asOpt[String],
+        (content \ "name").asOpt[String],
+        (content \ "location" \ "street").asOpt[String],
+        (content \ "location" \ "zip").asOpt[String])).toOption
+
+      val maybeLocation = if (locationAddress.contains(LocationAddress(None, None, None, None, None))) {
+        None
+      }
+      else {
+        locationAddress
+      }
+
+      val location = locationGeo.orElse(maybeLocation).map(_ ⇒ DataFeedItemLocation(locationGeo, maybeLocation, None))
+
+      DataFeedItem("facebook", date, tags, Some(title), Some(itemContent), location)
+    }
+  }
+}
+
 class NotablesFeedMapper extends DataEndpointMapper {
   def dataQueries(fromDate: Option[DateTime], untilDate: Option[DateTime]): Seq[PropertyQuery] = {
     Seq(PropertyQuery(

hat/app/org/hatdex/hat/phata/assets/js/runtime.bundle.js.gz

@@ -54,6 +54,7 @@ class FeedGeneratorService @Inject() ()(
   private val dataMappers: Seq[(String, DataEndpointMapper)] = Seq(
     "facebook/profile" -> new FacebookProfileMapper(),
     "facebook/feed" → new FacebookFeedMapper(),
+    "facebook/likes/pages" → new FacebookPagesLikesMapper(),
     "facebook/events" → new FacebookEventMapper(),
     "twitter/profile" → new TwitterProfileMapper(),
     "twitter/tweets" → new TwitterFeedMapper(),