template.yaml

AWSTemplateFormatVersion: '2010-09-09'
Transform: AWS::Serverless-2016-10-31
Description: Serverless PySpark
Parameters:
  AccessKey:
    Type: String
    Description: AWS Access Key
  SecretKey:
    Type: String
    Description: AWS Secret Key
  LogLevel:
    Type: String
    Description: INFO, DEBUG, ERROR or WARNING
    Default: INFO
Globals:
  Function:
    MemorySize: 512
    Timeout: 900
    Tags:
      AppName: pyspark

Resources:
  SparkRuntime:
    Type: AWS::Serverless::LayerVersion
    Properties:
      LayerName: spark-runtime
      Description: Spark Runtime
      ContentUri: s3://didone-spark/spark.zip
      CompatibleRuntimes:
        - python2.7
      RetentionPolicy: Delete
  SparkDrivers:
    Type: AWS::Serverless::LayerVersion
    Properties:
      LayerName: spark-drivers
      Description: Spark Extra Drivers
      ContentUri: s3://didone-spark/drivers.zip
      CompatibleRuntimes:
        - python2.7
      RetentionPolicy: Delete
  ############# Your functions #############
  HelloFunction:
    Type: AWS::Serverless::Function
    Properties:
      Description: Hello World with spark context
      CodeUri: functions/HelloFunction
      Handler: job.run
      Runtime: python2.7
      Layers:
        - !Ref SparkRuntime
        - !Ref SparkDrivers
      Policies:
        - AWSLambdaBasicExecutionRole
        - AmazonS3FullAccess
      Environment:
        Variables:
          LOG_LEVEL: !Ref LogLevel
          ACCESS_KEY: !Ref AccessKey
          SECRET_KEY: !Ref SecretKey