Skip to content

Commit

Permalink
Merge pull request #10 from NYPL-discovery/lambda
Browse files Browse the repository at this point in the history
Lambda
  • Loading branch information
EdwinGuzman authored Mar 17, 2017
2 parents 13cecd2 + 887bf6b commit e56af65
Show file tree
Hide file tree
Showing 10 changed files with 955 additions and 155 deletions.
6 changes: 6 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -34,3 +34,9 @@ node_modules

# Optional REPL history
.node_repl_history

# AWS credentials
.env

# node-lambda build zip file
build
28 changes: 28 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -73,3 +73,31 @@ All search queries support `sort`ing on `title` or `date`. To set a non-default
.. Or by any item @id:

> /resources/b15704876-i25375512
## AWS Services

Currently, the Discovery API is deployed as an AWS Lambda and the endpoints from the Express app are behind an AWS API Gateway called NYPL API - Lambda. Because the API Gateway has a specific endpoint structure, the Discovery API had to conform to that, specifically updating to `/api/v[VERSION_OF_API]/discovery/resources`. The Discovery API can still be run locally, or on a server, through the `node app.js` command.

### Lambda

The Discovery API is a NodeJS Express app and we wanted to convert this into an AWS Lambda. Lambdas are serverless so we used the `aws-serverless-express` npm package to "convert" the server and its endpoints into paths that the Lambda would understand. This was done in order to have all and any NYPL API endpoints in the same location.

#### node-lambda
The node-lambda npm package is used to invoke the lambda locally and to deploy it to AWS. In order to run the Lambda locally, the following files are needed:

* .env - should be updated to include the following credentials:
* AWS_ACCESS_KEY_ID
* AWS_SECRET_ACCESS_KEY
* AWS_ROLE_ARN
* AWS_REGION

AWS_ROLE_ARN is the role for the Lambda. Add this file to .gitignore.
* Index.js - is the wrapper file and handler that the Lambda uses. The `aws-serverless-express` npm package is used to allow the Express server to be access as a Lambda, turning the server application into a serverless application.

To push to AWS run `node-lambda deploy`.

### API Gateway

The Discovery API has endpoints which are currently behind an API Gateway called NYPL API - Lambda. Currently, the endpoints are manually created in the "Resources" section for the NYPL API in the AWS API Gateway admin. For a specific endpoint, say `/api/v0.1/discoverty/resources`, a new "GET" action is created and the Integration Request has to be set to "LAMBDA_PROXY". The API Gateway endpoint request will be automatically picked by the Discovery API Lambda and the request routed to the appropriate endpoint.

When creating the "GET" action and adding the Discovery API Lambda as the LAMBDA_PROXY, make sure to also update the "Method Request" section and add caching to the URL Query String Parameters. For example, `q` is the query search parameter and needs to be added or else `/api/v0.1/discoverty/resources?q=locofocos` will return the same data as ``/api/v0.1/discoverty/resources?q=war`.
85 changes: 41 additions & 44 deletions app.js
Original file line number Diff line number Diff line change
@@ -1,57 +1,54 @@
var cluster = require('cluster')
const config = require('config');
const log = require('loglevel');
const swaggerDocs = require('./swagger.v0.2.json');
const pjson = require('./package.json');

const config = require('config')
const log = require('loglevel')
require('dotenv').config();

log.setLevel(config.get('loglevel'))
var express = require('express');
var elasticsearch = require('elasticsearch');

if (cluster.isMaster) {
// var numCPUs = require('os').cpus().length
var app = express();

for (var i = 0; i < 1; i++) {
cluster.fork()
}
app.thesaurus = config.thesaurus;

cluster.on('exit', function () {
console.log('A worker process died, restarting...')
cluster.fork()
})
} else {
var express = require('express')
var elasticsearch = require('elasticsearch')
var pjson = require('./package.json')
require('./lib/agents')(app);
require('./lib/resources')(app);

var app = express()
// routes
require('./routes/agents')(app);
require('./routes/resources')(app);
require('./routes/misc')(app);

app.thesaurus = config.thesaurus
app.esClient = new elasticsearch.Client({
host: config['elasticsearch'].host
});

require('./lib/agents')(app)
require('./lib/resources')(app)
app.all('*', function (req, res, next) {
res.header('Access-Control-Allow-Origin', '*');
res.header('Access-Control-Allow-Methods', 'GET, OPTIONS');
res.header('Access-Control-Allow-Headers', 'Content-Type');
next();
});

// routes
require('./routes/agents')(app)
require('./routes/resources')(app)
require('./routes/misc')(app)
app.get('/', function (req, res) {
res.send(pjson.version)
});

app.esClient = new elasticsearch.Client({
host: config['elasticsearch'].host
})
// Just testing route
app.get('/api/v0.1/discovery', function (req, res) {
res.send(pjson.version)
});

app.all('*', function (req, res, next) {
res.header('Access-Control-Allow-Origin', '*')
res.header('Access-Control-Allow-Methods', 'GET, OPTIONS')
res.header('Access-Control-Allow-Headers', 'Content-Type')
next()
})
app.get('/api/v0.1/discovery/swagger', function (req, res) {
res.send(swaggerDocs);
});

app.get('/', function (req, res) {
res.send(pjson.version)
})

require('./lib/globals')(app).then((app) => {
app.listen(config['port'], function () {
console.log('Server started on port ' + config['port'])
})
})
}
// Could be removed for the Lambda but necessary for locally running the app.
require('./lib/globals')(app).then((app) => {
app.listen(config['port'], function () {
console.log('Server started on port ' + config['port']);
});
});

module.exports = app;
2 changes: 1 addition & 1 deletion config/default.json
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
"mongoPort": 27017
},

"major_version": 1,
"major_version": 0.1,

"contextAll": "http://api.data.nypl.org/api/v1/context_all.jsonld",

Expand Down
Empty file added deploy.env
Empty file.
7 changes: 7 additions & 0 deletions index.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
'use strict'

const awsServerlessExpress = require('aws-serverless-express');
const app = require('./app');
const server = awsServerlessExpress.createServer(app);

exports.handler = (event, context) => awsServerlessExpress.proxy(server, event, context);
6 changes: 4 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@
"author": "Matt Miller",
"dependencies": {
"async": "^1.5.2",
"aws-serverless-express": "^1.2.0",
"config": "1.12.0",
"dotenv": "^4.0.0",
"elasticsearch": "^10.1.2",
"express": "^4.14.0",
"fast-csv": "^2.3.0",
Expand All @@ -29,9 +31,9 @@
"scripts": {
"test": "standard && mocha test"
},
"description": "",
"description": "Discovery API as an AWS Lambda.",
"license": "MIT",
"name": "registry-api",
"name": "discovery-api",
"preferGlobal": false,
"private": false,
"version": "0.0.6"
Expand Down
8 changes: 4 additions & 4 deletions routes/resources.js
Original file line number Diff line number Diff line change
Expand Up @@ -32,31 +32,31 @@ module.exports = function (app) {
return false
}

app.get(`/api/v${VER}/resources$`, function (req, res) {
app.get(`/api/v${VER}/discovery/resources$`, function (req, res) {
var params = gatherParams(req, standardParams)

return app.resources.search(params)
.then((resp) => respond(res, resp, params))
.catch((error) => handleError(res, error, params))
})

app.get(`/api/v${VER}/resources/aggregations`, function (req, res) {
app.get(`/api/v${VER}/discovery/resources/aggregations`, function (req, res) {
var params = gatherParams(req, standardParams)

return app.resources.aggregations(params)
.then((resp) => respond(res, resp, params))
.catch((error) => handleError(res, error, params))
})

app.get(`/api/v${VER}/resources/aggregation/:field`, function (req, res) {
app.get(`/api/v${VER}/discovery/resources/aggregation/:field`, function (req, res) {
var params = req.params

return app.resources.aggregation(params)
.then((resp) => respond(res, resp, params))
.catch((error) => handleError(res, error, params))
})

app.get(`/api/v${VER}/resources/:uri\.:ext?`, function (req, res) {
app.get(`/api/v${VER}/discovery/resources/:uri\.:ext?`, function (req, res) {
var params = { uri: req.params.uri }

var handler = app.resources.findByUri
Expand Down
Loading

0 comments on commit e56af65

Please sign in to comment.