Getting Sequelize.js library to work on Amazon Lambda

Question

So I'm trying to run a lambda on amazon and narrowed down the error finally by testing the lambda in amazons testing console.

The error I got is this.

{
  "errorMessage": "Please install mysql2 package manually",
  "errorType": "Error",
  "stackTrace": [
    "new MysqlDialect (/var/task/node_modules/sequelize/lib/dialects/mysql/index.js:14:30)",
    "new Sequelize (/var/task/node_modules/sequelize/lib/sequelize.js:234:20)",
    "Object.exports.getSequelizeConnection (/var/task/src/twilio/twilio.js:858:20)",
    "Object.<anonymous> (/var/task/src/twilio/twilio.js:679:25)",
    "__webpack_require__ (/var/task/src/twilio/twilio.js:20:30)",
    "/var/task/src/twilio/twilio.js:63:18",
    "Object.<anonymous> (/var/task/src/twilio/twilio.js:66:10)",
    "Module._compile (module.js:570:32)",
    "Object.Module._extensions..js (module.js:579:10)",
    "Module.load (module.js:487:32)",
    "tryModuleLoad (module.js:446:12)",
    "Function.Module._load (module.js:438:3)",
    "Module.require (module.js:497:17)",
    "require (internal/module.js:20:19)"
  ]
}

Easy enough, so I have to install mysql2. So I added it to my package.json file.

{
  "name": "test-api",
  "version": "1.0.0",
  "description": "",
  "main": "handler.js",
  "scripts": {
    "test": "echo \"Error: no test specified\" && exit 0"
  },
  "keywords": [],
  "author": "",
  "license": "ISC",
  "devDependencies": {
    "aws-sdk": "^2.153.0",
    "babel-core": "^6.26.0",
    "babel-loader": "^7.1.2",
    "babel-plugin-transform-runtime": "^6.23.0",
    "babel-preset-es2015": "^6.24.1",
    "babel-preset-stage-3": "^6.24.1",
    "serverless-domain-manager": "^1.1.20",
    "serverless-dynamodb-autoscaling": "^0.6.2",
    "serverless-webpack": "^4.0.0",
    "webpack": "^3.8.1",
    "webpack-node-externals": "^1.6.0"
  },
  "dependencies": {
    "babel-runtime": "^6.26.0",
    "mailgun-js": "^0.13.1",
    "minimist": "^1.2.0",
    "mysql": "^2.15.0",
    "mysql2": "^1.5.1",
    "qs": "^6.5.1",
    "sequelize": "^4.31.2",
    "serverless": "^1.26.0",
    "serverless-plugin-scripts": "^1.0.2",
    "twilio": "^3.10.0",
    "uuid": "^3.1.0"
  }
}

I noticed when I do sls deploy however, it seems to only be packaging some of the modules?

Serverless: Package lock found - Using locked versions
Serverless: Packing external modules: babel-runtime@^6.26.0, twilio@^3.10.0, qs@^6.5.1, mailgun-js@^0.13.1, sequelize@^4.31.2, minimi
st@^1.2.0, uuid@^3.1.0
Serverless: Packaging service...
Serverless: Uploading CloudFormation file to S3...
Serverless: Uploading artifacts...
Serverless: Validating template...
Serverless: Updating Stack...

Serverless: Checking Stack update progress...
................................
Serverless: Stack update finished...

I think this is why it's not working. In short, how do I get mysql2 library to be packaged correctly with serverless so my lambda function will work with the sequelize library?

Please note that when I test locally my code works fine.

My serverless file is below

service: testapi

# Use serverless-webpack plugin to transpile ES6/ES7
plugins:
  - serverless-webpack
  - serverless-plugin-scripts
  # - serverless-domain-manager

custom:
  #Define the Stage or default to Staging.
  stage: ${opt:stage, self:provider.stage}
  webpackIncludeModules: true
  #Define Databases Here
  databaseName: "${self:service}-${self:custom.stage}"
  #Define Bucket Names Here
  uploadBucket: "${self:service}-uploads-${self:custom.stage}"
  #Custom Script setup
  scripts:
    hooks:
      #Script below will run schema changes to the database as neccesary and update according to stage.
      'deploy:finalize':  node database-schema-update.js --stage ${self:custom.stage}
  #Domain Setup
  # customDomain:
  #    basePath: "/"
  #    domainName: "api-${self:custom.stage}.test.com"
  #    stage: "${self:custom.stage}"
  #    certificateName: "*.test.com"
  #    createRoute53Record: true

provider:
  name: aws
  runtime: nodejs6.10
  stage: staging
  region: us-east-1
  environment:
    DOMAIN_NAME: "api-${self:custom.stage}.test.com"
    DATABASE_NAME: ${self:custom.databaseName}
    DATABASE_USERNAME: ${env:RDS_USERNAME}
    DATABASE_PASSWORD: ${env:RDS_PASSWORD}
    UPLOAD_BUCKET: ${self:custom.uploadBucket}
    TWILIO_ACCOUNT_SID: ""
    TWILIO_AUTH_TOKEN: ""
    USER_POOL_ID: ""
    APP_CLIENT_ID: ""
    REGION: "us-east-1"
    IDENTITY_POOL_ID: ""
    RACKSPACE_API_KEY: ""
  #Below controls permissions for lambda functions.
  iamRoleStatements:
    - Effect: Allow
      Action:
        - dynamodb:DescribeTable
        - dynamodb:UpdateTable
        - dynamodb:Query
        - dynamodb:Scan
        - dynamodb:GetItem
        - dynamodb:PutItem
        - dynamodb:UpdateItem
        - dynamodb:DeleteItem
      Resource: "arn:aws:dynamodb:us-east-1:*:*"

functions:
  create_visit:
    handler: src/visits/create.main
    events:
      - http:
          path: visits
          method: post
          cors: true
          authorizer: aws_iam
  get_visit:
    handler: src/visits/get.main
    events:
      - http:
          path: visits/{id}
          method: get
          cors: true
          authorizer: aws_iam
  list_visit:
    handler: src/visits/list.main
    events:
      - http:
          path: visits
          method: get
          cors: true
          authorizer: aws_iam
  update_visit:
    handler: src/visits/update.main
    events:
      - http:
          path: visits/{id}
          method: put
          cors: true
          authorizer: aws_iam
  delete_visit:
    handler: src/visits/delete.main
    events:
      - http:
          path: visits/{id}
          method: delete
          cors: true
          authorizer: aws_iam
  twilio_send_text_message:
    handler: src/twilio/twilio.send_text_message
    events:
      - http:
          path: twilio/sendtextmessage
          method: post
          cors: true
          authorizer: aws_iam
  #This function handles incoming calls and where to route it to.
  twilio_incoming_call:
    handler: src/twilio/twilio.incoming_calls
    events:
      - http:
          path: twilio/calls
          method: post
  twilio_failure:
    handler: src/twilio/twilio.twilio_failure
    events:
      - http:
          path: twilio/failure
          method: post
  twilio_statuschange:
    handler: src/twilio/twilio.statuschange
    events:
      - http:
          path: twilio/statuschange
          method: post
  twilio_incoming_message:
    handler: src/twilio/twilio.incoming_message
    events:
      - http:
          path: twilio/messages
          method: post
  twilio_whisper:
    handler: src/twilio/twilio.whisper
    events:
      - http:
          path: twilio/whisper
          method: post
      - http:
          path: twilio/whisper
          method: get
  twilio_start_call:
    handler: src/twilio/twilio.start_call
    events:
      - http:
          path: twilio/startcall
          method: post
      - http:
          path: twilio/startcall
          method: get

resources:
  Resources:
    uploadBucket:
       Type: AWS::S3::Bucket
       Properties:
         BucketName: ${self:custom.uploadBucket}
    RDSDatabase:
      Type: AWS::RDS::DBInstance
      Properties:
        Engine : mysql
        MasterUsername: ${env:RDS_USERNAME}
        MasterUserPassword: ${env:RDS_PASSWORD}
        DBInstanceClass : db.t2.micro
        AllocatedStorage: '5'
        PubliclyAccessible: true
        #TODO: The Value of Stage is also available as a TAG automatically which I may use to replace this manually being put here..
        Tags:
          -
            Key: "Name"
            Value: ${self:custom.databaseName}
      DeletionPolicy: Snapshot
    DNSRecordSet:
      Type: AWS::Route53::RecordSet
      Properties:
        HostedZoneName: test.com.
        Name: database-${self:custom.stage}.test.com
        Type: CNAME
        TTL: '300'
        ResourceRecords:
        - {"Fn::GetAtt": ["RDSDatabase","Endpoint.Address"]}
      DependsOn: RDSDatabase

UPDATE:: So I confirmed that running sls package --stage dev seems to create this in the zip folder that would eventually upload to AWS. This confirms that serverless is not creating the package correctly with the mysql2 reference for some reason? Why is this?

webpack config file as requested

const slsw = require("serverless-webpack");
const nodeExternals = require("webpack-node-externals");

module.exports = {
  entry: slsw.lib.entries,
  target: "node",
  // Since 'aws-sdk' is not compatible with webpack,
  // we exclude all node dependencies
  externals: [nodeExternals()],
  // Run babel on all .js files and skip those in node_modules
  module: {
    rules: [
      {
        test: /\.js$/,
        loader: "babel-loader",
        include: __dirname,
        exclude: /node_modules/
      }
    ]
  }
};

You may have to add `mysql2` as an external module. Can you show your `webpack.config.js`? — Noel Llevares, Feb 01 '18 at 07:24
sure one moment, I am using serverless so got to see where they hide that. — Joseph Astrahan, Feb 01 '18 at 07:24

score 11 · Accepted Answer · answered Feb 01 '18 at 08:02

Thanks to dashmugs comment after some investigation on this page (https://github.com/serverless-heaven/serverless-webpack), there is a section on Forced Inclusion. I'll paraphrase it here.

Forced inclusion Sometimes it might happen that you use dynamic requires in your code, i.e. you require modules that are only known at runtime. Webpack is not able to detect such externals and the compiled package will miss the needed dependencies. In such cases you can force the plugin to include certain modules by setting them in the forceInclude array property. However the module must appear in your service's production dependencies in package.json.

# serverless.yml
custom:
  webpackIncludeModules:
    forceInclude:
      - module1
      - module2

So I simply did this...

webpackIncludeModules:
    forceInclude:
      - mysql
      - mysql2

Now it works! Hope this helps someone else with the same issue.

score 2 · Answer 2 · answered Jul 02 '19 at 03:08

None of the previous helped me, I used this solution: https://github.com/sequelize/sequelize/issues/9489#issuecomment-493304014

The trick is to use dialectModule property and override sequelize.

import Sequelize from 'sequelize';
import mysql2 from 'mysql2'; // Needed to fix sequelize issues with WebPack

const sequelize = new Sequelize(
  process.env.DB_NAME,
  process.env.DB_USER,
  process.env.DB_PASSWORD,
  {
    dialect: 'mysql',
    dialectModule: mysql2, // Needed to fix sequelize issues with WebPack
    host: process.env.DB_HOST,
    port: process.env.DB_PORT
  }
)

export async function connectToDatabase() {
  console.log('Trying to connect via sequelize')
  await sequelize.sync()
  await sequelize.authenticate()
  console.log('=> Created a new connection.')

  // Do something 
}

The previous so far works on MySql but is not working with Postgres

Getting Sequelize.js library to work on Amazon Lambda

2 Answers2