can not decompress with gunzip and serverless-http, incorrect header check error, but works for ordinary nodejs express server

Question

I am working with serverless framework and serverless-http library and trying to decompress the binary request. The weird thing that the identical part of code works in ordinary expressjs server but fails to work with serverless-http.

The working minimal representation part of ordinary expressjs server that works fine.

const express = require("express");
const { unzip } = require("node:zlib");
const zlib = require("node:zlib");
const { promisify } = require("node:util");
const compression = require("compression");
const http = require("http");

const app = express();
const port = process.env.PORT || 5002;

app.use(express.json());
app.use(compression());

const do_unzip = promisify(unzip);


app.post("/unzip", async (req, res, next) => {
  const isAccepted = req.headers["content-type"] === "application/gzip";
  if (!isAccepted) return res.status(406).json({ message: "Not Acceptable" });

  const data = [];

  req.addListener("data", (chunk) => {
    data.push(Buffer.from(chunk));
  });

  req.addListener("end", async () => {
    const buff = Buffer.concat(data);

    try {
      const buffRes = await do_unzip(buff);
      const result = buffRes.toString();

      return res.status(200).json({
        result,
      });
    } catch (error) {
      next(error);
    }
  });
});



const server = http.createServer(app).listen(port, () => {
  const addressInfo = server.address();
  console.log(`Listening on port ${addressInfo.port}`);
});

The minimal representation of serverless lambda handler with serverless-http that does not work:

const serverless = require("serverless-http");
const express = require("express");
const { gunzip } = require("node:zlib");
const zlib = require("node:zlib");
const { promisify } = require("node:util");
const compression = require("compression");

const app = express();

app.use(express.json());
app.use(compression());

const do_unzip = promisify(gunzip);

app.post("/unzip", async (req, res, next) => {
  const isAccepted = req.headers["content-type"] === "application/gzip";
  if (!isAccepted) return res.status(406).json({ message: "Not Acceptable" });

  const data = [];
  req.addListener("data", (chunk) => {
    data.push(Buffer.from(chunk));
  });
  req.addListener("end", async () => {
    const buff = Buffer.concat(data);
    try {
      const buffRes = await do_unzip(buff, {
        finishFlush: zlib.constants.Z_SYNC_FLUSH,
      });
      const result = buffRes.toString();

      return res.status(200).json({
        result,
      });
    } catch (error) {
      next(error);
    }
  });
});

app.use((err, req, res, next) => {
  console.error(err);
  return res.status(500).json({
    error: err,
    message: err.message,
  });
});


module.exports.handler = serverless(app);

Sending binary files with postman

The part of code shown above fails with error:

Error: incorrect header check
    at Zlib.zlibOnError [as onerror] (node:zlib:189:17)
    at Zlib.callbackTrampoline (node:internal/async_hooks:130:17) {
  errno: -3,
  code: 'Z_DATA_ERROR'
}

and error response

{
    "error": {
        "errno": -3,
        "code": "Z_DATA_ERROR"
    },
    "message": "incorrect header check"
}

Can not understand what am I doing wrong. Or maybe I should use different approach to decompress binary file with gz extension in the serverless expressjs app ?

Answer 1

This solution can't be treated as exact but it suggests useful workaround.

It worked for this handler handler.js

'use strict';
const zlib = require('zlib');

module.exports.handler = async (input, context) => {
  console.log('input', input);
  const payload = Buffer.from(input.awslogs.data, 'base64');
  zlib.gunzip(payload, function (error, result) {
    if (error) {
      console.error('an error occurred ', error);
      context.fail(error);
    } else {
      result = JSON.parse(result.toString());
      console.log('Event Data:', JSON.stringify(result, null, 2));
      context.succeed();

      return {
        statusCode: 200,
        body: JSON.stringify(
          {
            result,
          },
          null,
          2
        ),
      };
    }
  });
};

I am using serverless invoke command test.sh

#!/bin/sh
exec serverless invoke local -f hello \
-p ./tests/2_input.json

The content of the 2_input.json

{
  "awslogs": {
    "data": "H4sIAAAAAAAAAHWPwQqCQBCGX0Xm7EFtK+smZBEUgXoLCdMhFtKV3akI8d0bLYmibvPPN3wz00CJxmQnTO41whwWQRIctmEcB6sQbFC3CjW3XW8kxpOpP+OC22d1Wml1qZkQGtoMsScxaczKN3plG8zlaHIta5KqWsozoTYw3/djzwhpLwivWFGHGpAFe7DL68JlBUk+l7KSN7tCOEJ4M3/qOI49vMHj+zCKdlFqLaU2ZHV2a4Ct/an0/ivdX8oYc1UVX860fQDQiMdxRQEAAA=="
  }
}

I run the test.sh file and get this output

$./test.sh
Running "serverless" from node_modules
input {
  awslogs: {
    data: 'H4sIAAAAAAAAAHWPwQqCQBCGX0Xm7EFtK+smZBEUgXoLCdMhFtKV3akI8d0bLYmibvPPN3wz00CJxmQnTO41whwWQRIctmEcB6sQbFC3CjW3XW8kxpOpP+OC22d1Wml1qZkQGtoMsScxaczKN3plG8zlaHIta5KqWsozoTYw3/djzwhpLwivWFGHGpAFe7DL68JlBUk+l7KSN7tCOEJ4M3/qOI49vMHj+zCKdlFqLaU2ZHV2a4Ct/an0/ivdX8oYc1UVX860fQDQiMdxRQEAAA=='
  }
}
Event Data: {
  "messageType": "DATA_MESSAGE",
  "owner": "123456789123",
  "logGroup": "testLogGroup",
  "logStream": "testLogStream",
  "subscriptionFilters": [
    "testFilter"
  ],
  "logEvents": [
    {
      "id": "eventId1",
      "timestamp": 1440442987000,
      "message": "[ERROR] First test message"
    },
    {
      "id": "eventId2",
      "timestamp": 1440442987001,
      "message": "[ERROR] Second test message"
    }
  ]
}

I used aws extesion for vscode to get the 2_input.json