node.js - How to upload HTTP response as zip from AWS lambda to S3 bucket?

Question

In my lambda function, I have a series of HTTP requests that make external get/post requests to obtain files from Qualtrics. The requested files are returned in the response body as a zip file, which I want to upload to my S3 bucket.

This is the HTTP request code that works from my project, but it does not work in lambda

var data = JSON.stringify({"format":"csv"});

    var config = {
      method: 'get',
      url: qualtricsURL+id+'/export-responses/'+file+'/file',
      headers: { 
        'Content-Type': 'application/json', 
        'Authorization': auth
      },
      responseType: 'stream',
      data : data
    };
    
    axios(config)
    .then(function (response) {
      response.data.pipe(fs.createWriteStream(__dirname+'/public/data.zip')).on('close', async()=> {
        console.log('Response file written');
        await unzipFile();
        resolve(file);
      });
    })
    .catch(function (error) {
      console.log(error);
    })

This method writes the zip file to a local folder I have (/public/data.zip). I instead want to save the contents to an S3 bucket.

I have tried the following code using the https node module instead of axios in my lambda function:

var options = {
        host: url,
        port: 443,
        path: path+id+'/export-responses/' + file + '/file',
        method: 'GET',
        headers: {
                      'Authorization': auth,
                      'Content-Type': 'application/json'
                 }
      
    };
    var messageString = JSON.stringify({"format":"csv"});
     // Setup the HTTP request
    var req = https.request(options, function (res) {

        res.setEncoding('utf-8');
              
        // Collect response data as it comes back.
        var responseString;
        res.on('data', function (data) {
            responseString += data;
            
        });
        res.on('end', async function () {
            //var responseJSON = JSON.parse(responseString);
            console.log('API request sent successfully. ' + responseString);
            
            console.log('Response file written');
                //await unzipFile();
            
            const { writeStream, uploadPromise } = createWriteStream(bucket, key);
            writeStream.write(responseString);
            writeStream.end();
            const uploadResponse = await uploadPromise;
            
           
            resolve();
        
        });

    });

    // Handler for HTTP request errors.
    req.on('error', function (e) {
        console.error('HTTP error: ' + e.message);
        console.log('API request completed with error(s).');
    });
    
    req.write(messageString);
    req.end();


function createWriteStream(Bucket, Key) {
    const writeStream = new stream.PassThrough();
    const uploadPromise = s3
        .upload({
            Bucket,
            Key,
            Body: writeStream,
            CreateBucketConfiguration:
              {
                // Set your region here
                LocationConstraint: "us-east-1"
                  
             }
        })
        .promise();
    return { writeStream, uploadPromise };
}

This uploads the contents to the correct S3 bucket, but the formatting is not right and I am unable to extract the contents. Does anyone know how to get zip contents from a response file through Lambda and upload them to an S3 bucket?

question from:https://stackoverflow.com/questions/66065444/how-to-upload-http-response-as-zip-from-aws-lambda-to-s3-bucket

Answer 1

Waitting for answers