Azure Function: Writing into ADLS Gen2 with PowerShell Script inside Azure Function App

Question

I am newbie to Azure Functions. So I'm trying to write into ADLS Gen2 Storage Account Containers with the help of writing a PowerShell script inside Azure Function in Azure Function APP with Elastic Premium Plan. Here, I am able to write the Data File to Storage Account but there is "No Data" in that file.

I'm attaching the Code of PowerShell Script written inside the Azure Function along with the Error File after execution.

Experts help me with this. Thanks in Advance.

Code

using namespace System.Net

# Input bindings are passed in via param block.
param($Request, $TriggerMetadata)

# Connect to Azure Account

# $username = "[email protected]"
# $password = Get-Content D:PSpasswordspassword.txt | ConvertTo-SecureString -Key (Get-Content D:PSpasswordsaes.key)
# $credential = New-Object System.Management.Automation.PsCredential($username,$password)
# Connect-AzAccount -Credential $userCredential

   $username = "[email protected]"
   $pass = ConvertTo-SecureString "**********" -AsPlainText -Force
   $cred = New-Object PSCredential($username,$pass)
   Connect-AzAccount -Credential $cred

# Input Variables

 $dataFactoryName="dna-production-gen2"
 $resourceGroupName="DataLake-Gen2"

# get dataFactory triggers

$triggers=Get-AzDataFactoryV2Trigger -DataFactoryName $dataFactoryName  -ResourceGroupName $resourceGroupName
$datas=@()
foreach ($trigger in $triggers) {
    # get the trigger run history
    $today = Get-Date
    $yesterday = $today.AddDays(-1)
     $splat = @{ 
        ResourceGroupName       = $trigger.ResourceGroupName
        DataFactoryName         = $trigger.DataFactoryName
        TriggerName             = $trigger.Name
        TriggerRunStartedAfter  = $yesterday
        TriggerRunStartedBefore = $today
   }
    
   $historys =Get-AzDataFactoryV2TriggerRun @splat
   if($historys -ne $null){
     # create date
     foreach($history in $historys){
        $obj =[PsCustomObject]@{
            'TriggerRunTimestamp '     = $history.TriggerRunTimestamp
            'ResourceGroupName '   =$history.ResourceGroupName
            'DataFactoryName' =$history.DataFactoryName
            'TriggerName '  = $history.TriggerName
            'TriggerRunId'= $history.TriggerRunId
            'TriggerType'=$history.TriggerType
            'Status' =$history.Status

        }
        # add data to an array
        $datas += $obj
     }
   } 
   
  
 }

#  convert data to csv string

 $contents =(($datas | ConvertTo-Csv -NoTypeInformation) -join [Environment]::NewLine)

 # upload to Azure Data Lake Store Gen2

 #1. Create a sas token
 
  $accountName="dna2020gen2"
 
# $path = New-Item -ItemType Directory -Path ".$((Get-Date).ToString('yyyy-MM-dd'))"

 $YY = (Get-Date).year
 $MM = (Get-Date).month
 $DD = get-date –f dd
 
 $fileSystemName="dev"
 $filePath="Input/Triggers/YYYY=$YY/MM=$MM/DD=$DD/data.csv"
 $account = Get-AzStorageAccount -ResourceGroupName 'DataLake-Gen2' -Name $accountName
 $sas= New-AzStorageAccountSASToken -Service Blob  -ResourceType Service,Container,Object `
      -Permission "racwdlup" -StartTime (Get-Date).AddMinutes(-10) `
      -ExpiryTime (Get-Date).AddHours(2) -Context $account.Context
$baseUrl ="https://{0}.dfs.core.windows.net/{1}/{2}{3}" -f $accountName ,  $fileSystemName, $filePath, $sas

#2. Create file
$endpoint =$baseUrl +"&resource=file"

Invoke-RestMethod -Method Put -Uri $endpoint -Headers @{"Content-Length" = 0} -UseBasicParsing

$r = Invoke-WebRequest -Uri https://management.azure.com/subscriptions/{guid}/resourcegroups?api-version=2016-09-01 -Method GET -Headers $authHeaders
$r.Headers["x-ms-ratelimit-remaining-subscription-reads"]

#3 append data
$endpoint =$baseUrl +"&action=append&position=0"
Invoke-RestMethod -Method Patch -Uri $endpoint -Headers @{"Content-Length" = $contents.Length} -Body $contents -UseBasicParsing

#4 flush data
#$endpoint =$baseUrl + ("&action=flush&position={0}" -f $contents.Length)
#Invoke-RestMethod -Method Patch -Uri $endpoint -UseBasicParsing

#Check the result (get data)

Invoke-RestMethod -Method Get -Uri $baseUrl -UseBasicParsing


# Associate values to output bindings by calling 'Push-OutputBinding'.
Push-OutputBinding -Name Response -Value ([HttpResponseContext]@{
    StatusCode = [HttpStatusCode]::OK
    Body =  $contents
})

Error File:

Connected!
2021-01-22T06:56:02.230 [Information] Executing 'Functions.HttpTrigger1' (Reason='This function was programmatically called via the host APIs.', Id=795e78c8-d6d7-4fcc-b4c0-115951b27807)
2021-01-22T06:56:16.039 [Warning] WARNING: TenantId '0c777d2e-f69e-41e4-8dc2-28fcf4c9604b' contains more than one active subscription. First one will be selected for further use. To select another subscription, use Set-AzContext.
2021-01-22T06:56:16.323 [Information] OUTPUT:
2021-01-22T06:56:31.207 [Information] OUTPUT: Account                            SubscriptionName TenantId                             Environment
2021-01-22T06:56:31.216 [Information] OUTPUT: -------                            ---------------- --------                             -----------
2021-01-22T06:56:31.216 [Information] OUTPUT: [email protected] Data Analytics   0c777d2e-f69e-41e4-8dc2-28fcf4c9604b AzureCloud
2021-01-22T06:56:31.216 [Information] OUTPUT:
2021-01-22T06:56:31.775 [Error] ERROR: The cmdlet cannot run because the -ContentType parameter is not a valid Content-Type header. Specify a valid Content-Type for -ContentType, then retry. To suppress header validation, supply the -SkipHeaderValidation parameter.Exception             :Type           : System.Management.Automation.ValidationMetadataExceptionErrorRecord    :Exception             :Type    : System.Management.Automation.ParentContainsErrorRecordExceptionMessage : The cmdlet cannot run because the -ContentType parameter is not a valid Content-Type header. Specify a valid Content-Type for -ContentType, then retry. To suppress header validation, supply the -SkipHeaderValidation parameter.HResult : -2146233087CategoryInfo          : MetadataError: (:) [], ParentContainsErrorRecordExceptionFullyQualifiedErrorId : RuntimeExceptionTargetSite     :Name          : ThrowTerminatingErrorDeclaringType : System.Management.Automation.MshCommandRuntime, System.Management.Automation, Version=7.0.3.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35MemberType    : MethodModule        : System.Management.Automation.dllStackTrace     :at System.Management.Automation.MshCommandRuntime.ThrowTerminatingError(ErrorRecord errorRecord)Message        : The cmdlet cannot run because the -ContentType parameter is not a valid Content-Type header. Specify a valid Content-Type for -ContentType, then retry. To suppress header validation, supply the -SkipHeaderValidation parameter.InnerException :Type       : System.FormatExceptionTargetSite :Name          : ParseAndAddValueDeclaringType : System.Net.Http.Headers.HttpHeadersMemberType    : MethodModule        : System.Net.Http.dllStackTrace :at System.Net.Http.Headers.HttpHeaders.ParseAndAddValue(HeaderDescriptor descriptor, HeaderStoreItemInfo info, String value)at Microsoft.PowerShell.Commands.WebRequestPSCmdlet.FillRequestStream(HttpRequestMessage request)Message    : Cannot add value because header 'Content-Length' does not support multiple values.Source     : System.Net.HttpHResult    : -2146233033Source         : System.Management.AutomationHResult        : -2146233087CategoryInfo          : InvalidArgument: (:) [Invoke-RestMethod], ValidationMetadataExceptionFullyQualifiedErrorId : WebCmdletContentTypeException,Microsoft.PowerShell.Commands.InvokeRestMethodCommandInvocationInfo        :MyCommand        : Invoke-RestMethodScriptLineNumber : 92OffsetInLine     : 1HistoryId        : 1ScriptName       : C:homesitewwwrootHttpTrigger1
un.ps1Line             : Invoke-RestMethod -Method Patch -Uri $endpoint -Headers @{"Content-Length" = $contents.Length} -Body $contents -UseBasicParsingPositionMessage  : At C:homesitewwwrootHttpTrigger1
un.ps1:92 char:1+ Invoke-RestMethod -Method Patch -Uri $endpoint -Headers @{"Content-Le …+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~PSScriptRoot     : C:homesitewwwrootHttpTrigger1PSCommandPath    : C:homesitewwwrootHttpTrigger1
un.ps1InvocationName   : Invoke-RestMethodCommandOrigin    : InternalScriptStackTrace      : at <ScriptBlock>, C:homesitewwwrootHttpTrigger1
un.ps1: line 92Microsoft.Azure.WebJobs.Script.Workers.Rpc.RpcException : Result: ERROR: The cmdlet cannot run because the -ContentType parameter is not a valid Content-Type header. Specify a valid Content-Type for -ContentType, then retry. To suppress header validation, supply the -SkipHeaderValidation parameter.Exception             :Type           : System.Management.Automation.ValidationMetadataExceptionErrorRecord    :Exception             :Type    : System.Management.Automation.ParentContainsErrorRecordExceptionMessage : The cmdlet cannot run because the -ContentType parameter is not a valid Content-Type header. Specify a valid Content-Type for -ContentType, then retry. To suppress header validation, supply the -SkipHeaderValidation parameter.HResult : -2146233087CategoryInfo          : MetadataError: (:) [], ParentContainsErrorRecordExceptionFullyQualifiedErrorId : RuntimeExceptionTargetSite     :Name          : ThrowTerminatingErrorDeclaringType : System.Management.Automation.MshCommandRuntime, System.Management.Automation, Version=7.0.3.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35MemberType    : MethodModule        : System.Management.Automation.dllStackTrace     :at System.Management.Automation.MshCommandRuntime.ThrowTerminatingError(ErrorRecord errorRecord)Message        : The cmdlet cannot run because the -ContentType parameter is not a valid Content-Type header. Specify a valid Content-Type for -ContentType, then retry. To suppress header validation, supply the -SkipHeaderValidation parameter.InnerException :Type       : System.FormatExceptionTargetSite :Name          : ParseAndAddValueDeclaringType : System.Net.Http.Headers.HttpHeadersMemberType    : MethodModule        : System.Net.Http.dllStackTrace :at System.Net.Http.Headers.HttpHeaders.ParseAndAddValue(HeaderDescriptor descriptor, HeaderStoreItemInfo info, String value)at Microsoft.PowerShell.Commands.WebRequestPSCmdlet.FillRequestStream(HttpRequestMessage request)Message    : Cannot add value because header 'Content-Length' does not support multiple values.Source     : System.Net.HttpHResult    : -2146233033Source         : System.Management.AutomationHResult        : -2146233087CategoryInfo          : InvalidArgument: (:) [Invoke-RestMethod], ValidationMetadataExceptionFullyQualifiedErrorId : WebCmdletContentTypeException,Microsoft.PowerShell.Commands.InvokeRestMethodCommandInvocationInfo        :MyCommand        : Invoke-RestMethodScriptLineNumber : 92OffsetInLine     : 1HistoryId        : 1ScriptName       : C:homesitewwwrootHttpTrigger1
un.ps1Line             : Invoke-RestMethod -Method Patch -Uri $endpoint -Headers @{"Content-Length" = $contents.Length} -Body $contents -UseBasicParsingPositionMessage  : At C:homesitewwwrootHttpTrigger1
un.ps1:92 char:1+ Invoke-RestMethod -Method Patch -Uri $endpoint -Headers @{"Content-Le …+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~PSScriptRoot     : C:homesitewwwrootHttpTrigger1PSCommandPath    : C:homesitewwwrootHttpTrigger1
un.ps1InvocationName   : Invoke-RestMethodCommandOrigin    : InternalScriptStackTrace      : at <Sc

Answer 1

Regarding the issue, please refer to the following script

using namespace System.Net

# Input bindings are passed in via param block.
param($Request, $TriggerMetadata)

# Write to the Azure Functions log stream.
Write-Host "PowerShell HTTP trigger function processed a request."
$username = "jimxxxx.onmicrosoft.com"
$pass = ConvertTo-SecureString "xxx" -AsPlainText -Force
$cred = New-Object PSCredential($username,$pass)


Connect-AzAccount -Credential $cred -Tenant "xxx.onmicrosoft.com"

$triggers=Get-AzDataFactoryV2Trigger -ResourceGroupName test001 -DataFactoryName testfactory05

$datas=@()
foreach ($trigger in $triggers) {
    # get the trigger run history
    $today = Get-Date
    $yesterday = $today.AddDays(-1)
     $splat = @{ 
        ResourceGroupName       = $trigger.ResourceGroupName
        DataFactoryName         = $trigger.DataFactoryName
        TriggerName             = $trigger.Name
        TriggerRunStartedAfter  = $yesterday
        TriggerRunStartedBefore = $today
   }
    
   $historys =Get-AzDataFactoryV2TriggerRun @splat
   if($historys -ne $null){
     # create date
     foreach($history in $historys){
        $obj =[PsCustomObject]@{
            'TriggerRunTimestamp '     = $history.TriggerRunTimestamp
            'ResourceGroupName '   =$history.ResourceGroupName
            'DataFactoryName' =$history.DataFactoryName
            'TriggerName '  = $history.TriggerName
            'TriggerRunId'= $history.TriggerRunId
            'TriggerType'=$history.TriggerType
            'Status' =$history.Status

        }
        # add data to an array
        $datas += $obj
     }
   } 
   
  
 }

  $contents =(($datas | ConvertTo-Csv -NoTypeInformation) -join [Environment]::NewLine)

$accountName="testadls05"

 $YY = (Get-Date).year
 $MM = (Get-Date).month
 $DD = get-date –f dd
 
 $fileSystemName="test"
 $filePath="Input/Triggers/YYYY=$YY/MM=$MM/DD=$DD/data.csv"
 $account = Get-AzStorageAccount -ResourceGroupName 'andywin7' -Name $accountName
 $sas= New-AzStorageAccountSASToken -Service Blob  -ResourceType Service,Container,Object `
      -Permission "racwdlup" -StartTime (Get-Date).AddMinutes(-10) `
      -ExpiryTime (Get-Date).AddHours(2) -Context $account.Context
$baseUrl ="https://{0}.dfs.core.windows.net/{1}/{2}{3}" -f $accountName ,  $fileSystemName, $filePath, $sas

#2. Create file
Write-Host "Create file"
$endpoint =$baseUrl +"&resource=file"

Invoke-RestMethod -Method Put -Uri $endpoint -Headers @{"Content-Length" = 0} -UseBasicParsing

$currentAzureContext = Get-AzContext

$azureRmProfile = [Microsoft.Azure.Commands.Common.Authentication.Abstractions.AzureRmProfileProvider]::Instance.Profile;
$profileClient = New-Object Microsoft.Azure.Commands.ResourceManager.Common.RMProfileClient($azureRmProfile);
$token=$profileClient.AcquireAccessToken($currentAzureContext.Subscription.TenantId).AccessToken;
$authHeaders=@{"Authorization" ="Bearer $token"}
$r = Invoke-WebRequest -Uri "https://management.azure.com/subscriptions/$($currentAzureContext.Subscription.Id)/resourcegroups?api-version=2016-09-01" -Method GET -Headers $authHeaders
$r.Headers["x-ms-ratelimit-remaining-subscription-reads"]

#3 append data
Write-Host "append data"
$endpoint =$baseUrl +"&action=append&position=0"
$body= [system.Text.Encoding]::UTF8.GetBytes($contents)
Invoke-RestMethod -Method Patch -Uri $endpoint  -Body $body -UseBasicParsing 
#4 flush data
$endpoint =$baseUrl + ("&action=flush&position={0}" -f $body.Length)
Invoke-RestMethod -Method Patch -Uri $endpoint -UseBasicParsing

#Check the result (get data)

Invoke-RestMethod -Method Get -Uri $baseUrl -UseBasicParsing


# Associate values to output bindings by calling 'Push-OutputBinding'.
Push-OutputBinding -Name Response -Value ([HttpResponseContext]@{
    StatusCode = [HttpStatusCode]::OK
    Body = $contents
})