repo
[
{
"name": "[concat(parameters('factoryName'), '/LS_New')]",
"type": "Microsoft.DataFactory/factories/linkedServices",
"apiVersion": "2018-06-01",
"properties": {
"description": "Connection",
"annotations": [],
"type": "AzureDatabricks",
"typeProperties": {
"domain": "https://url.net",
"accessToken": {
"type": "AzureKeyVaultSecret",
"store": {
"referenceName": "LS_vault",
"type": "LinkedServiceReference"
},
"secretName": "TOKEN"
},
"newClusterNodeType": "Standard_DS4_v2",
"newClusterNumOfWorker": "2:10",
"newClusterSparkEnvVars": {
"PYSPARK_PYTHON": "/databricks/python3/bin/python3"
},
"newClusterVersion": "7.2.x-scala2.12"
}
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/LS_evaKeyVault')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/PIP_Log')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"description": "Unzip",
"activities": [
{
"name": "Parse",
"description": "This notebook",
"type": "DatabricksNotebook",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"notebookPath": "/dataPipelines/main_notebook.py",
"baseParameters": {
"businessgroup": {
"value": "@pipeline().parameters.businessgroup",
"type": "Expression"
},
"project": {
"value": "@pipeline().parameters.project",
"type": "Expression"
}
},
"libraries": [
{
"pypi": {
"package": "cytoolz"
}
},
{
"pypi": {
"package": "log",
"repo": "https://b73gxyht"
}
}
]
},
"linkedServiceName": {
"referenceName": "LS_o",
"type": "LinkedServiceReference"
}
}
],
"parameters": {
"businessgroup": {
"type": "string",
"defaultValue": "test"
},
"project": {
"type": "string",
"defaultValue": "log-analytics"
}
},
"annotations": []
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/LS_o')]"
]
}
]