azure-hdi-vnet.ps1
|
<#PSScriptInfo
.VERSION 1.0 .GUID 374a0cff-4d83-44d0-a65d-69f2b65ae1c7 .AUTHOR Andrej Kyselica .COMPANYNAME .COPYRIGHT .TAGS Azure HDInsight .LICENSEURI .PROJECTURI .ICONURI .EXTERNALMODULEDEPENDENCIES .REQUIREDSCRIPTS .EXTERNALSCRIPTDEPENDENCIES .RELEASENOTES #> <# .SYNOPSIS Deploys an HDInsight cluster to Azure into an existing VNet. .DESCRIPTION Deploys an HDInsight cluster to Azure into an existing VNet. The Vnet can be in a separate RG from the new HDI cluster. .NOTES Based on https://docs.microsoft.com/en-us/azure/hdinsight/hdinsight-hadoop-create-linux-clusters-azure-powershell #> $token ="<unique name>" # no dashes or special characters allowed $subscriptionID = "<subscription id>" # Provide your Subscription ID (GUID) $vnetRG = "<resouce group of existing vnet>" $vnetName = "<existing vnet name>" $subnetName = "<target subnet name>" # name of an existing subnet (eg. "default") $resourceGroupName = $token + "rg" # Provide a Resource Group name $clusterName = $token $defaultStorageAccountName = $token + "store" # Provide a Storage account name $defaultStorageContainerName = $token + "container" $location = "East US" # Change the location if needed $clusterNodes = 1 # The number of nodes in the HDInsight cluster # Sign in to Azure Login-AzureRmAccount # Select the subscription to use if you have multiple subscriptions Select-AzureRmSubscription -SubscriptionId $subscriptionID # Get exixting VNet $vnet = Get-AzureRmVirtualNetwork -Name $vnetName -ResourceGroupName $vnetRG $subnet = $vnet.Subnets | Where-Object {$_.Name -eq $subnetName} # Create an Azure Resource Group New-AzureRmResourceGroup -Name $resourceGroupName -Location $location # Create an Azure Storage account and container used as the default storage New-AzureRmStorageAccount ` -ResourceGroupName $resourceGroupName ` -StorageAccountName $defaultStorageAccountName ` -Location $location ` -Type Standard_LRS $defaultStorageAccountKey = (Get-AzureRmStorageAccountKey -Name $defaultStorageAccountName -ResourceGroupName $resourceGroupName)[0].Value $destContext = New-AzureStorageContext -StorageAccountName $defaultStorageAccountName -StorageAccountKey $defaultStorageAccountKey New-AzureStorageContainer -Name $defaultStorageContainerName -Context $destContext # Create an HDInsight cluster $credentials = Get-Credential -Message "Enter Cluster user credentials" -UserName "admin" $sshCredentials = Get-Credential -Message "Enter SSH user credentials" # The location of the HDInsight cluster must be in the same data center as the Storage account. $location = Get-AzureRmStorageAccount -ResourceGroupName $resourceGroupName -StorageAccountName $defaultStorageAccountName | %{$_.Location} New-AzureRmHDInsightCluster ` -ClusterName $clusterName ` -ResourceGroupName $resourceGroupName ` -HttpCredential $credentials ` -Location $location ` -DefaultStorageAccountName "$defaultStorageAccountName.blob.core.windows.net" ` -DefaultStorageAccountKey $defaultStorageAccountKey ` -DefaultStorageContainer $defaultStorageContainerName ` -ClusterSizeInNodes $clusterNodes ` -ClusterType Spark ` -VirtualNetworkId $vnet.Id ` -SubnetName $subnet.Id ` -OSType Linux ` -Version "3.4" ` -SshCredential $sshCredentials |