Skip to content

Commit

Permalink
Merge pull request Azure#77 from mjlshen/ARO-6161-cosmos
Browse files Browse the repository at this point in the history
Initial CosmosDB Bicep Template for the RP
  • Loading branch information
mjlshen authored Apr 29, 2024
2 parents 7f0d630 + 28b40b2 commit df8dab3
Show file tree
Hide file tree
Showing 8 changed files with 228 additions and 42 deletions.
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ node_modules
/.vscode
/*.crt
/*.key
/*kubeconfig
*kubeconfig
/*.pem
.DS_Store
.idea
Expand Down
30 changes: 10 additions & 20 deletions dev-infrastructure/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,12 @@ HCPDEVSUBSCRIPTION="ARO Hosted Control Planes (EA Subscription 1)"
HCPDEVSUBSCRIPTIONID=1d3378d3-5a3f-4712-85a1-2485495dfc4b

AKSTEMPLATE=templates/aks-development.bicep
VPNTEMPLATE=templates/vpn-development.bicep
VPNCA=$(shell base64 -w0 < secrets/vpn-ca.crt)
CURRENTUSER=$(shell az ad signed-in-user show | jq -r '.id')
DEPLOYMENTNAME=hcp-$(USER)-aks-cluster
PRIVATE_AKS?=true
DEPLOYMENTNAME=hcp-$(USER)
LOCATION?=eastus
RESOURCEGROUP=aro-hcp-${AKSCONFIG}-$(USER)

ifdef AKSCONFIG
include ./configurations/$(AKSCONFIG)
else
ifndef AKSCONFIG
$(error "Must set AKSCONFIG")
endif

Expand All @@ -40,26 +37,19 @@ rg: setsubscription
--location $(LOCATION) \
--tags "CreatedByConfig=${AKSCONFIG}"

aks.linked: setsubscription rg
dev.infrastructure: setsubscription rg
az deployment group create \
--name "$(DEPLOYMENTNAME)-aks" \
--name "$(DEPLOYMENTNAME)-dev-infra" \
--resource-group $(RESOURCEGROUP) \
--template-file $(AKSTEMPLATE) \
--confirm-with-what-if \
--parameters \
kubernetesVersion=$(AKSVERSION) \
vnetAddressPrefix=$(VNETCIDR) \
subnetPrefix=$(NODESUBNETCIDR) \
podSubnetPrefix=$(PODSUBNETCIDR) \
currentUserId=$(CURRENTUSER) \
enablePrivateCluster=$(PRIVATE_AKS) \
createdByConfigTag=${AKSCONFIG}


aks.standalone: setsubscription rg aks.linked
configurations/${AKSCONFIG}.bicepparam \
--parameters \
currentUserId=$(CURRENTUSER)

aks.kubeconfig:
az aks get-credentials -n aro-hcp-cluster-001 -g $(RESOURCEGROUP) --public-fqdn -a -f aks.kubeconfig
az aks get-credentials -n aro-hcp-cluster-001 -g $(RESOURCEGROUP) -a -f aks.kubeconfig

secrets-download: setsubscription
@[ "${SECRET_SA_ACCOUNT_NAME}" ] || ( echo ">> SECRET_SA_ACCOUNT_NAME is not set"; exit 1 )
Expand Down
12 changes: 12 additions & 0 deletions dev-infrastructure/configurations/private.bicepparam
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
using '../templates/aks-development.bicep'

param kubernetesVersion = '1.29.2'
param vnetAddressPrefix = '10.132.0.0/14'
param subnetPrefix = '10.132.8.0/21'
param podSubnetPrefix = '10.132.64.0/18'
param deployFrontendCosmos = true
param enablePrivateCluster = true
param createdByConfigTag = 'private'

// This parameter is always overriden in the Makefile
param currentUserId = ''
12 changes: 12 additions & 0 deletions dev-infrastructure/configurations/public.bicepparam
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
using '../templates/aks-development.bicep'

param kubernetesVersion = '1.29.2'
param vnetAddressPrefix = '10.132.0.0/14'
param subnetPrefix = '10.132.8.0/21'
param podSubnetPrefix = '10.132.64.0/18'
param deployFrontendCosmos = true
param enablePrivateCluster = false
param createdByConfigTag = 'public'

// This parameter is always overriden in the Makefile
param currentUserId = ''
13 changes: 0 additions & 13 deletions dev-infrastructure/configurations/standalone

This file was deleted.

15 changes: 7 additions & 8 deletions dev-infrastructure/docs/development-setup.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,24 +2,23 @@

## Background

The idea of this repo is to provide means to create a development environment that resemble the (future) production setup in a repeatable way. In order to do so, the creation of all infrastructure resources is based on bicep templates that are fed with named configurations through a Makefile.
The idea of this repo is to provide means to create a development environment that resemble the (future) production setup in a repeatable way. In order to do so, the creation of all infrastructure resources is based on bicep templates and parameter files.

## Prerequisites

* `az`, `jq`, `make`
* `az login` with your Red Hat email


## Procedure

1. Provision a Public/Private AKS cluster

```bash
# Private AKS Cluster
AKSCONFIG=standalone make aks.standalone
AKSCONFIG=private make dev.infrastructure

# Public AKS Cluster
AKSCONFIG=standalone PRIVATE_AKS=false make aks.standalone
AKSCONFIG=public make dev.infrastructure
```

1. Access private AKS clusters with:
Expand Down Expand Up @@ -62,7 +61,7 @@ This will create:
1. A service principal/application using the created cert as its authentication, and given access based on the custom role definition

### Step 3 (optional) - log in as the mock application
You may need to manually interact with resources as the service principal, however this shouldn't be required. If you do need to, the 'login' command will download the cert and login with it. Don't forget to logout of the service principal in order to log back in via your personal account.
You may need to manually interact with resources as the service principal, however this shouldn't be required. If you do need to, the 'login' command will download the cert and login with it. Don't forget to logout of the service principal in order to log back in via your personal account.

```bash
cd dev-infrastructure/scripts
Expand All @@ -88,8 +87,8 @@ This will delete:

> Please note that all resource groups not tagged with `persist=true` will be deleted by our cleanup pipeline after 48 hours
1. This will cleanup all resources created in Azure
1. Setting the correct `AKSCONFIG`, this will cleanup all resources created in Azure

```bash
AKSCONFIG=standalone make clean
```
AKSCONFIG=private make clean
```
20 changes: 20 additions & 0 deletions dev-infrastructure/templates/aks-development.bicep
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
@description('The name of the AKS Managed Cluster resource.')
param aksClusterName string = 'aro-hcp-cluster-001'

// TODO: When the work around workload identity for the RP is finalized,
// change this to true
@description('disableLocalAuth for the ARO HCP RP CosmosDB')
param disableLocalAuth bool = false

@description('Optional DNS prefix to use with hosted Kubernetes API server FQDN.')
param dnsPrefix string = aksClusterName

Expand Down Expand Up @@ -285,3 +290,18 @@ resource currentUserAksRbacClusterAdmin 'Microsoft.Authorization/roleAssignments
principalType: 'User'
}
}

@description('Deploy ARO HCP RP Azure Cosmos DB if true')
param deployFrontendCosmos bool = true

module nestedPeeringTemplate './rp-cosmos.bicep' =
if (deployFrontendCosmos) {
name: 'nestedTemplate1'
scope: resourceGroup()
params: {
location: location
aksNodeSubnetId: aksNodeSubnet.id
vnetId: vnet.id
disableLocalAuth: disableLocalAuth
}
}
166 changes: 166 additions & 0 deletions dev-infrastructure/templates/rp-cosmos.bicep
Original file line number Diff line number Diff line change
@@ -0,0 +1,166 @@
param name string = '${resourceGroup().name}-rp-cosmos'
param disableLocalAuth bool = true
param location string
param aksNodeSubnetId string
param vnetId string

var containerNames = [
'Subscriptions'
'AsyncOperations'
'Clusters'
'Billing'
]

resource cosmosDbAccount 'Microsoft.DocumentDB/databaseAccounts@2023-11-15' = {
kind: 'GlobalDocumentDB'
identity: {
type: 'None'
}
name: name
location: location
properties: {
backupPolicy: {
type: 'Continuous'
continuousModeProperties: {
tier: 'Continuous7Days'
}
}
consistencyPolicy: {
defaultConsistencyLevel: 'Session'
maxIntervalInSeconds: 5
maxStalenessPrefix: 100
}
databaseAccountOfferType: 'Standard'
disableLocalAuth: disableLocalAuth
locations: [
{
locationName: location
}
]
publicNetworkAccess: 'Disabled'
enableAutomaticFailover: false
enableMultipleWriteLocations: false
isVirtualNetworkFilterEnabled: false
virtualNetworkRules: []
disableKeyBasedMetadataWriteAccess: false
enableFreeTier: false
enableAnalyticalStorage: false
analyticalStorageConfiguration: {
schemaType: 'WellDefined'
}
createMode: 'Default'
defaultIdentity: 'FirstPartyIdentity'
networkAclBypass: 'None'
enablePartitionMerge: false
enableBurstCapacity: false
minimalTlsVersion: 'Tls12'
}
}

resource cosmosDbPrivateEndpoint 'Microsoft.Network/privateEndpoints@2023-09-01' = {
name: '${name}-private-endpoint'
location: location
properties: {
privateLinkServiceConnections: [
{
name: '${name}-private-endpoint'
properties: {
privateLinkServiceId: cosmosDbAccount.id
groupIds: [
'Sql'
]
}
}
]
subnet: {
id: aksNodeSubnetId
}
}
}

resource cosmosPrivateEndpointDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' = {
// https://github.com/Azure/bicep/issues/12482
// There is no environments().suffixes constant for this
name: 'privatelink.documents.azure.com'
location: 'global'
properties: {}
}

resource cosmosPrivateEndpointDnsZoneLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2020-06-01' = {
parent: cosmosPrivateEndpointDnsZone
name: 'link'
location: 'global'
properties: {
registrationEnabled: false
virtualNetwork: {
id: vnetId
}
}
}

resource cosmosPrivateEndpointDnsGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2023-09-01' = {
parent: cosmosDbPrivateEndpoint
name: '${name}-dns-group'
properties: {
privateDnsZoneConfigs: [
{
name: 'config1'
properties: {
privateDnsZoneId: cosmosPrivateEndpointDnsZone.id
}
}
]
}
}

resource cosmosDb 'Microsoft.DocumentDB/databaseAccounts/sqlDatabases@2023-11-15' = {
name: name
parent: cosmosDbAccount
properties: {
resource: {
id: name
}
options: {}
}
}

resource cosmosDbContainers 'Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers@2023-11-15' = [
for containerName in containerNames: {
parent: cosmosDb
name: containerName
properties: {
resource: {
id: containerName
indexingPolicy: {
indexingMode: 'consistent'
automatic: true
includedPaths: [
{
path: '/*'
}
]
excludedPaths: [
{
path: '/"_etag"/?'
}
]
}
partitionKey: {
paths: [
'/Id'
]
kind: 'Hash'
version: 2
}
uniqueKeyPolicy: {
uniqueKeys: []
}
conflictResolutionPolicy: {
mode: 'LastWriterWins'
conflictResolutionPath: '/_ts'
}
computedProperties: []
}
}
}
]

0 comments on commit df8dab3

Please sign in to comment.