GC-Build.ps1
1 <# 2 GC-Build.ps1 — AIOPS Google Cloud Foundation Bootstrap (per SME) 3 4 Creates/updates (idempotent where possible): 5 - Project: aiops-gc-<clientcode>-<env> 6 - Service account: aiops-gc-app 7 - Bucket: aiops-gc-<clientcode>-<env>-uploads-<unique> 8 - Enables APIs: aiplatform, discoveryengine, storage, secretmanager 9 - Grants IAM: roles/aiplatform.user, roles/discoveryengine.user (plus optional secret accessor) 10 - Ensures service agents exist (Discovery Engine + Vertex AI) 11 - Hardens bucket: Uniform bucket-level access + Public access prevention 12 - Bucket IAM: app SA objectAdmin; Discovery Engine service agent objectViewer 13 - Smoke test: Upload/delete test file to verify bucket permissions 14 - (Optional) Generates a dev SA key JSON (DO NOT use long-lived keys in production) 15 16 Aligned with: docs/Operational_AI_for_SMEs_GCP_Prereq_Checklist_AIOPS_Naming.md 17 Covers: Phases 1-4 (Project, APIs, IAM, Storage) + Service Account Key + Smoke Test 18 19 ================================================================================ 20 QUICK START INSTRUCTIONS 21 ================================================================================ 22 23 PREREQUISITES: 24 1. Install Google Cloud SDK (if not already installed): 25 - Download from: https://cloud.google.com/sdk/docs/install 26 - Ensure gcloud and gsutil are on your PATH 27 28 2. Authenticate with Google Cloud: 29 PowerShell> gcloud auth login 30 (Opens browser to authenticate. Verify with: gcloud auth list) 31 32 3. Navigate to project directory: 33 PowerShell> cd E:\sme-ops-center 34 35 RUNNING THE SCRIPT: 36 Basic run (defaults: poc/pilot): 37 PowerShell> .\Scripts\GC-Build.ps1 38 39 With custom parameters: 40 PowerShell> .\Scripts\GC-Build.ps1 -ClientCode "acme" -Env "prod" 41 PowerShell> .\Scripts\GC-Build.ps1 -Region "us-central1" 42 PowerShell> .\Scripts\GC-Build.ps1 -SecretsDir "E:\sme-ops-center-secrets" 43 PowerShell> .\Scripts\GC-Build.ps1 -BillingAccountId "01ABCD-EFGHIJ-2KLMNO" 44 PowerShell> .\Scripts\GC-Build.ps1 -GenerateDevKey $false 45 46 WHAT THE SCRIPT DOES: 47 - Preflight checks: Verifies gcloud/gsutil exist and you're authenticated 48 - Creates project: aiops-gc-<clientcode>-<env> (default: aiops-gc-poc-pilot) 49 - Enables APIs: Vertex AI, Discovery Engine, Storage, Secret Manager 50 - Creates service account: aiops-gc-app with required IAM roles 51 - Creates bucket: aiops-gc-<clientcode>-<env>-uploads-<unique> with hardening 52 - Runs smoke test: Verifies bucket permissions work 53 - Generates dev key: Saved to secrets/ directory (project-scoped filename) 54 - Creates state file: secrets/gc-foundation.json with all IDs for handoff pack 55 56 AFTER THE SCRIPT COMPLETES: 57 1. Link billing account (if not automated with -BillingAccountId) 58 2. Create Vertex AI Search resources via console (Steps 13-16 in checklist) 59 3. Run final verification checks (Step 19 in checklist) 60 61 State file location: secrets/gc-foundation.json contains all foundation values. 62 63 TROUBLESHOOTING: 64 - Permission errors: Run PowerShell as Administrator 65 - Preflight fails: Error message shows what's missing (usually need gcloud auth login) 66 - For detailed manual steps: See docs/Operational_AI_for_SMEs_GCP_Prereq_Checklist_AIOPS_Naming.md 67 68 ================================================================================ 69 #> 70 71 [CmdletBinding()] 72 param( 73 [Parameter(Mandatory=$false)] 74 [ValidatePattern('^[a-z][a-z0-9-]{2,11}$')] 75 [string]$ClientCode = "poc", 76 77 [Parameter(Mandatory=$false)] 78 [ValidateSet("pilot","prod")] 79 [string]$Env = "pilot", 80 81 [Parameter(Mandatory=$false)] 82 [string]$Region = "africa-south1", 83 84 # Optional: link billing automatically (partner-owned pilot). For client-owned, leave blank and do Step 2 in console. 85 [Parameter(Mandatory=$false)] 86 [string]$BillingAccountId = "", 87 88 # Where to store the dev key JSON (if generated). Defaults to repo-relative path, can override to absolute path. 89 [Parameter(Mandatory=$false)] 90 [string]$SecretsDir = "$PSScriptRoot\..\secrets", 91 92 # Optional: override bucket name instead of using the standard pattern/persisted value. 93 [Parameter(Mandatory=$false)] 94 [string]$BucketName = "", 95 96 # Generate a dev key JSON by default for local Docker prototypes 97 [Parameter(Mandatory=$false)] 98 [bool]$GenerateDevKey = $true, 99 100 # Add Secret Manager read access for the app SA (recommended if you plan to store OAuth/client secrets there) 101 [Parameter(Mandatory=$false)] 102 [bool]$GrantSecretAccessor = $true 103 ) 104 105 Set-StrictMode -Version Latest 106 $ErrorActionPreference = "Stop" 107 108 function Invoke-Checked { 109 param( 110 [Parameter(Mandatory=$true)][string]$Cmd, 111 [Parameter(Mandatory=$false)][string]$Description = "", 112 [Parameter(Mandatory=$false)][bool]$ContinueOnError = $false 113 ) 114 if ($Description) { Write-Host "`n==> $Description" -ForegroundColor Cyan } 115 Write-Host " $Cmd" -ForegroundColor DarkGray 116 Invoke-Expression $Cmd 117 if ($LASTEXITCODE -ne 0) { 118 if ($ContinueOnError) { 119 Write-Host " WARNING: Command failed (exit $LASTEXITCODE) but continuing: $Cmd" -ForegroundColor Yellow 120 } else { 121 throw "Command failed (exit $LASTEXITCODE): $Cmd" 122 } 123 } 124 } 125 126 function New-RandomSuffix { 127 param([int]$Len = 6) 128 $chars = "abcdefghijklmnopqrstuvwxyz0123456789".ToCharArray() 129 -join (1..$Len | ForEach-Object { $chars | Get-Random }) 130 } 131 132 function Test-Preflight { 133 Write-Host "`n==> Preflight checks" -ForegroundColor Cyan 134 135 $gcloudCmd = Get-Command gcloud -ErrorAction SilentlyContinue 136 if (-not $gcloudCmd) { 137 throw "gcloud CLI not found on PATH. Install gcloud and run 'gcloud auth login' before using this script." 138 } 139 140 $activeAccount = (& gcloud auth list --format="value(account)" --filter="status:ACTIVE" 2>$null) 141 if (-not $activeAccount) { 142 throw "No active gcloud account found. Run 'gcloud auth login' and select an account before using this script." 143 } 144 145 Write-Host " gcloud: $($gcloudCmd.Source)" -ForegroundColor Gray 146 Write-Host " Active account: $activeAccount" -ForegroundColor Gray 147 } 148 149 function Test-BillingEnabled { 150 param([string]$ProjectId) 151 152 $billingEnabled = (& gcloud beta billing projects describe $ProjectId --format="value(billingEnabled)" 2>$null) 153 return ($billingEnabled -eq "True") 154 } 155 156 # Normalize to lowercase IDs as per checklist (GCP resource IDs must be lowercase) 157 $ClientCode = $ClientCode.ToLower() 158 $Env = $Env.ToLower() 159 160 # Run basic environment checks before doing any work 161 Test-Preflight 162 163 # Project and bucket naming aligned with checklist naming standard 164 $ProjectId = ("aiops-gc-{0}-{1}" -f $ClientCode, $Env).ToLower() 165 $ProjectDisplay = ("AIOPS-GC-{0}-{1}" -f $ClientCode.ToUpper(), $Env.ToUpper()) 166 $SaId = "aiops-gc-app" 167 $SaEmail = "$SaId@$ProjectId.iam.gserviceaccount.com" 168 169 # Foundation state file (used to persist chosen bucket, etc.) 170 $KeyPath = $null 171 $StatePath = Join-Path $SecretsDir "gc-foundation.json" 172 173 # Bucket must be globally unique (4-8 chars per checklist) but stable per project on reruns. 174 $Bucket = $null 175 176 # 1) Explicit override wins if provided 177 if ($BucketName -and $BucketName.Trim().Length -gt 0) { 178 $Bucket = $BucketName.ToLower() 179 } 180 # 2) Otherwise, reuse from persisted state if available for this project 181 elseif (Test-Path $StatePath) { 182 try { 183 $existingState = Get-Content $StatePath -Raw | ConvertFrom-Json 184 } catch { 185 $existingState = $null 186 } 187 if ($existingState -and $existingState.ProjectId -eq $ProjectId -and $existingState.Bucket) { 188 $Bucket = $existingState.Bucket 189 } 190 } 191 # 3) Fallback: generate a new suffix and derive bucket name 192 if (-not $Bucket) { 193 $suffix = New-RandomSuffix -Len 6 194 $Bucket = ("aiops-gc-{0}-{1}-uploads-{2}" -f $ClientCode, $Env, $suffix).ToLower() 195 } 196 197 Write-Host "`n=== AIOPS Google Cloud Foundation Bootstrap ===" -ForegroundColor White 198 Write-Host " PROJECT_ID: $ProjectId" 199 Write-Host " PROJECT_NAME: $ProjectDisplay" 200 Write-Host " SA_ID: $SaId" 201 Write-Host " SA_EMAIL: $SaEmail" 202 Write-Host " BUCKET: gs://$Bucket" 203 Write-Host " REGION: $Region" 204 Write-Host "" 205 206 # 1) Create project (or verify exists) — Step 1 207 $existing = (& gcloud projects describe $ProjectId --format="value(projectId)" 2>$null) 208 if (-not $existing) { 209 Invoke-Checked "gcloud projects create $ProjectId --name `"$ProjectDisplay`"" "Step 1 — Create project" 210 } else { 211 Write-Host "`n==> Step 1 — Project already exists: $ProjectId" -ForegroundColor Yellow 212 } 213 214 Invoke-Checked "gcloud config set project $ProjectId" "Set active project" 215 216 # 2) Billing (optional automation) — Step 2 217 if ($BillingAccountId -and $BillingAccountId.Trim().Length -gt 0) { 218 Invoke-Checked "gcloud beta billing projects link $ProjectId --billing-account $BillingAccountId" "Step 2 (optional) — Link billing" 219 } else { 220 Write-Host "`n==> Step 2 — Billing link is MANUAL (recommended client-owned billing)." -ForegroundColor Yellow 221 Write-Host " Verify: gcloud beta billing projects describe $ProjectId --format=`"value(billingEnabled)`"" 222 Write-Host " See checklist Step 2-3 for billing account link + budget configuration." 223 } 224 225 # Check billing status before enabling APIs 226 Write-Host "`n==> Checking billing status..." -ForegroundColor Cyan 227 $billingEnabled = Test-BillingEnabled -ProjectId $ProjectId 228 229 if (-not $billingEnabled) { 230 Write-Host "`n⚠️ WARNING: Billing is NOT enabled for project $ProjectId" -ForegroundColor Yellow 231 Write-Host "" 232 Write-Host "Some APIs require billing to be enabled. You have two options:" -ForegroundColor Yellow 233 Write-Host "" 234 Write-Host "Option 1: Link billing account now (if you have a billing account ID):" -ForegroundColor Cyan 235 Write-Host " .\Scripts\GC-Build.ps1 -BillingAccountId `"YOUR_BILLING_ACCOUNT_ID`"" -ForegroundColor White 236 Write-Host "" 237 Write-Host "Option 2: Link billing manually, then rerun this script:" -ForegroundColor Cyan 238 Write-Host " 1. Go to: https://console.cloud.google.com/billing/linkedaccount?project=$ProjectId" -ForegroundColor White 239 Write-Host " 2. Link your billing account" -ForegroundColor White 240 Write-Host " 3. Rerun this script (it will skip already-created resources)" -ForegroundColor White 241 Write-Host "" 242 Write-Host "Attempting to enable Storage API (may work without billing)..." -ForegroundColor Yellow 243 244 # Try to enable storage API first (usually doesn't require billing for basic operations) 245 Invoke-Checked "gcloud services enable storage.googleapis.com" "Step 4 — Enable Storage API (no billing required)" -ContinueOnError $true 246 247 Write-Host "" 248 Write-Host "❌ Cannot continue: APIs that require billing cannot be enabled:" -ForegroundColor Red 249 Write-Host " - aiplatform.googleapis.com (Vertex AI)" -ForegroundColor Red 250 Write-Host " - discoveryengine.googleapis.com (Vertex AI Search)" -ForegroundColor Red 251 Write-Host " - secretmanager.googleapis.com (Secret Manager)" -ForegroundColor Red 252 Write-Host "" 253 Write-Host "Please link billing and rerun this script. The script is idempotent and will" -ForegroundColor Yellow 254 Write-Host "skip already-created resources (project, etc.) and continue from where it left off." -ForegroundColor Yellow 255 exit 1 256 } else { 257 Write-Host " ✓ Billing is enabled" -ForegroundColor Green 258 } 259 260 # 4) Enable APIs — Step 4 261 Invoke-Checked "gcloud services enable aiplatform.googleapis.com discoveryengine.googleapis.com storage.googleapis.com secretmanager.googleapis.com" "Step 4 — Enable mandatory APIs" 262 263 # 5) Create app service account (idempotent) — Step 5 264 $saExists = (& gcloud iam service-accounts list --format="value(email)" | Select-String -SimpleMatch $SaEmail) 265 if (-not $saExists) { 266 Invoke-Checked "gcloud iam service-accounts create $SaId --display-name `"AIOPS-GC-APP`"" "Step 5 — Create app service account" 267 } else { 268 Write-Host "`n==> Step 5 — Service account already exists: $SaEmail" -ForegroundColor Yellow 269 } 270 271 # 6) Grant project IAM roles — Step 6 272 Invoke-Checked "gcloud projects add-iam-policy-binding $ProjectId --member `"serviceAccount:$SaEmail`" --role roles/aiplatform.user" "Step 6 — Grant roles/aiplatform.user to app SA" 273 Invoke-Checked "gcloud projects add-iam-policy-binding $ProjectId --member `"serviceAccount:$SaEmail`" --role roles/discoveryengine.user" "Step 6 — Grant roles/discoveryengine.user to app SA" 274 275 if ($GrantSecretAccessor) { 276 Invoke-Checked "gcloud projects add-iam-policy-binding $ProjectId --member `"serviceAccount:$SaEmail`" --role roles/secretmanager.secretAccessor" "Step 6 (optional) — Grant roles/secretmanager.secretAccessor to app SA" 277 } 278 279 # 7) Ensure Google-managed service agents exist — Step 7 280 # (Some environments only create these on first use; creating explicitly avoids later confusion.) 281 Invoke-Checked "gcloud beta services identity create --service discoveryengine.googleapis.com --project $ProjectId" "Step 7 — Ensure Discovery Engine service agent exists" -ContinueOnError $true 282 Invoke-Checked "gcloud beta services identity create --service aiplatform.googleapis.com --project $ProjectId" "Step 7 — Ensure Vertex AI service agent exists" -ContinueOnError $true 283 284 # Get project number for service agent identities 285 $ProjectNumber = (& gcloud projects describe $ProjectId --format="value(projectNumber)").Trim() 286 if (-not $ProjectNumber) { throw "Could not resolve project number for $ProjectId" } 287 $DeServiceAgent = "service-$ProjectNumber@gcp-sa-discoveryengine.iam.gserviceaccount.com" 288 289 Write-Host "`nResolved identities:" -ForegroundColor White 290 Write-Host " PROJECT_NUMBER: $ProjectNumber" 291 Write-Host " DE_SERVICE_AGENT: $DeServiceAgent" 292 Write-Host "" 293 294 # 8) Create bucket — Step 8 295 # Use gcloud storage for bucket management. 296 # First check if the specific bucket exists 297 $bucketExists = (& gcloud storage buckets describe "gs://$Bucket" --format="value(name)" 2>$null) 298 if (-not $bucketExists) { 299 # If bucket doesn't exist, check if ANY bucket matching our pattern exists for this project 300 # (in case state file was lost but bucket still exists) 301 $bucketPattern = "aiops-gc-$ClientCode-$Env-uploads-*" 302 $existingBuckets = (& gcloud storage buckets list --project $ProjectId --filter="name:$bucketPattern" --format="value(name)" 2>$null) 303 304 if ($existingBuckets) { 305 # Found existing bucket(s) matching pattern - use the first one 306 $existingBucket = ($existingBuckets -split "`n" | Where-Object { $_.Trim() } | Select-Object -First 1).Trim() 307 Write-Host "`n==> Step 8 — Found existing bucket matching pattern: $existingBucket" -ForegroundColor Yellow 308 Write-Host " Reusing existing bucket instead of creating new one." -ForegroundColor Yellow 309 $Bucket = $existingBucket -replace "^gs://", "" # Remove gs:// prefix if present 310 } else { 311 # No existing bucket found - create new one 312 Invoke-Checked "gcloud storage buckets create gs://$Bucket --project $ProjectId --location $Region" "Step 8 — Create uploads bucket" 313 } 314 } else { 315 Write-Host "`n==> Step 8 — Bucket already exists: gs://$Bucket" -ForegroundColor Yellow 316 } 317 318 # 8B) Bucket hardening (UBLA + PAP) — Step 8 recommended settings 319 Invoke-Checked "gcloud storage buckets update gs://$Bucket --uniform-bucket-level-access --pap" "Step 8B — Harden bucket (UBLA + PAP)" 320 321 # 10) Bucket IAM — Step 10 322 Invoke-Checked -Cmd "gcloud storage buckets add-iam-policy-binding gs://$Bucket --member `"serviceAccount:$SaEmail`" --role roles/storage.objectAdmin" -Description "Step 10A — Bucket IAM for app SA (objectAdmin)" 323 Invoke-Checked -Cmd "gcloud storage buckets add-iam-policy-binding gs://$Bucket --member `"serviceAccount:$DeServiceAgent`" --role roles/storage.objectViewer" -Description "Step 10B — Bucket IAM for Discovery Engine service agent (objectViewer)" 324 325 # 10C) Grant script runner (current gcloud account) objectAdmin so smoke test can upload/delete 326 $CurrentAccount = (& gcloud config get-value account 2>$null).Trim() 327 if ($CurrentAccount) { 328 Invoke-Checked -Cmd "gcloud storage buckets add-iam-policy-binding gs://$Bucket --member `"user:$CurrentAccount`" --role roles/storage.objectAdmin" -Description "Step 10C — Bucket IAM for script runner (smoke test)" 329 } 330 331 # 11) Bucket smoke test — Step 11 332 Write-Host "`n==> Step 11 — Bucket smoke test" -ForegroundColor Cyan 333 $tmp = Join-Path $env:TEMP ("aiops-smoke-{0}.txt" -f ([guid]::NewGuid().ToString("N"))) 334 "hello" | Out-File -FilePath $tmp -Encoding ascii 335 try { 336 Invoke-Checked "gcloud storage cp `"$tmp`" gs://$Bucket/smoke/$(Split-Path $tmp -Leaf)" " Upload test file" 337 Invoke-Checked "gcloud storage rm gs://$Bucket/smoke/$(Split-Path $tmp -Leaf)" " Delete test file" 338 Write-Host " Smoke test PASSED" -ForegroundColor Green 339 } catch { 340 Write-Host " Smoke test FAILED — check bucket permissions" -ForegroundColor Red 341 throw 342 } finally { 343 if (Test-Path $tmp) { Remove-Item $tmp -Force } 344 } 345 346 # 18) Dev key (optional; for local Docker only) — Step 18 347 if ($GenerateDevKey) { 348 New-Item -ItemType Directory -Force -Path $SecretsDir | Out-Null 349 $KeyPath = Join-Path $SecretsDir ("{0}__aiops-gc-app-key.json" -f $ProjectId) 350 Write-Host "`n==> Step 18 — Generate dev key" -ForegroundColor Cyan 351 Write-Host "WARNING: Generating a long-lived service account key (dev only). Do not use this approach in production." -ForegroundColor Yellow 352 Invoke-Checked "gcloud iam service-accounts keys create `"$KeyPath`" --iam-account $SaEmail" " Create key file" 353 Write-Host " Key written to: $KeyPath" -ForegroundColor White 354 } else { 355 Write-Host "`n==> Step 18 — Dev key generation skipped (-GenerateDevKey:`$false)." -ForegroundColor Yellow 356 } 357 358 # Persist foundation state for reruns / handoff pack 359 try { 360 New-Item -ItemType Directory -Force -Path $SecretsDir | Out-Null 361 $state = [ordered]@{ 362 ProjectId = $ProjectId 363 ProjectNumber = $ProjectNumber 364 Region = $Region 365 ClientCode = $ClientCode 366 Env = $Env 367 Bucket = $Bucket 368 SaEmail = $SaEmail 369 DiscoveryEngineServiceAgent = $DeServiceAgent 370 } 371 if ($GenerateDevKey -and $KeyPath) { 372 $state.KeyPath = $KeyPath 373 } 374 $state | ConvertTo-Json | Out-File -FilePath $StatePath -Encoding utf8 375 Write-Host "`nFoundation state written to: $StatePath" -ForegroundColor Gray 376 } catch { 377 Write-Host "`nWARNING: Failed to write foundation state file: $StatePath" -ForegroundColor Yellow 378 } 379 380 # Update .env with GCP parameters (merge into existing .env without overwriting other vars) 381 $EnvPath = Join-Path $PSScriptRoot "..\.env" 382 $EnvExamplePath = Join-Path $PSScriptRoot "..\.env.example" 383 try { 384 $envContent = if (Test-Path $EnvPath) { Get-Content $EnvPath -Raw } else { if (Test-Path $EnvExamplePath) { Get-Content $EnvExamplePath -Raw } else { "" } } 385 $updates = @{ 386 "GOOGLE_CLOUD_PROJECT" = $ProjectId 387 "GCS_BUCKET_NAME" = $Bucket 388 "APP_REGION" = $Region 389 "DISCOVERY_ENGINE_LOCATION" = "global" 390 "STORAGE_BACKEND" = "gcs" 391 } 392 foreach ($key in $updates.Keys) { 393 $val = $updates[$key] 394 $escapedKey = [regex]::Escape($key) 395 $pattern = "(?m)^$escapedKey\s*=.*$" 396 $replacement = "${key}=$val" 397 if ($envContent -match $pattern) { 398 $envContent = $envContent -replace $pattern, $replacement 399 } else { 400 $envContent += "`n$key=$val`n" 401 } 402 } 403 $envContent = $envContent.Trim() 404 Set-Content -Path $EnvPath -Value $envContent -Encoding utf8 -NoNewline:$false 405 Write-Host "Updated .env with GOOGLE_CLOUD_PROJECT, GCS_BUCKET_NAME, APP_REGION, DISCOVERY_ENGINE_LOCATION, STORAGE_BACKEND" -ForegroundColor Gray 406 } catch { 407 Write-Host "`nWARNING: Failed to update .env: $_" -ForegroundColor Yellow 408 } 409 410 # Summary 411 Write-Host "`n=== Setup Complete! ===" -ForegroundColor Green 412 Write-Host "" 413 Write-Host "Created Resources:" -ForegroundColor Cyan 414 Write-Host " Project ID: $ProjectId" 415 Write-Host " Project Name: $ProjectDisplay" 416 Write-Host " Project Number: $ProjectNumber" 417 Write-Host " Bucket: gs://$Bucket" 418 Write-Host " Service Account: $SaEmail" 419 if ($GenerateDevKey) { 420 Write-Host " Key File: $KeyPath" 421 } 422 Write-Host "" 423 Write-Host "Next Manual Steps (per checklist):" -ForegroundColor Yellow 424 Write-Host " [Step 2-3] Link billing account and configure budget/alerts (if not automated)" -ForegroundColor Yellow 425 Write-Host " [Step 13-15] Create Vertex AI Search datastore/app from console and capture DATA_STORE_ID / ENGINE_ID" -ForegroundColor Yellow 426 Write-Host " [Step 19] Run platform verification checks" -ForegroundColor Yellow 427 Write-Host " [Step 20] Test your local /gcs/smoke endpoint once Docker credentials are wired" -ForegroundColor Yellow 428 Write-Host "" 429 Write-Host "For full checklist, see: docs/Operational_AI_for_SMEs_GCP_Prereq_Checklist_AIOPS_Naming.md" -ForegroundColor Gray