Ramsess/fix release pipeline

.azure-pipelines/common-templates/install-tools.yml

@@ -48,23 +48,70 @@ steps:
     inputs:
       workingFile: $(Build.SourcesDirectory)/autorest.powershell/common/config/rush/.npmrc
 
+  - task: PowerShell@2
+    displayName: Apply npm auth config to user profile
+    inputs:
+      targetType: inline
+      pwsh: true
+      script: |
+        # Copy authenticated .npmrc to the user home dir so ALL npm subprocesses
+        # (including autorest's internal npm calls) use the private feed + auth tokens.
+        $src = "$(Build.SourcesDirectory)/.npmrc"
+        $dst = Join-Path $env:USERPROFILE ".npmrc"
+        Copy-Item -Path $src -Destination $dst -Force
+        Write-Host "Copied npm config to $dst"
+
   - task: Npm@1
     displayName: Install AutoRest
     inputs:
       command: custom
-      customCommand: install -g autorest@3.7.2
-
+      workingDir: $(Build.SourcesDirectory)
+      customCommand: install --no-package-lock autorest@3.7.2
+
   - task: Npm@1
     displayName: Install AutorestCore
     inputs:
       command: custom
-      customCommand: install -g @autorest/core@3.10.4
+      workingDir: $(Build.SourcesDirectory)
+      customCommand: install --no-package-lock @autorest/core@3.10.4
+
+  - task: PowerShell@2
+    displayName: Pre-populate autorest extension cache
+    inputs:
+      targetType: inline
+      pwsh: true
+      script: |
+        # Autorest resolves extensions from ~/.autorest/<pkg>/<version>/node_modules/.
+        # Pre-install every extension referenced by autorest-configuration.md so autorest
+        # makes zero npm network calls during module generation.
+        # Registry and auth are read from ~/.npmrc (copied from the authenticated .npmrc above).
+        $extensions = @(
+            "@autorest/core@3.10.4",
+            "@autorest/modelerfour@4.24.3"
+        )
+        foreach ($ext in $extensions) {
+            $parts   = $ext -split '@(?=[^@]+$)'   # split on last @
+            $pkg     = $parts[0]
+            $ver     = $parts[1]
+            $cacheDir = Join-Path $env:USERPROFILE ".autorest\$($pkg.Replace('/','\'))\$ver"
+            $nodeModules = Join-Path $cacheDir "node_modules\$($pkg.Replace('/','\'))"
+            if (Test-Path $nodeModules) {
+                Write-Host "Cache already present: $ext"
+                continue
+            }
+            New-Item -ItemType Directory -Force -Path $cacheDir | Out-Null
+            Write-Host "Pre-installing $ext into $cacheDir"
+            npm install $ext --prefix $cacheDir
+            if ($LASTEXITCODE -ne 0) { throw "Failed to pre-install $ext (exit $LASTEXITCODE)" }
+            Write-Host "Done: $ext"
+        }
 
   - task: Npm@1
     displayName: Install Rush
     inputs:
       command: custom
-      customCommand: install -g @microsoft/rush
+      workingDir: $(Build.SourcesDirectory)
+      customCommand: install --no-package-lock @microsoft/rush
 
   - task: PowerShell@2
     displayName: Rush Build
@@ -73,6 +120,9 @@ steps:
       pwsh: true
       workingDirectory: "autorest.powershell"
       script: |
-        rush install
-        rush link
-        rush rebuild
+        npx --no-install rush install
+        if ($LASTEXITCODE -ne 0) { throw "rush install failed with exit code $LASTEXITCODE" }
+        npx --no-install rush link
+        if ($LASTEXITCODE -ne 0) { throw "rush link failed with exit code $LASTEXITCODE" }
+        npx --no-install rush rebuild
+        if ($LASTEXITCODE -ne 0) { throw "rush rebuild failed with exit code $LASTEXITCODE" }

tools/Configure-PrivateNpmFeed.ps1

@@ -35,3 +35,24 @@ Write-Host "Created $rootNpmrc"
 $rushNpmrc = Join-Path $SourcesDirectory "autorest.powershell/common/config/rush/.npmrc"
 Set-Content -Path $rushNpmrc -Value $npmrcContent -NoNewline
 Write-Host "Updated $rushNpmrc"
+
+# Create NuGet.config to redirect dotnet restore to the private feed
+# Normalize the registry URL and validate the expected suffix before transforming
+$normalizedRegistry = $Registry.TrimEnd('/')
+$npmSuffix = "/npm/registry"
+if (-not $normalizedRegistry.EndsWith($npmSuffix)) {
+    throw "Cannot derive NuGet feed URL: Registry '$Registry' does not end with '$npmSuffix'. Pass an explicit Azure Artifacts npm registry URL or add a -NugetFeed parameter."
+}
+$nugetFeed = $normalizedRegistry.Substring(0, $normalizedRegistry.Length - $npmSuffix.Length) + "/nuget/v3/index.json"
+$nugetConfig = @"
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <packageSources>
+    <clear />
+    <add key="PowerShell_V2_Build" value="$nugetFeed" />
+  </packageSources>
+</configuration>
+"@
+$nugetConfigPath = Join-Path $SourcesDirectory "NuGet.config"
+Set-Content -Path $nugetConfigPath -Value $nugetConfig -NoNewline
+Write-Host "Created $nugetConfigPath"

tools/GenerateModules.ps1

@@ -32,10 +32,6 @@ if (-not $Isolated) {
 # Module import.
 Import-Module PowerShellGet
 
-# Install Powershell-yaml
-if (!(Get-Module -Name powershell-yaml -ListAvailable)) {
-    Install-Module powershell-yaml -Repository PSGallery -Scope CurrentUser -Force
-}
 
 $ScriptRoot = $PSScriptRoot
 $ModulesSrc = Join-Path $ScriptRoot "..\src\"
@@ -52,8 +48,43 @@ if (-not (Test-Path $ModuleMappingPath)) {
 
 # Build AutoREST.PowerShell submodule.
 Set-Location (Join-Path $ScriptRoot "../autorest.powershell")
-rush update --purge
-rush build
+npx --no-install rush install
+npx --no-install rush build
+
+# Diagnostic: show autorest cache state and npm registry config before generation.
+# Gated behind ENABLE_AUTOREST_DIAGNOSTICS to avoid npm calls in network-isolated release pipelines.
+if ($env:ENABLE_AUTOREST_DIAGNOSTICS) {
+    Write-Host "--- Autorest/npm diagnostics ---"
+    $autorestHome = if ($env:AUTOREST_HOME) { $env:AUTOREST_HOME } else { Join-Path $env:USERPROFILE ".autorest" }
+    Write-Host "AUTOREST_HOME: $autorestHome"
+    $coreCacheDir = Join-Path $autorestHome "@autorest\core"
+    if (Test-Path $coreCacheDir) {
+        Write-Host "Autorest core cache versions: $(Get-ChildItem $coreCacheDir -Directory | Select-Object -ExpandProperty Name)"
+        $nodeModulesDir = Join-Path $coreCacheDir "3.10.4\node_modules\@autorest\core"
+        Write-Host "Cache 3.10.4 node_modules present: $(Test-Path $nodeModulesDir)"
+    } else {
+        Write-Host "WARNING: Autorest core cache directory not found at $coreCacheDir"
+    }
+    $modelerfourCacheDir = Join-Path $autorestHome "@autorest\modelerfour"
+    if (Test-Path $modelerfourCacheDir) {
+        Write-Host "Autorest modelerfour cache versions: $(Get-ChildItem $modelerfourCacheDir -Directory | Select-Object -ExpandProperty Name)"
+        $modelerfourNodeModules = Join-Path $modelerfourCacheDir "4.24.3\node_modules\@autorest\modelerfour"
+        Write-Host "Cache modelerfour 4.24.3 node_modules present: $(Test-Path $modelerfourNodeModules)"
+    } else {
+        Write-Host "WARNING: Autorest modelerfour cache directory not found at $modelerfourCacheDir"
+    }
+    Write-Host "npm registry: $(npm config get registry 2>&1)"
+    Write-Host "npm_config_registry env: $env:npm_config_registry"
+    Write-Host "NPM_CONFIG_USERCONFIG: $env:NPM_CONFIG_USERCONFIG"
+    $userNpmrc = Join-Path $env:USERPROFILE ".npmrc"
+    Write-Host "~/.npmrc exists: $(Test-Path $userNpmrc)"
+    if (Test-Path $userNpmrc) {
+        Write-Host "~/.npmrc registry line: $(Select-String -Path $userNpmrc -Pattern '^registry=' | Select-Object -First 1 -ExpandProperty Line)"
+    }
+    Write-Host "--- End diagnostics ---"
+} else {
+    Write-Host "Autorest diagnostics skipped (set ENABLE_AUTOREST_DIAGNOSTICS=1 to enable)"
+}
 
 $RequiredGraphModules = @()
 $AuthModuleManifest = Join-Path $ModulesSrc "Authentication" "Authentication" "artifacts" "Microsoft.Graph.Authentication.psd1"

tools/GenerateServiceModule.ps1

@@ -69,10 +69,33 @@ $ApiVersion | ForEach-Object {
             else {
                 $FullModuleVersion = $ModuleMetadata.versions[$CurrentApiVersion].version
             }
-            npx autorest --max-memory-size=$MaxMemorySize --module-version:$FullModuleVersion --module-name:$ModuleFullName --service-name:$Module --input-file:$OpenApiFile $AutoRestModuleConfig --max-cpu=2 --network-calls=2
-            if ($LastExitCode -ne 0) {
-                Write-Host -ForegroundColor Red "AutoREST failed to generate '$ModuleFullName' module."
-                return $LastExitCode
+            # Pass @autorest/modelerfour as a local --use:<path> argument so autorest loads it
+            # directly from the pre-populated cache without calling fetchPackageMetadata.
+            # fetchPackageMetadata is called unconditionally before the cache check inside
+            # ExtensionManager.findPackage, and it hits the npm registry which is DNS-blocked
+            # by 1ES supply-chain security on release pipelines. A local --use path bypasses
+            # findPackage entirely: the extension goes straight into localExtensions, and when
+            # use-extension in autorest-configuration.md is processed, resolveExtension finds
+            # it there without any registry call.
+            $autorestHome = if ($env:AUTOREST_HOME) { $env:AUTOREST_HOME } else { Join-Path $env:USERPROFILE ".autorest" }
+            $modelerFourPath = Join-Path $autorestHome "@autorest" "modelerfour" "4.24.3" "node_modules" "@autorest" "modelerfour"
+            # @(if ...) always produces [object[]], preventing PowerShell from unwrapping the
+            # single-element array to a [string] scalar. A scalar string splatted with @
+            # enumerates IEnumerable<char>, passing each character as a separate argument.
+            $modelerFourUseFlag = @(if (Test-Path $modelerFourPath) { "--use:$modelerFourPath" })
+            if ($modelerFourUseFlag.Count -eq 0) {
+                Write-Host -ForegroundColor Yellow "WARNING: @autorest/modelerfour local cache not found at $modelerFourPath — autorest will attempt npm registry lookup (may fail in network-isolated environment)"
+            }
+
+            $autorestLog = [System.IO.Path]::Combine([System.IO.Path]::GetTempPath(), "autorest-$($ModuleFullName -replace '[^a-zA-Z0-9-]', '-').log")
+            npx --no-install autorest @modelerFourUseFlag --verbose --max-memory-size=$MaxMemorySize --module-version:$FullModuleVersion --module-name:$ModuleFullName --service-name:$Module --input-file:$OpenApiFile $AutoRestModuleConfig --max-cpu=2 --network-calls=2 2>&1 | Out-File -FilePath $autorestLog -Encoding utf8
+            $autorestExitCode = $LASTEXITCODE
+            if ($autorestExitCode -ne 0) {
+                Write-Host -ForegroundColor Red "AutoREST failed (exit $autorestExitCode) generating '$ModuleFullName'."
+                Write-Host -ForegroundColor Yellow "=== AutoREST log: $autorestLog ==="
+                if (Test-Path $autorestLog) { Get-Content $autorestLog | ForEach-Object { Write-Host $_ } }
+                Write-Host -ForegroundColor Yellow "=== End AutoREST log ==="
+                return $autorestExitCode
             }
             Write-Debug "AutoRest generated '$ModuleFullName' successfully."
 

tools/ManageGeneratedModule.ps1

@@ -47,7 +47,7 @@ foreach ($Package in $NugetPackagesToRemove) {
 # Add nuget packages from generate modules.
 foreach ($Package in $NugetPackagesToAdd) {
     Write-Debug "Executing: dotnet add $ModuleCsProj package $Package"
-    dotnet add $ModuleCsProj package $Package -s https://api.nuget.org/v3/index.json | Out-Null
+    dotnet add $ModuleCsProj package $Package | Out-Null
     if ($LastExitCode) {
         Write-Error "Failed to execute: dotnet add $ModuleCsProj package $Package"
     }

tools/ReadModuleReadMe.ps1

@@ -6,12 +6,24 @@ param(
     [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][string] $FieldToRead
 ) 
 $ErrorActionPreference = "Stop"
+
+function ConvertFrom-SimpleYaml {
+    param([string]$Yaml)
+    $result = @{}
+    $Yaml -split "`n" | ForEach-Object {
+        if ($_.Trim() -match '^([^:]+):\s*(.*)$') {
+            $result[$Matches[1].Trim()] = $Matches[2].Trim()
+        }
+    }
+    return $result
+}
+
 $FieldValue = $null
 # Read readme.md.
 $ReadMeContent = Get-Content $ReadMePath -Delimiter "### Versioning"
 if ($ReadMeContent.Length -eq 2) {
     # Convert versioning section to yaml.
-    $VersioningSection = $ReadMeContent[1].Replace("``", "").Replace("yaml", "") | ConvertFrom-Yaml
+    $VersioningSection = $ReadMeContent[1].Replace("``", "").Replace("yaml", "") | ConvertFrom-SimpleYaml
     $FieldValue = $VersioningSection[$FieldToRead]
 }
 return $FieldValue

tools/WriteToModuleReadMe.ps1

@@ -7,12 +7,24 @@ param(
     [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][string] $NewFieldValue
 ) 
 $ErrorActionPreference = "Stop"
+
+function ConvertFrom-SimpleYaml {
+    param([string]$Yaml)
+    $result = @{}
+    $Yaml -split "`n" | ForEach-Object {
+        if ($_.Trim() -match '^([^:]+):\s*(.*)$') {
+            $result[$Matches[1].Trim()] = $Matches[2].Trim()
+        }
+    }
+    return $result
+}
+
 # Read readme.md.
 $ReadMeContent = Get-Content $ReadMePath -Delimiter "### Versioning"
 if ($ReadMeContent.Length -eq 2) {
     # Convert versioning section to yaml.
     $UpdatedVersionSection = "### Versioning" + $ReadMeContent[1]
-    $VersioningSection = $ReadMeContent[1].Replace("``", "").Replace("yaml", "") | ConvertFrom-Yaml
+    $VersioningSection = $ReadMeContent[1].Replace("``", "").Replace("yaml", "") | ConvertFrom-SimpleYaml
     $FieldValue = $VersioningSection[$FieldName]
     $RegexPattern = "$FieldName`:\s*$FieldValue"
     $UpdatedVersionSection = $UpdatedVersionSection -replace $RegexPattern, "$FieldName`: $NewFieldValue"