diff --git a/.azure-pipelines/common-templates/install-tools.yml b/.azure-pipelines/common-templates/install-tools.yml index a4c9c40cc8..9e4ff10141 100644 --- a/.azure-pipelines/common-templates/install-tools.yml +++ b/.azure-pipelines/common-templates/install-tools.yml @@ -48,23 +48,70 @@ steps: inputs: workingFile: $(Build.SourcesDirectory)/autorest.powershell/common/config/rush/.npmrc + - task: PowerShell@2 + displayName: Apply npm auth config to user profile + inputs: + targetType: inline + pwsh: true + script: | + # Copy authenticated .npmrc to the user home dir so ALL npm subprocesses + # (including autorest's internal npm calls) use the private feed + auth tokens. + $src = "$(Build.SourcesDirectory)/.npmrc" + $dst = Join-Path $env:USERPROFILE ".npmrc" + Copy-Item -Path $src -Destination $dst -Force + Write-Host "Copied npm config to $dst" + - task: Npm@1 displayName: Install AutoRest inputs: command: custom - customCommand: install -g autorest@3.7.2 - + workingDir: $(Build.SourcesDirectory) + customCommand: install --no-package-lock autorest@3.7.2 + - task: Npm@1 displayName: Install AutorestCore inputs: command: custom - customCommand: install -g @autorest/core@3.10.4 + workingDir: $(Build.SourcesDirectory) + customCommand: install --no-package-lock @autorest/core@3.10.4 + + - task: PowerShell@2 + displayName: Pre-populate autorest extension cache + inputs: + targetType: inline + pwsh: true + script: | + # Autorest resolves extensions from ~/.autorest///node_modules/. + # Pre-install every extension referenced by autorest-configuration.md so autorest + # makes zero npm network calls during module generation. + # Registry and auth are read from ~/.npmrc (copied from the authenticated .npmrc above). + $extensions = @( + "@autorest/core@3.10.4", + "@autorest/modelerfour@4.24.3" + ) + foreach ($ext in $extensions) { + $parts = $ext -split '@(?=[^@]+$)' # split on last @ + $pkg = $parts[0] + $ver = $parts[1] + $cacheDir = Join-Path $env:USERPROFILE ".autorest\$($pkg.Replace('/','\'))\$ver" + $nodeModules = Join-Path $cacheDir "node_modules\$($pkg.Replace('/','\'))" + if (Test-Path $nodeModules) { + Write-Host "Cache already present: $ext" + continue + } + New-Item -ItemType Directory -Force -Path $cacheDir | Out-Null + Write-Host "Pre-installing $ext into $cacheDir" + npm install $ext --prefix $cacheDir + if ($LASTEXITCODE -ne 0) { throw "Failed to pre-install $ext (exit $LASTEXITCODE)" } + Write-Host "Done: $ext" + } - task: Npm@1 displayName: Install Rush inputs: command: custom - customCommand: install -g @microsoft/rush + workingDir: $(Build.SourcesDirectory) + customCommand: install --no-package-lock @microsoft/rush - task: PowerShell@2 displayName: Rush Build @@ -73,6 +120,9 @@ steps: pwsh: true workingDirectory: "autorest.powershell" script: | - rush install - rush link - rush rebuild \ No newline at end of file + npx --no-install rush install + if ($LASTEXITCODE -ne 0) { throw "rush install failed with exit code $LASTEXITCODE" } + npx --no-install rush link + if ($LASTEXITCODE -ne 0) { throw "rush link failed with exit code $LASTEXITCODE" } + npx --no-install rush rebuild + if ($LASTEXITCODE -ne 0) { throw "rush rebuild failed with exit code $LASTEXITCODE" } \ No newline at end of file diff --git a/tools/Configure-PrivateNpmFeed.ps1 b/tools/Configure-PrivateNpmFeed.ps1 index e8d6f75ada..1cfcb9bbde 100644 --- a/tools/Configure-PrivateNpmFeed.ps1 +++ b/tools/Configure-PrivateNpmFeed.ps1 @@ -35,3 +35,24 @@ Write-Host "Created $rootNpmrc" $rushNpmrc = Join-Path $SourcesDirectory "autorest.powershell/common/config/rush/.npmrc" Set-Content -Path $rushNpmrc -Value $npmrcContent -NoNewline Write-Host "Updated $rushNpmrc" + +# Create NuGet.config to redirect dotnet restore to the private feed +# Normalize the registry URL and validate the expected suffix before transforming +$normalizedRegistry = $Registry.TrimEnd('/') +$npmSuffix = "/npm/registry" +if (-not $normalizedRegistry.EndsWith($npmSuffix)) { + throw "Cannot derive NuGet feed URL: Registry '$Registry' does not end with '$npmSuffix'. Pass an explicit Azure Artifacts npm registry URL or add a -NugetFeed parameter." +} +$nugetFeed = $normalizedRegistry.Substring(0, $normalizedRegistry.Length - $npmSuffix.Length) + "/nuget/v3/index.json" +$nugetConfig = @" + + + + + + + +"@ +$nugetConfigPath = Join-Path $SourcesDirectory "NuGet.config" +Set-Content -Path $nugetConfigPath -Value $nugetConfig -NoNewline +Write-Host "Created $nugetConfigPath" diff --git a/tools/GenerateModules.ps1 b/tools/GenerateModules.ps1 index ce7f44b411..8a38ba7b0f 100644 --- a/tools/GenerateModules.ps1 +++ b/tools/GenerateModules.ps1 @@ -32,10 +32,6 @@ if (-not $Isolated) { # Module import. Import-Module PowerShellGet -# Install Powershell-yaml -if (!(Get-Module -Name powershell-yaml -ListAvailable)) { - Install-Module powershell-yaml -Repository PSGallery -Scope CurrentUser -Force -} $ScriptRoot = $PSScriptRoot $ModulesSrc = Join-Path $ScriptRoot "..\src\" @@ -52,8 +48,43 @@ if (-not (Test-Path $ModuleMappingPath)) { # Build AutoREST.PowerShell submodule. Set-Location (Join-Path $ScriptRoot "../autorest.powershell") -rush update --purge -rush build +npx --no-install rush install +npx --no-install rush build + +# Diagnostic: show autorest cache state and npm registry config before generation. +# Gated behind ENABLE_AUTOREST_DIAGNOSTICS to avoid npm calls in network-isolated release pipelines. +if ($env:ENABLE_AUTOREST_DIAGNOSTICS) { + Write-Host "--- Autorest/npm diagnostics ---" + $autorestHome = if ($env:AUTOREST_HOME) { $env:AUTOREST_HOME } else { Join-Path $env:USERPROFILE ".autorest" } + Write-Host "AUTOREST_HOME: $autorestHome" + $coreCacheDir = Join-Path $autorestHome "@autorest\core" + if (Test-Path $coreCacheDir) { + Write-Host "Autorest core cache versions: $(Get-ChildItem $coreCacheDir -Directory | Select-Object -ExpandProperty Name)" + $nodeModulesDir = Join-Path $coreCacheDir "3.10.4\node_modules\@autorest\core" + Write-Host "Cache 3.10.4 node_modules present: $(Test-Path $nodeModulesDir)" + } else { + Write-Host "WARNING: Autorest core cache directory not found at $coreCacheDir" + } + $modelerfourCacheDir = Join-Path $autorestHome "@autorest\modelerfour" + if (Test-Path $modelerfourCacheDir) { + Write-Host "Autorest modelerfour cache versions: $(Get-ChildItem $modelerfourCacheDir -Directory | Select-Object -ExpandProperty Name)" + $modelerfourNodeModules = Join-Path $modelerfourCacheDir "4.24.3\node_modules\@autorest\modelerfour" + Write-Host "Cache modelerfour 4.24.3 node_modules present: $(Test-Path $modelerfourNodeModules)" + } else { + Write-Host "WARNING: Autorest modelerfour cache directory not found at $modelerfourCacheDir" + } + Write-Host "npm registry: $(npm config get registry 2>&1)" + Write-Host "npm_config_registry env: $env:npm_config_registry" + Write-Host "NPM_CONFIG_USERCONFIG: $env:NPM_CONFIG_USERCONFIG" + $userNpmrc = Join-Path $env:USERPROFILE ".npmrc" + Write-Host "~/.npmrc exists: $(Test-Path $userNpmrc)" + if (Test-Path $userNpmrc) { + Write-Host "~/.npmrc registry line: $(Select-String -Path $userNpmrc -Pattern '^registry=' | Select-Object -First 1 -ExpandProperty Line)" + } + Write-Host "--- End diagnostics ---" +} else { + Write-Host "Autorest diagnostics skipped (set ENABLE_AUTOREST_DIAGNOSTICS=1 to enable)" +} $RequiredGraphModules = @() $AuthModuleManifest = Join-Path $ModulesSrc "Authentication" "Authentication" "artifacts" "Microsoft.Graph.Authentication.psd1" diff --git a/tools/GenerateServiceModule.ps1 b/tools/GenerateServiceModule.ps1 index d5385173ac..8eb8481da7 100644 --- a/tools/GenerateServiceModule.ps1 +++ b/tools/GenerateServiceModule.ps1 @@ -69,10 +69,33 @@ $ApiVersion | ForEach-Object { else { $FullModuleVersion = $ModuleMetadata.versions[$CurrentApiVersion].version } - npx autorest --max-memory-size=$MaxMemorySize --module-version:$FullModuleVersion --module-name:$ModuleFullName --service-name:$Module --input-file:$OpenApiFile $AutoRestModuleConfig --max-cpu=2 --network-calls=2 - if ($LastExitCode -ne 0) { - Write-Host -ForegroundColor Red "AutoREST failed to generate '$ModuleFullName' module." - return $LastExitCode + # Pass @autorest/modelerfour as a local --use: argument so autorest loads it + # directly from the pre-populated cache without calling fetchPackageMetadata. + # fetchPackageMetadata is called unconditionally before the cache check inside + # ExtensionManager.findPackage, and it hits the npm registry which is DNS-blocked + # by 1ES supply-chain security on release pipelines. A local --use path bypasses + # findPackage entirely: the extension goes straight into localExtensions, and when + # use-extension in autorest-configuration.md is processed, resolveExtension finds + # it there without any registry call. + $autorestHome = if ($env:AUTOREST_HOME) { $env:AUTOREST_HOME } else { Join-Path $env:USERPROFILE ".autorest" } + $modelerFourPath = Join-Path $autorestHome "@autorest" "modelerfour" "4.24.3" "node_modules" "@autorest" "modelerfour" + # @(if ...) always produces [object[]], preventing PowerShell from unwrapping the + # single-element array to a [string] scalar. A scalar string splatted with @ + # enumerates IEnumerable, passing each character as a separate argument. + $modelerFourUseFlag = @(if (Test-Path $modelerFourPath) { "--use:$modelerFourPath" }) + if ($modelerFourUseFlag.Count -eq 0) { + Write-Host -ForegroundColor Yellow "WARNING: @autorest/modelerfour local cache not found at $modelerFourPath — autorest will attempt npm registry lookup (may fail in network-isolated environment)" + } + + $autorestLog = [System.IO.Path]::Combine([System.IO.Path]::GetTempPath(), "autorest-$($ModuleFullName -replace '[^a-zA-Z0-9-]', '-').log") + npx --no-install autorest @modelerFourUseFlag --verbose --max-memory-size=$MaxMemorySize --module-version:$FullModuleVersion --module-name:$ModuleFullName --service-name:$Module --input-file:$OpenApiFile $AutoRestModuleConfig --max-cpu=2 --network-calls=2 2>&1 | Out-File -FilePath $autorestLog -Encoding utf8 + $autorestExitCode = $LASTEXITCODE + if ($autorestExitCode -ne 0) { + Write-Host -ForegroundColor Red "AutoREST failed (exit $autorestExitCode) generating '$ModuleFullName'." + Write-Host -ForegroundColor Yellow "=== AutoREST log: $autorestLog ===" + if (Test-Path $autorestLog) { Get-Content $autorestLog | ForEach-Object { Write-Host $_ } } + Write-Host -ForegroundColor Yellow "=== End AutoREST log ===" + return $autorestExitCode } Write-Debug "AutoRest generated '$ModuleFullName' successfully." diff --git a/tools/ManageGeneratedModule.ps1 b/tools/ManageGeneratedModule.ps1 index 5d0e9ff48a..9ea2761727 100644 --- a/tools/ManageGeneratedModule.ps1 +++ b/tools/ManageGeneratedModule.ps1 @@ -47,7 +47,7 @@ foreach ($Package in $NugetPackagesToRemove) { # Add nuget packages from generate modules. foreach ($Package in $NugetPackagesToAdd) { Write-Debug "Executing: dotnet add $ModuleCsProj package $Package" - dotnet add $ModuleCsProj package $Package -s https://api.nuget.org/v3/index.json | Out-Null + dotnet add $ModuleCsProj package $Package | Out-Null if ($LastExitCode) { Write-Error "Failed to execute: dotnet add $ModuleCsProj package $Package" } diff --git a/tools/ReadModuleReadMe.ps1 b/tools/ReadModuleReadMe.ps1 index a6f8c5c196..93b07bfa48 100644 --- a/tools/ReadModuleReadMe.ps1 +++ b/tools/ReadModuleReadMe.ps1 @@ -6,12 +6,24 @@ param( [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][string] $FieldToRead ) $ErrorActionPreference = "Stop" + +function ConvertFrom-SimpleYaml { + param([string]$Yaml) + $result = @{} + $Yaml -split "`n" | ForEach-Object { + if ($_.Trim() -match '^([^:]+):\s*(.*)$') { + $result[$Matches[1].Trim()] = $Matches[2].Trim() + } + } + return $result +} + $FieldValue = $null # Read readme.md. $ReadMeContent = Get-Content $ReadMePath -Delimiter "### Versioning" if ($ReadMeContent.Length -eq 2) { # Convert versioning section to yaml. - $VersioningSection = $ReadMeContent[1].Replace("``", "").Replace("yaml", "") | ConvertFrom-Yaml + $VersioningSection = $ReadMeContent[1].Replace("``", "").Replace("yaml", "") | ConvertFrom-SimpleYaml $FieldValue = $VersioningSection[$FieldToRead] } return $FieldValue diff --git a/tools/WriteToModuleReadMe.ps1 b/tools/WriteToModuleReadMe.ps1 index 0a921551ee..5ac0234e47 100644 --- a/tools/WriteToModuleReadMe.ps1 +++ b/tools/WriteToModuleReadMe.ps1 @@ -7,12 +7,24 @@ param( [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][string] $NewFieldValue ) $ErrorActionPreference = "Stop" + +function ConvertFrom-SimpleYaml { + param([string]$Yaml) + $result = @{} + $Yaml -split "`n" | ForEach-Object { + if ($_.Trim() -match '^([^:]+):\s*(.*)$') { + $result[$Matches[1].Trim()] = $Matches[2].Trim() + } + } + return $result +} + # Read readme.md. $ReadMeContent = Get-Content $ReadMePath -Delimiter "### Versioning" if ($ReadMeContent.Length -eq 2) { # Convert versioning section to yaml. $UpdatedVersionSection = "### Versioning" + $ReadMeContent[1] - $VersioningSection = $ReadMeContent[1].Replace("``", "").Replace("yaml", "") | ConvertFrom-Yaml + $VersioningSection = $ReadMeContent[1].Replace("``", "").Replace("yaml", "") | ConvertFrom-SimpleYaml $FieldValue = $VersioningSection[$FieldName] $RegexPattern = "$FieldName`:\s*$FieldValue" $UpdatedVersionSection = $UpdatedVersionSection -replace $RegexPattern, "$FieldName`: $NewFieldValue"