From a494b8d63e5d5e014d99d9c3e7c920ef7b97315a Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Tue, 21 Apr 2026 09:38:32 -0700 Subject: [PATCH 1/5] Update cobalt cloud VM IP addresses for new Azure region (#2165) The cobalt cloud machines were moved to a new Azure region. Updated all VNet IP addresses in build/azure.profile.yml: - cobalt-server-lin: 10.2.2.15 -> 10.0.4.17 - cobalt-client-lin: 10.2.2.13 -> 10.0.4.18 - cobalt-db-lin: 10.2.2.14 -> 10.0.4.19 - cobalt-server-lin-azure3: 10.2.2.16 -> 10.0.4.20 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- build/azure.profile.yml | 84 ++++++++++++++++++++--------------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/build/azure.profile.yml b/build/azure.profile.yml index c400e05e5..ee57d9386 100644 --- a/build/azure.profile.yml +++ b/build/azure.profile.yml @@ -304,24 +304,24 @@ profiles: cobalt-cloud-lin: variables: serverPort: 5000 - serverAddress: 10.2.2.15 + serverAddress: 10.0.4.17 cores: 16 jobs: db: endpoints: - - http://10.2.2.14:5001 + - http://10.0.4.19:5001 aliases: - extra application: endpoints: - - http://10.2.2.15:5001 + - http://10.0.4.17:5001 variables: - databaseServer: 10.2.2.14 + databaseServer: 10.0.4.19 aliases: - main load: endpoints: - - http://10.2.2.13:5001 + - http://10.0.4.18:5001 aliases: - warmup - secondary @@ -329,7 +329,7 @@ profiles: cobalt-cloud-lin-relay: variables: serverPort: 5000 - serverAddress: 10.2.2.15 + serverAddress: 10.0.4.17 cores: 16 jobs: db: @@ -341,7 +341,7 @@ profiles: endpoints: - https://aspnetperf.servicebus.windows.net/cobaltcloudlinserver variables: - databaseServer: 10.2.2.14 + databaseServer: 10.0.4.19 aliases: - main load: @@ -354,24 +354,24 @@ profiles: cobalt-cloud-lin-al3: variables: serverPort: 5000 - serverAddress: 10.2.2.16 + serverAddress: 10.0.4.20 cores: 16 jobs: db: endpoints: - - http://10.2.2.14:5001 + - http://10.0.4.19:5001 aliases: - extra application: endpoints: - - http://10.2.2.16:5001 + - http://10.0.4.20:5001 variables: - databaseServer: 10.2.2.14 + databaseServer: 10.0.4.19 aliases: - main load: endpoints: - - http://10.2.2.13:5001 + - http://10.0.4.18:5001 aliases: - warmup - secondary @@ -379,7 +379,7 @@ profiles: cobalt-cloud-lin-al3-relay: variables: serverPort: 5000 - serverAddress: 10.2.2.16 + serverAddress: 10.0.4.20 cores: 16 jobs: db: @@ -391,7 +391,7 @@ profiles: endpoints: - https://aspnetperf.servicebus.windows.net/cobaltcloudlinserver_azurelinux3 variables: - databaseServer: 10.2.2.14 + databaseServer: 10.0.4.19 aliases: - main load: @@ -770,34 +770,34 @@ profiles: cobalt-cloud-lin-client-app: variables: serverPort: 5000 - serverAddress: 10.2.2.13 + serverAddress: 10.0.4.18 cores: 16 agents: application: endpoints: - - http://10.2.2.13:5001 + - http://10.0.4.18:5001 aliases: - main cobalt-cloud-lin-client-db: variables: - databaseServer: 10.2.2.13 - downstreamAddress: 10.2.2.13 + databaseServer: 10.0.4.18 + downstreamAddress: 10.0.4.18 agents: db: endpoints: - - http://10.2.2.13:5001 + - http://10.0.4.18:5001 aliases: - downstream - extra cobalt-cloud-lin-client-load: variables: - secondaryAddress: 10.2.2.13 + secondaryAddress: 10.0.4.18 agents: load: endpoints: - - http://10.2.2.13:5001 + - http://10.0.4.18:5001 aliases: - warmup - secondary @@ -806,34 +806,34 @@ profiles: cobalt-cloud-lin-db-app: variables: serverPort: 5000 - serverAddress: 10.2.2.14 + serverAddress: 10.0.4.19 cores: 16 agents: application: endpoints: - - http://10.2.2.14:5001 + - http://10.0.4.19:5001 aliases: - main cobalt-cloud-lin-db-db: variables: - databaseServer: 10.2.2.14 - downstreamAddress: 10.2.2.14 + databaseServer: 10.0.4.19 + downstreamAddress: 10.0.4.19 agents: db: endpoints: - - http://10.2.2.14:5001 + - http://10.0.4.19:5001 aliases: - downstream - extra cobalt-cloud-lin-db-load: variables: - secondaryAddress: 10.2.2.14 + secondaryAddress: 10.0.4.19 agents: load: endpoints: - - http://10.2.2.14:5001 + - http://10.0.4.19:5001 aliases: - warmup - secondary @@ -842,34 +842,34 @@ profiles: cobalt-cloud-lin-server-app: variables: serverPort: 5000 - serverAddress: 10.2.2.15 + serverAddress: 10.0.4.17 cores: 16 agents: application: endpoints: - - http://10.2.2.15:5001 + - http://10.0.4.17:5001 aliases: - main cobalt-cloud-lin-server-db: variables: - databaseServer: 10.2.2.15 - downstreamAddress: 10.2.2.15 + databaseServer: 10.0.4.17 + downstreamAddress: 10.0.4.17 agents: db: endpoints: - - http://10.2.2.15:5001 + - http://10.0.4.17:5001 aliases: - downstream - extra cobalt-cloud-lin-server-load: variables: - secondaryAddress: 10.2.2.15 + secondaryAddress: 10.0.4.17 agents: load: endpoints: - - http://10.2.2.15:5001 + - http://10.0.4.17:5001 aliases: - warmup - secondary @@ -878,34 +878,34 @@ profiles: cobalt-cloud-lin-server-azure-linux3-app: variables: serverPort: 5000 - serverAddress: 10.2.2.16 + serverAddress: 10.0.4.20 cores: 16 agents: application: endpoints: - - http://10.2.2.16:5001 + - http://10.0.4.20:5001 aliases: - main cobalt-cloud-lin-server-azure-linux3-db: variables: - databaseServer: 10.2.2.16 - downstreamAddress: 10.2.2.16 + databaseServer: 10.0.4.20 + downstreamAddress: 10.0.4.20 agents: db: endpoints: - - http://10.2.2.16:5001 + - http://10.0.4.20:5001 aliases: - downstream - extra cobalt-cloud-lin-server-azure-linux3-load: variables: - secondaryAddress: 10.2.2.16 + secondaryAddress: 10.0.4.20 agents: load: endpoints: - - http://10.2.2.16:5001 + - http://10.0.4.20:5001 aliases: - warmup - secondary From 8be8c1bebb2bb760b633510dffce10495bd80c7c Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Wed, 29 Apr 2026 12:33:20 -0700 Subject: [PATCH 2/5] Move eastus2 cobalt cloud jobs into main azure pipeline (#2166) - Merge cobalt cloud machines into benchmarks_ci_azure.json with machine_group support - Regenerate benchmarks-ci-azure.yml using crank-scheduler (14 groups, handles machine conflicts) - Update benchmarks.template.liquid header with scheduler instructions - Remove separate eastus2 pipeline files (benchmarks-ci-azure-eastus2.yml, benchmarks.matrix.azure.eastus2.yml, benchmarks_ci_azure_eastus2.json) - Remove benchmarks.matrix.azure.yml (replaced by JSON + scheduler approach) - Remove cobaltcloud service bus queue (cobalt jobs now use azure/azurearm64 queues) - Remove EAST US 2 MACHINES header from azure.profile.yml Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- build/azure.profile.yml | 5 - build/benchmarks-ci-azure-eastus2.yml | 223 ------------------ build/benchmarks-ci-azure.yml | 263 ++++++++++++++++++---- build/benchmarks.matrix.azure.eastus2.yml | 80 ------- build/benchmarks.matrix.azure.yml | 87 ------- build/benchmarks.template.liquid | 9 +- build/benchmarks_ci_azure.json | 142 +++++++++++- build/benchmarks_ci_azure_eastus2.json | 179 --------------- 8 files changed, 366 insertions(+), 622 deletions(-) delete mode 100644 build/benchmarks-ci-azure-eastus2.yml delete mode 100644 build/benchmarks.matrix.azure.eastus2.yml delete mode 100644 build/benchmarks.matrix.azure.yml delete mode 100644 build/benchmarks_ci_azure_eastus2.json diff --git a/build/azure.profile.yml b/build/azure.profile.yml index ee57d9386..23282690f 100644 --- a/build/azure.profile.yml +++ b/build/azure.profile.yml @@ -761,11 +761,6 @@ profiles: - warmup - secondary - ############################################# - # # - # EAST US 2 MACHINES # - # # - ############################################# # Individual machine profiles for cobalt-cloud-lin-client cobalt-cloud-lin-client-app: variables: diff --git a/build/benchmarks-ci-azure-eastus2.yml b/build/benchmarks-ci-azure-eastus2.yml deleted file mode 100644 index 60fe8ce24..000000000 --- a/build/benchmarks-ci-azure-eastus2.yml +++ /dev/null @@ -1,223 +0,0 @@ -# Do not change this file, it is generated using these steps: -# - The file benchmarks.matrix.yml defines how each job is run in parallel -# - Convert its content to json using https://jsonformatter.org/yaml-to-json -# - Use the template in benchmarks.template.liquid and the converted json using https://liquidjs.com/playground.html -# - Update this file with the result of the template generation - -trigger: none -pr: none - -schedules: -- cron: "0 2/12 * * *" - always: true # always run the pipeline even if there have not been source code changes since the last successful scheduled run. - branches: - include: - - main - -variables: - - template: job-variables.yml - - name: session - value: $(Build.BuildNumber) - - name: buildId - value: $(Build.BuildId) - - name: buildNumber - value: $(Build.BuildNumber) - - name: am - value: $[lt(format('{0:HH}', pipeline.startTime), 12)] - - name: pm - value: $[ge(format('{0:HH}', pipeline.startTime), 12)] - -jobs: - -# GROUP 1 - -- job: Containers_Cobalt_Cloud_Lin_Server - displayName: 1- Containers Cobalt Cloud Lin Server - pool: server - timeoutInMinutes: 120 - dependsOn: [] - condition: succeededOrFailed() - steps: - - template: containers-scenarios.yml - parameters: - connection: ASPNET Benchmarks Service Bus - serviceBusQueueName: cobaltcloud - serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-app --profile cobalt-cloud-lin-client-load --profile cobalt-cloud-lin-db-db " - -# GROUP 2 - -- job: Containers_Cobalt_Cloud_Lin_Server_Azure_Linux3 - displayName: 2- Containers Cobalt Cloud Lin Server Azure Linux3 - pool: server - timeoutInMinutes: 120 - dependsOn: [Containers_Cobalt_Cloud_Lin_Server] - condition: succeededOrFailed() - steps: - - template: containers-scenarios.yml - parameters: - connection: ASPNET Benchmarks Service Bus - serviceBusQueueName: cobaltcloud - serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-azure-linux3-app --profile cobalt-cloud-lin-client-load --profile cobalt-cloud-lin-db-db " - -# GROUP 3 - -- job: Baselines_Database_Cobalt_Cloud_Lin_Server - displayName: 3- Baselines Database Cobalt Cloud Lin Server - pool: server - timeoutInMinutes: 120 - dependsOn: [Containers_Cobalt_Cloud_Lin_Server_Azure_Linux3] - condition: succeededOrFailed() - steps: - - template: baselines-database-scenarios.yml - parameters: - connection: ASPNET Benchmarks Service Bus - serviceBusQueueName: cobaltcloud - serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-app --profile cobalt-cloud-lin-client-load --profile cobalt-cloud-lin-db-db " - -# GROUP 4 - -- job: Baselines_Database_Cobalt_Cloud_Lin_Server_Azure_Linux3 - displayName: 4- Baselines Database Cobalt Cloud Lin Server Azure Linux3 - pool: server - timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Cobalt_Cloud_Lin_Server] - condition: succeededOrFailed() - steps: - - template: baselines-database-scenarios.yml - parameters: - connection: ASPNET Benchmarks Service Bus - serviceBusQueueName: cobaltcloud - serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-azure-linux3-app --profile cobalt-cloud-lin-client-load --profile cobalt-cloud-lin-db-db " - -# GROUP 5 - -- job: Baselines_Cobalt_Cloud_Lin_Server - displayName: 5- Baselines Cobalt Cloud Lin Server - pool: server - timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Cobalt_Cloud_Lin_Server_Azure_Linux3] - condition: succeededOrFailed() - steps: - - template: baselines-scenarios.yml - parameters: - connection: ASPNET Benchmarks Service Bus - serviceBusQueueName: cobaltcloud - serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-app --profile cobalt-cloud-lin-client-load " - -# GROUP 6 - -- job: Baselines_Cobalt_Cloud_Lin_Server_Azure_Linux3 - displayName: 6- Baselines Cobalt Cloud Lin Server Azure Linux3 - pool: server - timeoutInMinutes: 120 - dependsOn: [Baselines_Cobalt_Cloud_Lin_Server] - condition: succeededOrFailed() - steps: - - template: baselines-scenarios.yml - parameters: - connection: ASPNET Benchmarks Service Bus - serviceBusQueueName: cobaltcloud - serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-azure-linux3-app --profile cobalt-cloud-lin-client-load " - -# GROUP 7 - -- job: MVC_Cobalt_Cloud_Lin_Server - displayName: 7- MVC Cobalt Cloud Lin Server - pool: server - timeoutInMinutes: 120 - dependsOn: [Baselines_Cobalt_Cloud_Lin_Server_Azure_Linux3] - condition: succeededOrFailed() - steps: - - template: mvc-scenarios.yml - parameters: - connection: ASPNET Benchmarks Service Bus - serviceBusQueueName: cobaltcloud - serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-app --profile cobalt-cloud-lin-client-load --profile cobalt-cloud-lin-db-db " - -# GROUP 8 - -- job: MVC_Cobalt_Cloud_Lin_Server_Azure_Linux3 - displayName: 8- MVC Cobalt Cloud Lin Server Azure Linux3 - pool: server - timeoutInMinutes: 120 - dependsOn: [MVC_Cobalt_Cloud_Lin_Server] - condition: succeededOrFailed() - steps: - - template: mvc-scenarios.yml - parameters: - connection: ASPNET Benchmarks Service Bus - serviceBusQueueName: cobaltcloud - serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-azure-linux3-app --profile cobalt-cloud-lin-client-load --profile cobalt-cloud-lin-db-db " - -# GROUP 9 - -- job: Trends_Cobalt_Cloud_Lin_Server - displayName: 9- Trends Cobalt Cloud Lin Server - pool: server - timeoutInMinutes: 120 - dependsOn: [MVC_Cobalt_Cloud_Lin_Server_Azure_Linux3] - condition: succeededOrFailed() - steps: - - template: trend-scenarios.yml - parameters: - connection: ASPNET Benchmarks Service Bus - serviceBusQueueName: cobaltcloud - serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-app --profile cobalt-cloud-lin-client-load " - -# GROUP 10 - -- job: Trends_Cobalt_Cloud_Lin_Server_Azure_Linux3 - displayName: 10- Trends Cobalt Cloud Lin Server Azure Linux3 - pool: server - timeoutInMinutes: 120 - dependsOn: [Trends_Cobalt_Cloud_Lin_Server] - condition: succeededOrFailed() - steps: - - template: trend-scenarios.yml - parameters: - connection: ASPNET Benchmarks Service Bus - serviceBusQueueName: cobaltcloud - serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-azure-linux3-app --profile cobalt-cloud-lin-client-load " - -# GROUP 11 - -- job: Trends_Database_Cobalt_Cloud_Lin_Server - displayName: 11- Trends Database Cobalt Cloud Lin Server - pool: server - timeoutInMinutes: 120 - dependsOn: [Trends_Cobalt_Cloud_Lin_Server_Azure_Linux3] - condition: succeededOrFailed() - steps: - - template: trend-database-scenarios.yml - parameters: - connection: ASPNET Benchmarks Service Bus - serviceBusQueueName: cobaltcloud - serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-app --profile cobalt-cloud-lin-client-load --profile cobalt-cloud-lin-db-db " - -# GROUP 12 - -- job: Trends_Database_Cobalt_Cloud_Lin_Server_Azure_Linux3 - displayName: 12- Trends Database Cobalt Cloud Lin Server Azure Linux3 - pool: server - timeoutInMinutes: 120 - dependsOn: [Trends_Database_Cobalt_Cloud_Lin_Server] - condition: succeededOrFailed() - steps: - - template: trend-database-scenarios.yml - parameters: - connection: ASPNET Benchmarks Service Bus - serviceBusQueueName: cobaltcloud - serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-azure-linux3-app --profile cobalt-cloud-lin-client-load --profile cobalt-cloud-lin-db-db " - diff --git a/build/benchmarks-ci-azure.yml b/build/benchmarks-ci-azure.yml index 76650c098..868e76e43 100644 --- a/build/benchmarks-ci-azure.yml +++ b/build/benchmarks-ci-azure.yml @@ -1,8 +1,9 @@ # Do not change this file, it is generated using these steps: -# - The file benchmarks.matrix.yml defines how each job is run in parallel -# - Convert its content to json using https://jsonformatter.org/yaml-to-json -# - Use the template in benchmarks.template.liquid and the converted json using https://liquidjs.com/playground.html -# - Update this file with the result of the template generation +# - The file benchmarks*.json defines how each pipeline set of jobs is run in parallel +# - Update the associated benchmarks*.json file with machine and scenario updates +# - Install python and install the requirements for the crank-scheduler in benchmarks/scripts/crank-scheduler/requirements.txt +# - Run the scheduler specifying the desired benchmarks*.json file, this template, and benchmarks/output to automatically overwrite the current pipeline. +# - Ex. python ./scripts/crank-scheduler/main.py --config ./build/benchmarks_ci.json --template ./build/benchmarks.template.liquid --yaml-output ./build trigger: none pr: none @@ -61,6 +62,20 @@ jobs: # GROUP 2 +- job: Containers_Cobalt_Cloud_Lin_Server + displayName: 2- Containers Cobalt Cloud Lin Server + pool: server + timeoutInMinutes: 120 + dependsOn: [Containers_Azure_Server_Arm64, Containers_Azure2_Server_Amd64] + condition: succeededOrFailed() + steps: + - template: containers-scenarios.yml + parameters: + connection: ASPNET Benchmarks Service Bus + serviceBusQueueName: azure + serviceBusNamespace: aspnetbenchmarks + arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-app --profile cobalt-cloud-lin-client-load --profile cobalt-cloud-lin-db-db " + - job: Baselines_Database_Azure_Server_Arm64 displayName: 2- Baselines Database Azure Server Arm64 pool: server @@ -71,15 +86,31 @@ jobs: - template: baselines-database-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus - serviceBusQueueName: azure + serviceBusQueueName: azurearm64 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile azure-server-arm64-app --profile azure-client-load --profile azure-db-db " +# GROUP 3 + +- job: Containers_Cobalt_Cloud_Lin_Server_Azure_Linux3 + displayName: 3- Containers Cobalt Cloud Lin Server Azure Linux3 + pool: server + timeoutInMinutes: 120 + dependsOn: [Containers_Cobalt_Cloud_Lin_Server, Baselines_Database_Azure_Server_Arm64] + condition: succeededOrFailed() + steps: + - template: containers-scenarios.yml + parameters: + connection: ASPNET Benchmarks Service Bus + serviceBusQueueName: azure + serviceBusNamespace: aspnetbenchmarks + arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-azure-linux3-app --profile cobalt-cloud-lin-client-load --profile cobalt-cloud-lin-db-db " + - job: Baselines_Database_Azure2_Server_Amd64 - displayName: 2- Baselines Database Azure2 Server Amd64 + displayName: 3- Baselines Database Azure2 Server Amd64 pool: server timeoutInMinutes: 120 - dependsOn: [Containers_Azure_Server_Arm64, Containers_Azure2_Server_Amd64] + dependsOn: [Containers_Cobalt_Cloud_Lin_Server, Baselines_Database_Azure_Server_Arm64] condition: succeededOrFailed() steps: - template: baselines-database-scenarios.yml @@ -89,27 +120,57 @@ jobs: serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile azure2-server-amd64-app --profile azure2-client-load --profile azure2-db-db " -# GROUP 3 +# GROUP 4 +- job: Baselines_Database_Cobalt_Cloud_Lin_Server + displayName: 4- Baselines Database Cobalt Cloud Lin Server + pool: server + timeoutInMinutes: 120 + dependsOn: [Containers_Cobalt_Cloud_Lin_Server_Azure_Linux3, Baselines_Database_Azure2_Server_Amd64] + condition: succeededOrFailed() + steps: + - template: baselines-database-scenarios.yml + parameters: + connection: ASPNET Benchmarks Service Bus + serviceBusQueueName: azure + serviceBusNamespace: aspnetbenchmarks + arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-app --profile cobalt-cloud-lin-client-load --profile cobalt-cloud-lin-db-db " + - job: Baselines_Azure_Server_Arm64 - displayName: 3- Baselines Azure Server Arm64 + displayName: 4- Baselines Azure Server Arm64 pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Azure_Server_Arm64, Baselines_Database_Azure2_Server_Amd64] + dependsOn: [Containers_Cobalt_Cloud_Lin_Server_Azure_Linux3, Baselines_Database_Azure2_Server_Amd64] condition: succeededOrFailed() steps: - template: baselines-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus - serviceBusQueueName: azure + serviceBusQueueName: azurearm64 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile azure-server-arm64-app --profile azure-client-load " +# GROUP 5 + +- job: Baselines_Database_Cobalt_Cloud_Lin_Server_Azure_Linux3 + displayName: 5- Baselines Database Cobalt Cloud Lin Server Azure Linux3 + pool: server + timeoutInMinutes: 120 + dependsOn: [Baselines_Database_Cobalt_Cloud_Lin_Server, Baselines_Azure_Server_Arm64] + condition: succeededOrFailed() + steps: + - template: baselines-database-scenarios.yml + parameters: + connection: ASPNET Benchmarks Service Bus + serviceBusQueueName: azure + serviceBusNamespace: aspnetbenchmarks + arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-azure-linux3-app --profile cobalt-cloud-lin-client-load --profile cobalt-cloud-lin-db-db " + - job: Baselines_Azure2_Server_Amd64 - displayName: 3- Baselines Azure2 Server Amd64 + displayName: 5- Baselines Azure2 Server Amd64 pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Azure_Server_Arm64, Baselines_Database_Azure2_Server_Amd64] + dependsOn: [Baselines_Database_Cobalt_Cloud_Lin_Server, Baselines_Azure_Server_Arm64] condition: succeededOrFailed() steps: - template: baselines-scenarios.yml @@ -119,27 +180,87 @@ jobs: serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile azure2-server-amd64-app --profile azure2-client-load " -# GROUP 4 +# GROUP 6 +- job: Baselines_Cobalt_Cloud_Lin_Server + displayName: 6- Baselines Cobalt Cloud Lin Server + pool: server + timeoutInMinutes: 120 + dependsOn: [Baselines_Database_Cobalt_Cloud_Lin_Server_Azure_Linux3, Baselines_Azure2_Server_Amd64] + condition: succeededOrFailed() + steps: + - template: baselines-scenarios.yml + parameters: + connection: ASPNET Benchmarks Service Bus + serviceBusQueueName: azure + serviceBusNamespace: aspnetbenchmarks + arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-app --profile cobalt-cloud-lin-client-load " + +- job: Baselines_Cobalt_Cloud_Lin_Server_Azure_Linux3 + displayName: 6- Baselines Cobalt Cloud Lin Server Azure Linux3 + pool: server + timeoutInMinutes: 120 + dependsOn: [Baselines_Database_Cobalt_Cloud_Lin_Server_Azure_Linux3, Baselines_Azure2_Server_Amd64] + condition: succeededOrFailed() + steps: + - template: baselines-scenarios.yml + parameters: + connection: ASPNET Benchmarks Service Bus + serviceBusQueueName: azurearm64 + serviceBusNamespace: aspnetbenchmarks + arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-azure-linux3-app --profile cobalt-cloud-lin-db-load " + +# GROUP 7 + +- job: MVC_Cobalt_Cloud_Lin_Server + displayName: 7- MVC Cobalt Cloud Lin Server + pool: server + timeoutInMinutes: 120 + dependsOn: [Baselines_Cobalt_Cloud_Lin_Server, Baselines_Cobalt_Cloud_Lin_Server_Azure_Linux3] + condition: succeededOrFailed() + steps: + - template: mvc-scenarios.yml + parameters: + connection: ASPNET Benchmarks Service Bus + serviceBusQueueName: azure + serviceBusNamespace: aspnetbenchmarks + arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-app --profile cobalt-cloud-lin-client-load --profile cobalt-cloud-lin-db-db " + - job: Trends_Azure_Server_Arm64 - displayName: 4- Trends Azure Server Arm64 + displayName: 7- Trends Azure Server Arm64 pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Azure_Server_Arm64, Baselines_Azure2_Server_Amd64] + dependsOn: [Baselines_Cobalt_Cloud_Lin_Server, Baselines_Cobalt_Cloud_Lin_Server_Azure_Linux3] condition: succeededOrFailed() steps: - template: trend-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus - serviceBusQueueName: azure + serviceBusQueueName: azurearm64 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile azure-server-arm64-app --profile azure-client-load " +# GROUP 8 + +- job: MVC_Cobalt_Cloud_Lin_Server_Azure_Linux3 + displayName: 8- MVC Cobalt Cloud Lin Server Azure Linux3 + pool: server + timeoutInMinutes: 120 + dependsOn: [MVC_Cobalt_Cloud_Lin_Server, Trends_Azure_Server_Arm64] + condition: succeededOrFailed() + steps: + - template: mvc-scenarios.yml + parameters: + connection: ASPNET Benchmarks Service Bus + serviceBusQueueName: azure + serviceBusNamespace: aspnetbenchmarks + arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-azure-linux3-app --profile cobalt-cloud-lin-client-load --profile cobalt-cloud-lin-db-db " + - job: Trends_Azure2_Server_Amd64 - displayName: 4- Trends Azure2 Server Amd64 + displayName: 8- Trends Azure2 Server Amd64 pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Azure_Server_Arm64, Baselines_Azure2_Server_Amd64] + dependsOn: [MVC_Cobalt_Cloud_Lin_Server, Trends_Azure_Server_Arm64] condition: succeededOrFailed() steps: - template: trend-scenarios.yml @@ -149,13 +270,13 @@ jobs: serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile azure2-server-amd64-app --profile azure2-client-load " -# GROUP 5 +# GROUP 9 -- job: Trends_Idna_Amd_Lin - displayName: 5- Trends Idna Amd Lin +- job: Trends_Cobalt_Cloud_Lin_Server + displayName: 9- Trends Cobalt Cloud Lin Server pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Azure_Server_Arm64, Trends_Azure2_Server_Amd64] + dependsOn: [MVC_Cobalt_Cloud_Lin_Server_Azure_Linux3, Trends_Azure2_Server_Amd64] condition: succeededOrFailed() steps: - template: trend-scenarios.yml @@ -163,13 +284,13 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: azure serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile idna-amd-lin-app --profile idna-intel-lin-load " + arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-app --profile cobalt-cloud-lin-client-load " -- job: Trends_Idna_Amd_Win - displayName: 5- Trends Idna Amd Win +- job: Trends_Cobalt_Cloud_Lin_Server_Azure_Linux3 + displayName: 9- Trends Cobalt Cloud Lin Server Azure Linux3 pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Azure_Server_Arm64, Trends_Azure2_Server_Amd64] + dependsOn: [MVC_Cobalt_Cloud_Lin_Server_Azure_Linux3, Trends_Azure2_Server_Amd64] condition: succeededOrFailed() steps: - template: trend-scenarios.yml @@ -177,15 +298,45 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: azurearm64 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile idna-amd-win-app --profile azure2-db-load " + arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-azure-linux3-app --profile cobalt-cloud-lin-db-load " -# GROUP 6 +# GROUP 10 +- job: Trends_Idna_Amd_Lin + displayName: 10- Trends Idna Amd Lin + pool: server + timeoutInMinutes: 120 + dependsOn: [Trends_Cobalt_Cloud_Lin_Server, Trends_Cobalt_Cloud_Lin_Server_Azure_Linux3] + condition: succeededOrFailed() + steps: + - template: trend-scenarios.yml + parameters: + connection: ASPNET Benchmarks Service Bus + serviceBusQueueName: azure + serviceBusNamespace: aspnetbenchmarks + arguments: "$(ciProfile) --profile idna-amd-lin-app --profile idna-amd-win-load " + - job: Trends_Idna_Intel_Lin - displayName: 6- Trends Idna Intel Lin + displayName: 10- Trends Idna Intel Lin + pool: server + timeoutInMinutes: 120 + dependsOn: [Trends_Cobalt_Cloud_Lin_Server, Trends_Cobalt_Cloud_Lin_Server_Azure_Linux3] + condition: succeededOrFailed() + steps: + - template: trend-scenarios.yml + parameters: + connection: ASPNET Benchmarks Service Bus + serviceBusQueueName: azurearm64 + serviceBusNamespace: aspnetbenchmarks + arguments: "$(ciProfile) --profile idna-intel-lin-app --profile idna-intel-win-load " + +# GROUP 11 + +- job: Trends_Idna_Amd_Win + displayName: 11- Trends Idna Amd Win pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Idna_Amd_Lin, Trends_Idna_Amd_Win] + dependsOn: [Trends_Idna_Amd_Lin, Trends_Idna_Intel_Lin] condition: succeededOrFailed() steps: - template: trend-scenarios.yml @@ -193,13 +344,13 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: azure serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile idna-intel-lin-app --profile idna-amd-lin-load " + arguments: "$(ciProfile) --profile idna-amd-win-app --profile idna-intel-lin-load " - job: Trends_Idna_Intel_Win - displayName: 6- Trends Idna Intel Win + displayName: 11- Trends Idna Intel Win pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Idna_Amd_Lin, Trends_Idna_Amd_Win] + dependsOn: [Trends_Idna_Amd_Lin, Trends_Idna_Intel_Lin] condition: succeededOrFailed() steps: - template: trend-scenarios.yml @@ -207,15 +358,15 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: azurearm64 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile idna-intel-win-app --profile azure2-db-load " + arguments: "$(ciProfile) --profile idna-intel-win-app --profile idna-amd-lin-load " -# GROUP 7 +# GROUP 12 - job: Trends_Database_Azure_Server_Arm64 - displayName: 7- Trends Database Azure Server Arm64 + displayName: 12- Trends Database Azure Server Arm64 pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Idna_Intel_Lin, Trends_Idna_Intel_Win] + dependsOn: [Trends_Idna_Amd_Win, Trends_Idna_Intel_Win] condition: succeededOrFailed() steps: - template: trend-database-scenarios.yml @@ -226,10 +377,10 @@ jobs: arguments: "$(ciProfile) --profile azure-server-arm64-app --profile azure-client-load --profile azure-db-db " - job: Trends_Database_Azure2_Server_Amd64 - displayName: 7- Trends Database Azure2 Server Amd64 + displayName: 12- Trends Database Azure2 Server Amd64 pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Idna_Intel_Lin, Trends_Idna_Intel_Win] + dependsOn: [Trends_Idna_Amd_Win, Trends_Idna_Intel_Win] condition: succeededOrFailed() steps: - template: trend-database-scenarios.yml @@ -239,3 +390,35 @@ jobs: serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile azure2-server-amd64-app --profile azure2-client-load --profile azure2-db-db " +# GROUP 13 + +- job: Trends_Database_Cobalt_Cloud_Lin_Server + displayName: 13- Trends Database Cobalt Cloud Lin Server + pool: server + timeoutInMinutes: 120 + dependsOn: [Trends_Database_Azure_Server_Arm64, Trends_Database_Azure2_Server_Amd64] + condition: succeededOrFailed() + steps: + - template: trend-database-scenarios.yml + parameters: + connection: ASPNET Benchmarks Service Bus + serviceBusQueueName: azure + serviceBusNamespace: aspnetbenchmarks + arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-app --profile cobalt-cloud-lin-client-load --profile cobalt-cloud-lin-db-db " + +# GROUP 14 + +- job: Trends_Database_Cobalt_Cloud_Lin_Server_Azure_Linux3 + displayName: 14- Trends Database Cobalt Cloud Lin Server Azure Linux3 + pool: server + timeoutInMinutes: 120 + dependsOn: [Trends_Database_Cobalt_Cloud_Lin_Server] + condition: succeededOrFailed() + steps: + - template: trend-database-scenarios.yml + parameters: + connection: ASPNET Benchmarks Service Bus + serviceBusQueueName: azure + serviceBusNamespace: aspnetbenchmarks + arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-azure-linux3-app --profile cobalt-cloud-lin-client-load --profile cobalt-cloud-lin-db-db " + diff --git a/build/benchmarks.matrix.azure.eastus2.yml b/build/benchmarks.matrix.azure.eastus2.yml deleted file mode 100644 index 7ac6145fb..000000000 --- a/build/benchmarks.matrix.azure.eastus2.yml +++ /dev/null @@ -1,80 +0,0 @@ -# This file describes all the scenarios which are run continuously on AzDo. -# It generates the file benchmarks.yml. See this file for instructions. - -queues: - - cobaltcloud - -schedule: "0 9/12 * * *" - -groups: - - jobs: - - name: Trends Database Cobalt Cloud Linux - template: trend-database-scenarios.yml - profiles: - - cobalt-cloud-lin - - - jobs: - - name: Trends Cobalt Cloud Linux - template: trend-scenarios.yml - profiles: - - cobalt-cloud-lin - - - jobs: - - name: Baselines Database Cobalt Cloud Linux - template: baselines-database-scenarios.yml - profiles: - - cobalt-cloud-lin - - - jobs: - - name: Baselines Cobalt Cloud Linux - template: baselines-scenarios.yml - profiles: - - cobalt-cloud-lin - - - jobs: - - name: Containers Cobalt Cloud Linux - template: containers-scenarios.yml - profiles: - - cobalt-cloud-lin - - - jobs: - - name: Mvc Cobalt Cloud Linux - template: mvc-scenarios.yml - profiles: - - cobalt-cloud-lin - - - jobs: - - name: Trends Database Cobalt Cloud Linux AL3 - template: trend-database-scenarios.yml - profiles: - - cobalt-cloud-lin-al3 - - - jobs: - - name: Trends Cobalt Cloud Linux AL3 - template: trend-scenarios.yml - profiles: - - cobalt-cloud-lin-al3 - - - jobs: - - name: Baselines Database Cobalt Cloud Linux AL3 - template: baselines-database-scenarios.yml - profiles: - - cobalt-cloud-lin-al3 - - - jobs: - - name: Baselines Cobalt Cloud Linux AL3 - template: baselines-scenarios.yml - profiles: - - cobalt-cloud-lin-al3 - - - jobs: - - name: Containers Cobalt Cloud Linux AL3 - template: containers-scenarios.yml - profiles: - - cobalt-cloud-lin-al3 - - - jobs: - - name: Mvc Cobalt Cloud Linux AL3 - template: mvc-scenarios.yml - profiles: - - cobalt-cloud-lin-al3 \ No newline at end of file diff --git a/build/benchmarks.matrix.azure.yml b/build/benchmarks.matrix.azure.yml deleted file mode 100644 index 087f5de3a..000000000 --- a/build/benchmarks.matrix.azure.yml +++ /dev/null @@ -1,87 +0,0 @@ -# This file describes all the scenarios which are run continuously on AzDo. -# It generates the file benchmarks.yml. See this file for instructions. - -queues: - - azure - - azurearm64 - -schedule: "0 9/12 * * *" - -groups: - - jobs: - - name: Trends Database Azure Linux - template: trend-database-scenarios.yml - profiles: - - aspnet-azure-lin - - - name: Trends Database Azure Arm64 Linux - template: trend-database-scenarios.yml - profiles: - - aspnet-azurearm64-lin - - - jobs: - - name: Trends Azure Linux - template: trend-scenarios.yml - profiles: - - aspnet-azure-lin - - - name: Trends Azure Arm64 Linux - template: trend-scenarios.yml - profiles: - - aspnet-azurearm64-lin - - - jobs: - - name: Baselines Database Azure Linux - template: baselines-database-scenarios.yml - profiles: - - aspnet-azure-lin - - - name: Baselines Database Azure Arm64 Linux - template: baselines-database-scenarios.yml - profiles: - - aspnet-azurearm64-lin - - - jobs: - - name: Baselines Azure Linux - template: baselines-scenarios.yml - profiles: - - aspnet-azure-lin - - - name: Baselines Azure Arm64 Linux - template: baselines-scenarios.yml - profiles: - - aspnet-azurearm64-lin - - - jobs: - - name: Containers Azure Intel Linux - template: containers-scenarios.yml - profiles: - - aspnet-azure-lin - - - name: Containers Azure Arm64 Linux - template: containers-scenarios.yml - profiles: - - aspnet-azurearm64-lin - - - jobs: - - name: IDNA Azure Amd Linux - template: trend-scenarios.yml - profiles: - - idna-amd-lin - - - jobs: - - name: IDNA Azure Intel Linux - template: trend-scenarios.yml - profiles: - - idna-intel-lin - - - jobs: - - name: IDNA Azure Amd Windows - template: trend-scenarios.yml - profiles: - - idna-amd-win - - - name: IDNA Azure Intel Windows - template: trend-scenarios.yml - profiles: - - idna-intel-win \ No newline at end of file diff --git a/build/benchmarks.template.liquid b/build/benchmarks.template.liquid index c4c80642a..2aa230237 100644 --- a/build/benchmarks.template.liquid +++ b/build/benchmarks.template.liquid @@ -1,8 +1,9 @@ # Do not change this file, it is generated using these steps: -# - The file benchmarks.matrix.yml defines how each job is run in parallel -# - Convert its content to json using https://jsonformatter.org/yaml-to-json -# - Use the template in benchmarks.template.liquid and the converted json using https://liquidjs.com/playground.html -# - Update this file with the result of the template generation +# - The file benchmarks*.json defines how each pipeline set of jobs is run in parallel +# - Update the associated benchmarks*.json file with machine and scenario updates +# - Install python and install the requirements for the crank-scheduler in benchmarks/scripts/crank-scheduler/requirements.txt +# - Run the scheduler specifying the desired benchmarks*.json file, this template, and benchmarks/output to automatically overwrite the current pipeline. +# - Ex. python ./scripts/crank-scheduler/main.py --config ./build/benchmarks_ci.json --template ./build/benchmarks.template.liquid --yaml-output ./build {%- assign defaultTimeout = 120 %} diff --git a/build/benchmarks_ci_azure.json b/build/benchmarks_ci_azure.json index a795c5577..20160bf17 100644 --- a/build/benchmarks_ci_azure.json +++ b/build/benchmarks_ci_azure.json @@ -16,6 +16,7 @@ "machines": [ { "name": "azure-db", + "machine_group": "azure", "capabilities": { "db": { "priority": 1, @@ -39,6 +40,7 @@ }, { "name": "azure-client", + "machine_group": "azure", "capabilities": { "load": { "priority": 1, @@ -62,6 +64,7 @@ }, { "name": "azure-server-arm64", + "machine_group": "azure", "capabilities": { "sut": { "priority": 1, @@ -89,6 +92,7 @@ }, { "name": "azure2-client", + "machine_group": "azure2", "capabilities": { "load": { "priority": 1, @@ -112,6 +116,7 @@ }, { "name": "azure2-db", + "machine_group": "azure2", "capabilities": { "db": { "priority": 1, @@ -135,6 +140,7 @@ }, { "name": "azure2-server-amd64", + "machine_group": "azure2", "capabilities": { "sut": { "priority": 1, @@ -162,6 +168,7 @@ }, { "name": "idna-amd-lin", + "machine_group": "idna", "capabilities": { "sut": { "priority": 1, @@ -189,6 +196,7 @@ }, { "name": "idna-amd-win", + "machine_group": "idna", "capabilities": { "sut": { "priority": 1, @@ -216,6 +224,7 @@ }, { "name": "idna-intel-lin", + "machine_group": "idna", "capabilities": { "sut": { "priority": 1, @@ -243,6 +252,7 @@ }, { "name": "idna-intel-win", + "machine_group": "idna", "capabilities": { "sut": { "priority": 1, @@ -267,6 +277,110 @@ "idna-amd-lin", "azure2-db" ] + }, + { + "name": "cobalt-cloud-lin-server", + "machine_group": "cobalt-cloud", + "capabilities": { + "sut": { + "priority": 1, + "profiles": [ + "cobalt-cloud-lin-server-app" + ] + }, + "load": { + "priority": 2, + "profiles": [ + "cobalt-cloud-lin-server-load" + ] + }, + "db": { + "priority": 3, + "profiles": [ + "cobalt-cloud-lin-server-db" + ] + } + }, + "preferred_partners": [ + "cobalt-cloud-lin-client", + "cobalt-cloud-lin-db" + ] + }, + { + "name": "cobalt-cloud-lin-client", + "machine_group": "cobalt-cloud", + "capabilities": { + "load": { + "priority": 1, + "profiles": [ + "cobalt-cloud-lin-client-load" + ] + }, + "db": { + "priority": 2, + "profiles": [ + "cobalt-cloud-lin-client-db" + ] + }, + "sut": { + "priority": 3, + "profiles": [ + "cobalt-cloud-lin-client-app" + ] + } + } + }, + { + "name": "cobalt-cloud-lin-db", + "machine_group": "cobalt-cloud", + "capabilities": { + "db": { + "priority": 1, + "profiles": [ + "cobalt-cloud-lin-db-db" + ] + }, + "load": { + "priority": 2, + "profiles": [ + "cobalt-cloud-lin-db-load" + ] + }, + "sut": { + "priority": 3, + "profiles": [ + "cobalt-cloud-lin-db-app" + ] + } + } + }, + { + "name": "cobalt-cloud-lin-server-azure-linux3", + "machine_group": "cobalt-cloud", + "capabilities": { + "sut": { + "priority": 1, + "profiles": [ + "cobalt-cloud-lin-server-azure-linux3-app" + ] + }, + "load": { + "priority": 2, + "profiles": [ + "cobalt-cloud-lin-server-azure-linux3-load" + ] + }, + "db": { + "priority": 3, + "profiles": [ + "cobalt-cloud-lin-server-azure-linux3-db" + ] + } + }, + "preferred_partners": [ + "cobalt-cloud-lin-client", + "cobalt-cloud-lin-db" + ] } ], "scenarios": [ @@ -276,7 +390,9 @@ "type": 2, "target_machines": [ "azure-server-arm64", - "azure2-server-amd64" + "azure2-server-amd64", + "cobalt-cloud-lin-server", + "cobalt-cloud-lin-server-azure-linux3" ], "estimated_runtime": 30.0 }, @@ -286,7 +402,9 @@ "type": 3, "target_machines": [ "azure-server-arm64", - "azure2-server-amd64" + "azure2-server-amd64", + "cobalt-cloud-lin-server", + "cobalt-cloud-lin-server-azure-linux3" ], "estimated_runtime": 45.0 }, @@ -296,10 +414,22 @@ "type": 3, "target_machines": [ "azure-server-arm64", - "azure2-server-amd64" + "azure2-server-amd64", + "cobalt-cloud-lin-server", + "cobalt-cloud-lin-server-azure-linux3" ], "estimated_runtime": 90.0 }, + { + "name": "MVC", + "template": "mvc-scenarios.yml", + "type": 3, + "target_machines": [ + "cobalt-cloud-lin-server", + "cobalt-cloud-lin-server-azure-linux3" + ], + "estimated_runtime": 20.0 + }, { "name": "Trends", "template": "trend-scenarios.yml", @@ -307,6 +437,8 @@ "target_machines": [ "azure-server-arm64", "azure2-server-amd64", + "cobalt-cloud-lin-server", + "cobalt-cloud-lin-server-azure-linux3", "idna-amd-lin", "idna-amd-win", "idna-intel-lin", @@ -320,7 +452,9 @@ "type": 3, "target_machines": [ "azure-server-arm64", - "azure2-server-amd64" + "azure2-server-amd64", + "cobalt-cloud-lin-server", + "cobalt-cloud-lin-server-azure-linux3" ], "estimated_runtime": 15.0 } diff --git a/build/benchmarks_ci_azure_eastus2.json b/build/benchmarks_ci_azure_eastus2.json deleted file mode 100644 index 08b011f24..000000000 --- a/build/benchmarks_ci_azure_eastus2.json +++ /dev/null @@ -1,179 +0,0 @@ -{ - "metadata": { - "name": "Azure CI Benchmarks Configuration", - "description": "Combined machines and scenarios for continuous integration benchmarks on Azure", - "version": "1.0", - "schedule": "0 2/12 * * *", - "queues": [ - "cobaltcloud" - ], - "yaml_generation": { - "target_yaml_count": 1, - "schedule_offset_hours": 6 - } - }, - "machines": [ - { - "name": "cobalt-cloud-lin-server", - "capabilities": { - "sut": { - "priority": 1, - "profiles": [ - "cobalt-cloud-lin-server-app" - ] - }, - "load": { - "priority": 2, - "profiles": [ - "cobalt-cloud-lin-server-load" - ] - }, - "db": { - "priority": 3, - "profiles": [ - "cobalt-cloud-lin-server-db" - ] - } - }, - "preferred_partners": [ - "cobalt-cloud-lin-client", - "cobalt-cloud-lin-db" - ] - }, - { - "name": "cobalt-cloud-lin-client", - "capabilities": { - "load": { - "priority": 1, - "profiles": [ - "cobalt-cloud-lin-client-load" - ] - }, - "db": { - "priority": 2, - "profiles": [ - "cobalt-cloud-lin-client-db" - ] - }, - "sut": { - "priority": 3, - "profiles": [ - "cobalt-cloud-lin-client-app" - ] - } - } - }, - { - "name": "cobalt-cloud-lin-db", - "capabilities": { - "db": { - "priority": 1, - "profiles": [ - "cobalt-cloud-lin-db-db" - ] - }, - "load": { - "priority": 2, - "profiles": [ - "cobalt-cloud-lin-db-load" - ] - }, - "sut": { - "priority": 3, - "profiles": [ - "cobalt-cloud-lin-db-app" - ] - } - } - }, - { - "name": "cobalt-cloud-lin-server-azure-linux3", - "capabilities": { - "sut": { - "priority": 1, - "profiles": [ - "cobalt-cloud-lin-server-azure-linux3-app" - ] - }, - "load": { - "priority": 2, - "profiles": [ - "cobalt-cloud-lin-server-azure-linux3-load" - ] - }, - "db": { - "priority": 3, - "profiles": [ - "cobalt-cloud-lin-server-azure-linux3-db" - ] - } - }, - "preferred_partners": [ - "cobalt-cloud-lin-client", - "cobalt-cloud-lin-db" - ] - } - ], - "scenarios": [ - { - "name": "Baselines", - "template": "baselines-scenarios.yml", - "type": 2, - "target_machines": [ - "cobalt-cloud-lin-server", - "cobalt-cloud-lin-server-azure-linux3" - ], - "estimated_runtime": 30.0 - }, - { - "name": "Baselines Database", - "template": "baselines-database-scenarios.yml", - "type": 3, - "target_machines": [ - "cobalt-cloud-lin-server", - "cobalt-cloud-lin-server-azure-linux3" - ], - "estimated_runtime": 45.0 - }, - { - "name": "Containers", - "template": "containers-scenarios.yml", - "type": 3, - "target_machines": [ - "cobalt-cloud-lin-server", - "cobalt-cloud-lin-server-azure-linux3" - ], - "estimated_runtime": 90.0 - }, - { - "name": "MVC", - "template": "mvc-scenarios.yml", - "type": 3, - "target_machines": [ - "cobalt-cloud-lin-server", - "cobalt-cloud-lin-server-azure-linux3" - ], - "estimated_runtime": 20.0 - }, - { - "name": "Trends", - "template": "trend-scenarios.yml", - "type": 2, - "target_machines": [ - "cobalt-cloud-lin-server", - "cobalt-cloud-lin-server-azure-linux3" - ], - "estimated_runtime": 20.0 - }, - { - "name": "Trends Database", - "template": "trend-database-scenarios.yml", - "type": 3, - "target_machines": [ - "cobalt-cloud-lin-server", - "cobalt-cloud-lin-server-azure-linux3" - ], - "estimated_runtime": 15.0 - } - ] -} From 02071a91716163672fb14164e5343c65036cfba6 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 4 May 2026 15:12:51 -0700 Subject: [PATCH 3/5] Add pod-based crank scheduler for simplified benchmark scheduling (#2167) * Add pod-based crank scheduler prototype Simplified alternative to PR #2106's full crank-scheduler. Uses a pod model where machines are fixed groups (SUT + load + DB) instead of individual machines with capability scoring and preferred partners. Key simplifications: - Pods define fixed machine groupings (no role priority/scoring) - Shared machines between pods handled via collision detection - Same greedy longest-job-first bin-packing algorithm - Same Liquid template YAML generation - ~570 lines vs ~2000 lines in the full scheduler Includes: - scripts/pod-scheduler/ (5 Python files + README) - build/benchmarks_ci_pods.json (pod-based config for CI benchmarks) * Add azure, azure-eastus2, and cobalt pod configs Pod-based configurations for all three additional CI environments: - benchmarks_ci_azure_pods.json: 6 pods, 14 runs (matches main) - benchmarks_ci_azure_eastus2_pods.json: 2 pods, 12 runs (matches main) - benchmarks_ci_cobalt_pods.json: 4 pods, 44 runs (matches main) Notable pod patterns: - Azure IDNA pods cross-use each other as load machines - Cobalt hosted has 28-core variant pods sharing physical machines with full-core pods (handled by collision detection) - Azure eastus2 pods share load/db, serialized automatically Also fixes unicode bar chars for Windows compatibility. * Update azure pod config: merge eastus2, keep IDNA on linux loads Reflects main branch changes from PR #2166: - Merged cobalt-cloud-lin pods (eastus2) into azure config - Removed separate benchmarks_ci_azure_eastus2_pods.json - Kept IDNA pod load profiles on linux machines (load jobs require linux), reverting the main branch profile change - Added cobalt-cloud-lin-azl3-dual pod for type-2 scenarios (uses cobalt-cloud-lin-db as load instead of client) - Total runs: 26 (matches main azure pipeline) * Regenerate pipeline YAMLs from pod-scheduler configs Generated via: python ./scripts/pod-scheduler/main.py --config ./build/benchmarks_ci_pods.json --template ./build/benchmarks.template.liquid --yaml-output ./build python ./scripts/pod-scheduler/main.py --config ./build/benchmarks_ci_azure_pods.json --template ./build/benchmarks.template.liquid --yaml-output ./build --base-name benchmarks-ci-azure python ./scripts/pod-scheduler/main.py --config ./build/benchmarks_ci_cobalt_pods.json --template ./build/benchmarks.template.liquid --yaml-output ./build --base-name benchmarks-ci-cobalt * Cap timeoutInMinutes at 240 (max 2x old 120 default) Formula is now max(120, min(240, 2 * estimated_runtime)). This prevents scenarios with long runtimes (e.g. Proxies at 150min) from setting unreasonably high timeouts compared to previous values. Resulting timeouts: 120 (default), 140 (Grpc), 180 (PGO/Containers), 240 (Proxies) * Address review feedback - Fix 4 incorrect template filenames in benchmarks_ci_pods.json: crossgen-scenarios -> crossgen2-scenarios, custom-proxies-scenarios -> proxies-custom-scenarios, single-file-scenarios -> singlefile-scenarios, websockets-scenarios -> websocket-scenarios - Fix machine utilization calculation bug (was inflating totals for machines not in current stage) - Remove unused imports (sys, Any, Dict, json, Pod) - Remove dead render_with_liquid function and --template CLI arg - Add guard against empty queues (ZeroDivisionError) - Update README and docstrings to reflect removed template arg Code: - Validate cron schedules at load time and raise on unsupported hour fields instead of silently no-op'ing the offset for split YAMLs - Add optional 'timeout' override per scenario; fall back to the runtime-derived formula when absent - Move pipeline plumbing (pool, service-bus connection/namespace) into JSON metadata.pipeline with the previous hardcoded values as defaults - Strict validation of duplicate pods, duplicate scenario.pods entries, empty queues; default scheduler to fail-fast on unknown/invalid pod references with a --lenient opt-out - Stricter job-id sanitization (handles '.', '/', parens, leading digits, unicode) and explicit duplicate detection in generated YAML - Replace id(stage) bookkeeping in split_schedule with explicit indices; add stable name tie-breaker to create_schedule for deterministic output - Use Run.job_name in the generator instead of duplicating the regex - Drop stale '--template' arg from generated YAML headers and README Tests: - 41 unit + snapshot tests covering models, config loader, scheduler, generator, and YAML parity with the committed *_pods.json configs Cleanup: - Revert benchmarks.template.liquid and benchmarks_ci_azure.json to main; the deleted crank-scheduler does not consume them - Regenerate all four pipeline YAMLs against the new generator * Remove unused benchmarks.template.liquid The Liquid template was only consumed by the deleted crank-scheduler. The pod-scheduler renders pipeline YAML directly via Python, and grep confirms no other script, pipeline, or build step reads this file. * Remove orphaned benchmarks.yml and benchmarks.matrix.0[12].yml These were artifacts of the old hand-driven matrix.yml -> json -> Liquid template -> benchmarks.yml workflow. Their only inbound references were stale documentation comments cross-pointing between each other; nothing in the repo (no script, no pipeline) consumed them. * Document pod-scheduler flow across READMEs and YAML headers - Generated YAML headers now embed the exact regen command (with the source config and base name) and a pointer to scripts/pod-scheduler/README.md, so each file documents how to reproduce itself - New build/README.md maps each *_pods.json config to the YAML it produces, lists the hand-maintained scenario templates, and explains the typical edit/regenerate workflow - Top-level README.md gains a 'Continuous benchmarking pipelines' section linking to the pod-scheduler and build/ docs - pod-scheduler README's Quick Start now uses repo-root-relative commands and points at the snapshot tests for verification - Tests cover the new _format_source_path helper and the snapshot test passes the source config so headers stay verified * Remove orphaned crank-scheduler JSON configs benchmarks_ci.json, benchmarks_ci_azure.json, and benchmarks_ci_cobalt.json used the old 'machines + capabilities' format consumed by the deleted crank-scheduler. Their replacements (benchmarks_ci_pods.json, benchmarks_ci_azure_pods.json, benchmarks_ci_cobalt_pods.json) drive the pod-scheduler. grep finds zero inbound references for any of the three across scripts, pipelines, docs, and tests. --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: Parker Bibus --- README.md | 7 + build/README.md | 71 ++ build/benchmarks-ci-01.yml | 436 ++++----- build/benchmarks-ci-02.yml | 504 +++++------ build/benchmarks-ci-azure.yml | 232 +++-- build/benchmarks-ci-cobalt.yml | 371 ++++---- build/benchmarks.matrix.01.yml | 299 ------- build/benchmarks.matrix.02.yml | 231 ----- build/benchmarks.template.liquid | 56 -- build/benchmarks.yml | 847 ------------------ build/benchmarks_ci.json | 522 ----------- build/benchmarks_ci_azure.json | 462 ---------- build/benchmarks_ci_azure_pods.json | 107 +++ build/benchmarks_ci_cobalt.json | 396 -------- build/benchmarks_ci_cobalt_pods.json | 124 +++ build/benchmarks_ci_pods.json | 213 +++++ scripts/pod-scheduler/README.md | 197 ++++ scripts/pod-scheduler/config_loader.py | 134 +++ scripts/pod-scheduler/generator.py | 246 +++++ scripts/pod-scheduler/main.py | 232 +++++ scripts/pod-scheduler/models.py | 194 ++++ scripts/pod-scheduler/scheduler.py | 121 +++ scripts/pod-scheduler/tests/__init__.py | 14 + .../pod-scheduler/tests/test_config_loader.py | 117 +++ scripts/pod-scheduler/tests/test_generator.py | 81 ++ scripts/pod-scheduler/tests/test_models.py | 106 +++ scripts/pod-scheduler/tests/test_scheduler.py | 160 ++++ scripts/pod-scheduler/tests/test_snapshots.py | 79 ++ 28 files changed, 2976 insertions(+), 3583 deletions(-) create mode 100644 build/README.md delete mode 100644 build/benchmarks.matrix.01.yml delete mode 100644 build/benchmarks.matrix.02.yml delete mode 100644 build/benchmarks.template.liquid delete mode 100644 build/benchmarks.yml delete mode 100644 build/benchmarks_ci.json delete mode 100644 build/benchmarks_ci_azure.json create mode 100644 build/benchmarks_ci_azure_pods.json delete mode 100644 build/benchmarks_ci_cobalt.json create mode 100644 build/benchmarks_ci_cobalt_pods.json create mode 100644 build/benchmarks_ci_pods.json create mode 100644 scripts/pod-scheduler/README.md create mode 100644 scripts/pod-scheduler/config_loader.py create mode 100644 scripts/pod-scheduler/generator.py create mode 100644 scripts/pod-scheduler/main.py create mode 100644 scripts/pod-scheduler/models.py create mode 100644 scripts/pod-scheduler/scheduler.py create mode 100644 scripts/pod-scheduler/tests/__init__.py create mode 100644 scripts/pod-scheduler/tests/test_config_loader.py create mode 100644 scripts/pod-scheduler/tests/test_generator.py create mode 100644 scripts/pod-scheduler/tests/test_models.py create mode 100644 scripts/pod-scheduler/tests/test_scheduler.py create mode 100644 scripts/pod-scheduler/tests/test_snapshots.py diff --git a/README.md b/README.md index b25618519..ebd214940 100644 --- a/README.md +++ b/README.md @@ -15,3 +15,10 @@ The scenarios that are available to use with Crank and some sample command lines # Contributing Microsoft employees can approve and merge contributions by joining [this team](https://repos.opensource.microsoft.com/orgs/aspnet/teams/aspnet-push) + +# Continuous benchmarking pipelines + +The Azure DevOps pipelines that drive the daily benchmark sweep are generated +from JSON configs by the [pod-scheduler](scripts/pod-scheduler/README.md). +See [`build/README.md`](build/README.md) for the mapping between configs and +generated YAML files, and how to regenerate them after editing. diff --git a/build/README.md b/build/README.md new file mode 100644 index 000000000..205883ac0 --- /dev/null +++ b/build/README.md @@ -0,0 +1,71 @@ +# `build/` — Pipeline definitions + +This directory contains Azure DevOps pipeline YAML files and the pod-scheduler +configs that generate them. + +## Generated files (do not edit by hand) + +The CI pipelines that run the daily benchmark sweep are generated by the +[pod-scheduler](../scripts/pod-scheduler/README.md). Each pipeline maps to a +JSON config that lists the available pods (machine groups) and scenarios. + +| Generated YAML | Source config | Notes | +|---|---|---| +| `benchmarks-ci-01.yml`, `benchmarks-ci-02.yml` | `benchmarks_ci_pods.json` | On-prem fleet (citrine + mono); split across two YAMLs to spread load | +| `benchmarks-ci-azure.yml` | `benchmarks_ci_azure_pods.json` | Azure cloud pods (includes the merged eastus2 cobalt cloud pods) | +| `benchmarks-ci-cobalt.yml` | `benchmarks_ci_cobalt_pods.json` | Cobalt-hosted pods | + +To regenerate, from the repo root: + +```bash +python ./scripts/pod-scheduler/main.py \ + --config ./build/benchmarks_ci_pods.json \ + --yaml-output ./build + +python ./scripts/pod-scheduler/main.py \ + --config ./build/benchmarks_ci_azure_pods.json \ + --base-name benchmarks-ci-azure \ + --yaml-output ./build + +python ./scripts/pod-scheduler/main.py \ + --config ./build/benchmarks_ci_cobalt_pods.json \ + --base-name benchmarks-ci-cobalt \ + --yaml-output ./build +``` + +The header of each generated YAML embeds the exact regen command for that +file. + +## Hand-maintained scenario templates + +Everything else in this directory is hand-maintained. The most important +groups: + +- `*-scenarios.yml` — AzDO job templates that the generated pipelines invoke + (e.g. `proxies-scenarios.yml`, `grpc-scenarios.yml`). One per benchmark + family. The pod-scheduler picks which template to run via the + `template` field on each scenario in the JSON config. +- `prbenchmarks.*.config.yml` — PR-trigger pipelines for individual PR runs. +- `job-variables.yml` — shared variable definitions consumed by the scenario + templates. + +When adding a new benchmark family, drop a `*-scenarios.yml` template in this +directory and reference it from the relevant `benchmarks_ci_*_pods.json`. + +## Editing the schedule + +Common edits, all done in JSON: + +- **Add a scenario**: append to `scenarios[]` in the appropriate + `benchmarks_ci_*_pods.json`. +- **Add a pod / machine class**: append to `pods[]` and reference its name + from any scenario you want to target it. +- **Change cadence or splits**: edit `metadata.schedule` (cron), + `metadata.yaml_generation.target_yaml_count`, or + `metadata.yaml_generation.schedule_offset_hours`. +- **Override a job's timeout**: add `"timeout": ` to the scenario. + +After any edit, regenerate the YAML and commit both the JSON and the YAML +together. The snapshot tests in +`scripts/pod-scheduler/tests/test_snapshots.py` will fail in CI if the +generated YAML drifts from the configs. diff --git a/build/benchmarks-ci-01.yml b/build/benchmarks-ci-01.yml index 975cd715d..30e73842e 100644 --- a/build/benchmarks-ci-01.yml +++ b/build/benchmarks-ci-01.yml @@ -1,16 +1,14 @@ -# Do not change this file, it is generated using these steps: -# - The file benchmarks*.json defines how each pipeline set of jobs is run in parallel -# - Update the associated benchmarks*.json file with machine and scenario updates -# - Install python and install the requirements for the crank-scheduler in benchmarks/scripts/crank-scheduler/requirements.txt -# - Run the scheduler specifying the desired benchmarks*.json file, this template, and benchmarks/output to automatically overwrite the current pipeline. -# - Ex. python ./scripts/crank-scheduler/main.py --config ./build/benchmarks_ci.json --template ./build/benchmarks.template.liquid --yaml-output ./build +# Do not change this file, it is generated by the pod-scheduler. +# Source of truth: see ../scripts/pod-scheduler/README.md +# To regenerate, run from the repo root: +# python ./scripts/pod-scheduler/main.py --config ./build/benchmarks_ci_pods.json --base-name benchmarks-ci --yaml-output ./build trigger: none pr: none schedules: - cron: "0 3/12 * * *" - always: true # always run the pipeline even if there have not been source code changes since the last successful scheduled run. + always: true branches: include: - main @@ -32,10 +30,10 @@ jobs: # GROUP 1 -- job: Proxies_Gold_Lin - displayName: 1- Proxies Gold Lin +- job: Proxies_gold_lin + displayName: 1- Proxies gold-lin pool: server - timeoutInMinutes: 120 + timeoutInMinutes: 240 dependsOn: [] condition: succeededOrFailed() steps: @@ -45,11 +43,11 @@ jobs: serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load --profile gold-db-db " - -- job: Proxies_Intel_Lin - displayName: 1- Proxies Intel Lin + +- job: Proxies_intel_lin + displayName: 1- Proxies intel-lin pool: server - timeoutInMinutes: 120 + timeoutInMinutes: 240 dependsOn: [] condition: succeededOrFailed() steps: @@ -59,11 +57,11 @@ jobs: serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile intel-db-db " - -- job: Grpc_Gold_Win - displayName: 1- Grpc Gold Win + +- job: Grpc_gold_win + displayName: 1- Grpc gold-win pool: server - timeoutInMinutes: 120 + timeoutInMinutes: 140 dependsOn: [] condition: succeededOrFailed() steps: @@ -73,11 +71,11 @@ jobs: serviceBusQueueName: citrine3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load " - -- job: Grpc_Intel_Win - displayName: 1- Grpc Intel Win + +- job: Grpc_intel_win + displayName: 1- Grpc intel-win pool: server - timeoutInMinutes: 120 + timeoutInMinutes: 140 dependsOn: [] condition: succeededOrFailed() steps: @@ -87,14 +85,14 @@ jobs: serviceBusQueueName: mono serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load " - + # GROUP 2 -- job: PGO_Gold_Win - displayName: 2- PGO Gold Win +- job: PGO_gold_win + displayName: 2- PGO gold-win pool: server - timeoutInMinutes: 120 - dependsOn: [Proxies_Gold_Lin, Proxies_Intel_Lin, Grpc_Gold_Win, Grpc_Intel_Win] + timeoutInMinutes: 180 + dependsOn: [Proxies_gold_lin, Proxies_intel_lin, Grpc_gold_win, Grpc_intel_win] condition: succeededOrFailed() steps: - template: pgo-scenarios.yml @@ -103,12 +101,12 @@ jobs: serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load --profile gold-db-db " - -- job: PGO_Intel_Win - displayName: 2- PGO Intel Win + +- job: PGO_intel_win + displayName: 2- PGO intel-win pool: server - timeoutInMinutes: 120 - dependsOn: [Proxies_Gold_Lin, Proxies_Intel_Lin, Grpc_Gold_Win, Grpc_Intel_Win] + timeoutInMinutes: 180 + dependsOn: [Proxies_gold_lin, Proxies_intel_lin, Grpc_gold_win, Grpc_intel_win] condition: succeededOrFailed() steps: - template: pgo-scenarios.yml @@ -117,12 +115,12 @@ jobs: serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load --profile intel-db-db " - -- job: Grpc_Gold_Lin - displayName: 2- Grpc Gold Lin + +- job: Grpc_gold_lin + displayName: 2- Grpc gold-lin pool: server - timeoutInMinutes: 120 - dependsOn: [Proxies_Gold_Lin, Proxies_Intel_Lin, Grpc_Gold_Win, Grpc_Intel_Win] + timeoutInMinutes: 140 + dependsOn: [Proxies_gold_lin, Proxies_intel_lin, Grpc_gold_win, Grpc_intel_win] condition: succeededOrFailed() steps: - template: grpc-scenarios.yml @@ -131,12 +129,12 @@ jobs: serviceBusQueueName: citrine3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load " - -- job: Grpc_Intel_Lin - displayName: 2- Grpc Intel Lin + +- job: Grpc_intel_lin + displayName: 2- Grpc intel-lin pool: server - timeoutInMinutes: 120 - dependsOn: [Proxies_Gold_Lin, Proxies_Intel_Lin, Grpc_Gold_Win, Grpc_Intel_Win] + timeoutInMinutes: 140 + dependsOn: [Proxies_gold_lin, Proxies_intel_lin, Grpc_gold_win, Grpc_intel_win] condition: succeededOrFailed() steps: - template: grpc-scenarios.yml @@ -145,14 +143,14 @@ jobs: serviceBusQueueName: mono serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load " - + # GROUP 3 -- job: Baselines_Database_Amd_Lin2 - displayName: 3- Baselines Database Amd Lin2 +- job: Baselines_Database_gold_win + displayName: 3- Baselines Database gold-win pool: server timeoutInMinutes: 120 - dependsOn: [PGO_Gold_Win, PGO_Intel_Win, Grpc_Gold_Lin, Grpc_Intel_Lin] + dependsOn: [PGO_gold_win, PGO_intel_win, Grpc_gold_lin, Grpc_intel_lin] condition: succeededOrFailed() steps: - template: baselines-database-scenarios.yml @@ -160,27 +158,27 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile amd-lin2-app --profile gold-load-load --profile gold-db-db " - -- job: HttpClient_Gold_Lin - displayName: 3- HttpClient Gold Lin + arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load --profile gold-db-db " + +- job: Blazor_gold_lin + displayName: 3- Blazor gold-lin pool: server timeoutInMinutes: 120 - dependsOn: [PGO_Gold_Win, PGO_Intel_Win, Grpc_Gold_Lin, Grpc_Intel_Lin] + dependsOn: [PGO_gold_win, PGO_intel_win, Grpc_gold_lin, Grpc_intel_lin] condition: succeededOrFailed() steps: - - template: httpclient-scenarios.yml + - template: blazor-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load2-load " - -- job: HttpClient_Intel_Lin - displayName: 3- HttpClient Intel Lin + arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load " + +- job: HttpClient_intel_lin + displayName: 3- HttpClient intel-lin pool: server timeoutInMinutes: 120 - dependsOn: [PGO_Gold_Win, PGO_Intel_Win, Grpc_Gold_Lin, Grpc_Intel_Lin] + dependsOn: [PGO_gold_win, PGO_intel_win, Grpc_gold_lin, Grpc_intel_lin] condition: succeededOrFailed() steps: - template: httpclient-scenarios.yml @@ -189,12 +187,12 @@ jobs: serviceBusQueueName: citrine3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load " - -- job: SignalR_Intel_Win - displayName: 3- SignalR Intel Win + +- job: SignalR_intel_win + displayName: 3- SignalR intel-win pool: server timeoutInMinutes: 120 - dependsOn: [PGO_Gold_Win, PGO_Intel_Win, Grpc_Gold_Lin, Grpc_Intel_Lin] + dependsOn: [PGO_gold_win, PGO_intel_win, Grpc_gold_lin, Grpc_intel_lin] condition: succeededOrFailed() steps: - template: signalr-scenarios.yml @@ -203,28 +201,28 @@ jobs: serviceBusQueueName: mono serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load " - + # GROUP 4 -- job: Baselines_Gold_Lin - displayName: 4- Baselines Gold Lin +- job: SslStream_gold_lin + displayName: 4- SslStream gold-lin pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Amd_Lin2, HttpClient_Gold_Lin, HttpClient_Intel_Lin, SignalR_Intel_Win] + dependsOn: [Baselines_Database_gold_win, Blazor_gold_lin, HttpClient_intel_lin, SignalR_intel_win] condition: succeededOrFailed() steps: - - template: baselines-scenarios.yml + - template: sslstream-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load " - -- job: Baselines_Intel_Lin - displayName: 4- Baselines Intel Lin + +- job: Baselines_intel_lin + displayName: 4- Baselines intel-lin pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Amd_Lin2, HttpClient_Gold_Lin, HttpClient_Intel_Lin, SignalR_Intel_Win] + dependsOn: [Baselines_Database_gold_win, Blazor_gold_lin, HttpClient_intel_lin, SignalR_intel_win] condition: succeededOrFailed() steps: - template: baselines-scenarios.yml @@ -233,26 +231,26 @@ jobs: serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load " - -- job: Frameworks_Database_Amd_Lin2 - displayName: 4- Frameworks Database Amd Lin2 + +- job: NativeAOT_gold_win + displayName: 4- NativeAOT gold-win pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Amd_Lin2, HttpClient_Gold_Lin, HttpClient_Intel_Lin, SignalR_Intel_Win] + dependsOn: [Baselines_Database_gold_win, Blazor_gold_lin, HttpClient_intel_lin, SignalR_intel_win] condition: succeededOrFailed() steps: - - template: frameworks-database-scenarios.yml + - template: nativeaot-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine3 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile amd-lin2-app --profile gold-load2-load --profile gold-db-db " - -- job: NativeAOT_Intel_Win - displayName: 4- NativeAOT Intel Win + arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load --profile gold-db-db " + +- job: NativeAOT_intel_win + displayName: 4- NativeAOT intel-win pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Amd_Lin2, HttpClient_Gold_Lin, HttpClient_Intel_Lin, SignalR_Intel_Win] + dependsOn: [Baselines_Database_gold_win, Blazor_gold_lin, HttpClient_intel_lin, SignalR_intel_win] condition: succeededOrFailed() steps: - template: nativeaot-scenarios.yml @@ -261,86 +259,86 @@ jobs: serviceBusQueueName: mono serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load --profile intel-db-db " - + # GROUP 5 -- job: SignalR_Gold_Lin - displayName: 5- SignalR Gold Lin +- job: Baselines_gold_lin + displayName: 5- Baselines gold-lin pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Gold_Lin, Baselines_Intel_Lin, Frameworks_Database_Amd_Lin2, NativeAOT_Intel_Win] + dependsOn: [SslStream_gold_lin, Baselines_intel_lin, NativeAOT_gold_win, NativeAOT_intel_win] condition: succeededOrFailed() steps: - - template: signalr-scenarios.yml + - template: baselines-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load " - -- job: SignalR_Intel_Lin - displayName: 5- SignalR Intel Lin + +- job: Frameworks_Database_intel_lin + displayName: 5- Frameworks Database intel-lin pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Gold_Lin, Baselines_Intel_Lin, Frameworks_Database_Amd_Lin2, NativeAOT_Intel_Win] + dependsOn: [SslStream_gold_lin, Baselines_intel_lin, NativeAOT_gold_win, NativeAOT_intel_win] condition: succeededOrFailed() steps: - - template: signalr-scenarios.yml + - template: frameworks-database-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load " - -- job: MVC_Gold_Win - displayName: 5- MVC Gold Win + arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile intel-db-db " + +- job: EF_Core_intel_win + displayName: 5- EF Core intel-win pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Gold_Lin, Baselines_Intel_Lin, Frameworks_Database_Amd_Lin2, NativeAOT_Intel_Win] + dependsOn: [SslStream_gold_lin, Baselines_intel_lin, NativeAOT_gold_win, NativeAOT_intel_win] condition: succeededOrFailed() steps: - - template: mvc-scenarios.yml + - template: efcore-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine3 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load --profile gold-db-db " - -- job: Trends_Intel_Win - displayName: 5- Trends Intel Win + arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load " + +- job: GC_gold_win + displayName: 5- GC gold-win pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Gold_Lin, Baselines_Intel_Lin, Frameworks_Database_Amd_Lin2, NativeAOT_Intel_Win] + dependsOn: [SslStream_gold_lin, Baselines_intel_lin, NativeAOT_gold_win, NativeAOT_intel_win] condition: succeededOrFailed() steps: - - template: trend-scenarios.yml + - template: gc-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: mono serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load " - + arguments: "$(ciProfile) --profile gold-win-app " + # GROUP 6 -- job: NativeAOT_Gold_Lin - displayName: 6- NativeAOT Gold Lin +- job: Frameworks_Database_amd_lin2 + displayName: 6- Frameworks Database amd-lin2 pool: server timeoutInMinutes: 120 - dependsOn: [SignalR_Gold_Lin, SignalR_Intel_Lin, MVC_Gold_Win, Trends_Intel_Win] + dependsOn: [Baselines_gold_lin, Frameworks_Database_intel_lin, EF_Core_intel_win, GC_gold_win] condition: succeededOrFailed() steps: - - template: nativeaot-scenarios.yml + - template: frameworks-database-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load --profile gold-db-db " - -- job: NativeAOT_Intel_Lin - displayName: 6- NativeAOT Intel Lin + arguments: "$(ciProfile) --profile amd-lin2-app --profile gold-load-load --profile gold-db-db " + +- job: NativeAOT_intel_lin + displayName: 6- NativeAOT intel-lin pool: server timeoutInMinutes: 120 - dependsOn: [SignalR_Gold_Lin, SignalR_Intel_Lin, MVC_Gold_Win, Trends_Intel_Win] + dependsOn: [Baselines_gold_lin, Frameworks_Database_intel_lin, EF_Core_intel_win, GC_gold_win] condition: succeededOrFailed() steps: - template: nativeaot-scenarios.yml @@ -349,56 +347,56 @@ jobs: serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile intel-db-db " - -- job: EF_Core_Gold_Win - displayName: 6- EF Core Gold Win + +- job: GC_gold_lin + displayName: 6- GC gold-lin pool: server timeoutInMinutes: 120 - dependsOn: [SignalR_Gold_Lin, SignalR_Intel_Lin, MVC_Gold_Win, Trends_Intel_Win] + dependsOn: [Baselines_gold_lin, Frameworks_Database_intel_lin, EF_Core_intel_win, GC_gold_win] condition: succeededOrFailed() steps: - - template: efcore-scenarios.yml + - template: gc-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine3 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load " - -- job: GC_Intel_Win - displayName: 6- GC Intel Win + arguments: "$(ciProfile) --profile gold-lin-app " + +- job: Single_File_gold_win + displayName: 6- Single File gold-win pool: server timeoutInMinutes: 120 - dependsOn: [SignalR_Gold_Lin, SignalR_Intel_Lin, MVC_Gold_Win, Trends_Intel_Win] + dependsOn: [Baselines_gold_lin, Frameworks_Database_intel_lin, EF_Core_intel_win, GC_gold_win] condition: succeededOrFailed() steps: - - template: gc-scenarios.yml + - template: singlefile-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: mono serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile intel-win-app " - + arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load " + # GROUP 7 -- job: Frameworks_Gold_Lin - displayName: 7- Frameworks Gold Lin +- job: MVC_gold_lin + displayName: 7- MVC gold-lin pool: server timeoutInMinutes: 120 - dependsOn: [NativeAOT_Gold_Lin, NativeAOT_Intel_Lin, EF_Core_Gold_Win, GC_Intel_Win] + dependsOn: [Frameworks_Database_amd_lin2, NativeAOT_intel_lin, GC_gold_lin, Single_File_gold_win] condition: succeededOrFailed() steps: - - template: frameworks-scenarios.yml + - template: mvc-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load --profile gold-db-db " - -- job: Frameworks_Intel_Lin - displayName: 7- Frameworks Intel Lin + +- job: Frameworks_intel_lin + displayName: 7- Frameworks intel-lin pool: server timeoutInMinutes: 120 - dependsOn: [NativeAOT_Gold_Lin, NativeAOT_Intel_Lin, EF_Core_Gold_Win, GC_Intel_Win] + dependsOn: [Frameworks_Database_amd_lin2, NativeAOT_intel_lin, GC_gold_lin, Single_File_gold_win] condition: succeededOrFailed() steps: - template: frameworks-scenarios.yml @@ -407,12 +405,12 @@ jobs: serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile intel-db-db " - -- job: Single_File_Gold_Win - displayName: 7- Single File Gold Win + +- job: Single_File_intel_win + displayName: 7- Single File intel-win pool: server timeoutInMinutes: 120 - dependsOn: [NativeAOT_Gold_Lin, NativeAOT_Intel_Lin, EF_Core_Gold_Win, GC_Intel_Win] + dependsOn: [Frameworks_Database_amd_lin2, NativeAOT_intel_lin, GC_gold_lin, Single_File_gold_win] condition: succeededOrFailed() steps: - template: singlefile-scenarios.yml @@ -420,43 +418,43 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine3 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load " - -- job: Single_File_Intel_Win - displayName: 7- Single File Intel Win + arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load " + +- job: Crossgen_gold_win + displayName: 7- Crossgen gold-win pool: server timeoutInMinutes: 120 - dependsOn: [NativeAOT_Gold_Lin, NativeAOT_Intel_Lin, EF_Core_Gold_Win, GC_Intel_Win] + dependsOn: [Frameworks_Database_amd_lin2, NativeAOT_intel_lin, GC_gold_lin, Single_File_gold_win] condition: succeededOrFailed() steps: - - template: singlefile-scenarios.yml + - template: crossgen2-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: mono serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load " - + arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load " + # GROUP 8 -- job: Trends_Database_Gold_Lin - displayName: 8- Trends Database Gold Lin +- job: Trends_amd_lin2 + displayName: 8- Trends amd-lin2 pool: server timeoutInMinutes: 120 - dependsOn: [Frameworks_Gold_Lin, Frameworks_Intel_Lin, Single_File_Gold_Win, Single_File_Intel_Win] + dependsOn: [MVC_gold_lin, Frameworks_intel_lin, Single_File_intel_win, Crossgen_gold_win] condition: succeededOrFailed() steps: - - template: trend-database-scenarios.yml + - template: trend-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load --profile gold-db-db " - -- job: Trends_Database_Intel_Lin - displayName: 8- Trends Database Intel Lin + arguments: "$(ciProfile) --profile amd-lin2-app --profile gold-load-load " + +- job: Trends_Database_intel_lin + displayName: 8- Trends Database intel-lin pool: server timeoutInMinutes: 120 - dependsOn: [Frameworks_Gold_Lin, Frameworks_Intel_Lin, Single_File_Gold_Win, Single_File_Intel_Win] + dependsOn: [MVC_gold_lin, Frameworks_intel_lin, Single_File_intel_win, Crossgen_gold_win] condition: succeededOrFailed() steps: - template: trend-database-scenarios.yml @@ -465,56 +463,56 @@ jobs: serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile intel-db-db " - -- job: WebSockets_Intel_Win - displayName: 8- WebSockets Intel Win + +- job: Crossgen_intel_win + displayName: 8- Crossgen intel-win pool: server timeoutInMinutes: 120 - dependsOn: [Frameworks_Gold_Lin, Frameworks_Intel_Lin, Single_File_Gold_Win, Single_File_Intel_Win] + dependsOn: [MVC_gold_lin, Frameworks_intel_lin, Single_File_intel_win, Crossgen_gold_win] condition: succeededOrFailed() steps: - - template: websocket-scenarios.yml + - template: crossgen2-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load " - -- job: Crossgen_Gold_Win - displayName: 8- Crossgen Gold Win + +- job: Build_gold_lin + displayName: 8- Build gold-lin pool: server timeoutInMinutes: 120 - dependsOn: [Frameworks_Gold_Lin, Frameworks_Intel_Lin, Single_File_Gold_Win, Single_File_Intel_Win] + dependsOn: [MVC_gold_lin, Frameworks_intel_lin, Single_File_intel_win, Crossgen_gold_win] condition: succeededOrFailed() steps: - - template: crossgen2-scenarios.yml + - template: build-perf-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: mono serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load " - + arguments: "$(ciProfile) --profile gold-lin-app " + # GROUP 9 -- job: Trends_Database_Amd_Lin2 - displayName: 9- Trends Database Amd Lin2 +- job: Frameworks_amd_lin2 + displayName: 9- Frameworks amd-lin2 pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Database_Gold_Lin, Trends_Database_Intel_Lin, WebSockets_Intel_Win, Crossgen_Gold_Win] + dependsOn: [Trends_amd_lin2, Trends_Database_intel_lin, Crossgen_intel_win, Build_gold_lin] condition: succeededOrFailed() steps: - - template: trend-database-scenarios.yml + - template: frameworks-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile amd-lin2-app --profile gold-load-load --profile gold-db-db " - -- job: WebSockets_Gold_Lin - displayName: 9- WebSockets Gold Lin + +- job: WebSockets_intel_lin + displayName: 9- WebSockets intel-lin pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Database_Gold_Lin, Trends_Database_Intel_Lin, WebSockets_Intel_Win, Crossgen_Gold_Win] + dependsOn: [Trends_amd_lin2, Trends_Database_intel_lin, Crossgen_intel_win, Build_gold_lin] condition: succeededOrFailed() steps: - template: websocket-scenarios.yml @@ -522,63 +520,97 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load2-load " - -- job: WebSockets_Intel_Lin - displayName: 9- WebSockets Intel Lin + arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load " + +# GROUP 10 + +- job: Frameworks_gold_lin + displayName: 10- Frameworks gold-lin pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Database_Gold_Lin, Trends_Database_Intel_Lin, WebSockets_Intel_Win, Crossgen_Gold_Win] + dependsOn: [Frameworks_amd_lin2, WebSockets_intel_lin] condition: succeededOrFailed() steps: - - template: websocket-scenarios.yml + - template: frameworks-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus - serviceBusQueueName: citrine3 + serviceBusQueueName: citrine1 + serviceBusNamespace: aspnetbenchmarks + arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load --profile gold-db-db " + +- job: Crossgen_intel_lin + displayName: 10- Crossgen intel-lin + pool: server + timeoutInMinutes: 120 + dependsOn: [Frameworks_amd_lin2, WebSockets_intel_lin] + condition: succeededOrFailed() + steps: + - template: crossgen2-scenarios.yml + parameters: + connection: ASPNET Benchmarks Service Bus + serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load " - -- job: Build_Gold_Win - displayName: 9- Build Gold Win + +# GROUP 11 + +- job: Trends_Database_gold_lin + displayName: 11- Trends Database gold-lin + pool: server + timeoutInMinutes: 120 + dependsOn: [Frameworks_gold_lin, Crossgen_intel_lin] + condition: succeededOrFailed() + steps: + - template: trend-database-scenarios.yml + parameters: + connection: ASPNET Benchmarks Service Bus + serviceBusQueueName: citrine1 + serviceBusNamespace: aspnetbenchmarks + arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load --profile gold-db-db " + +- job: Build_intel_lin + displayName: 11- Build intel-lin pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Database_Gold_Lin, Trends_Database_Intel_Lin, WebSockets_Intel_Win, Crossgen_Gold_Win] + dependsOn: [Frameworks_gold_lin, Crossgen_intel_lin] condition: succeededOrFailed() steps: - template: build-perf-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus - serviceBusQueueName: mono + serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-win-app " - -# GROUP 10 + arguments: "$(ciProfile) --profile intel-lin-app " + +# GROUP 12 -- job: Custom_Proxies_Gold_Lin - displayName: 10- Custom Proxies Gold Lin +- job: Crossgen_amd_lin2 + displayName: 12- Crossgen amd-lin2 pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Database_Amd_Lin2, WebSockets_Gold_Lin, WebSockets_Intel_Lin, Build_Gold_Win] + dependsOn: [Trends_Database_gold_lin, Build_intel_lin] condition: succeededOrFailed() steps: - - template: proxies-custom-scenarios.yml + - template: crossgen2-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load --profile gold-db-db " - -- job: Custom_Proxies_Intel_Lin - displayName: 10- Custom Proxies Intel Lin + arguments: "$(ciProfile) --profile amd-lin2-app --profile gold-load-load " + +# GROUP 13 + +- job: Custom_Proxies_gold_lin + displayName: 13- Custom Proxies gold-lin pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Database_Amd_Lin2, WebSockets_Gold_Lin, WebSockets_Intel_Lin, Build_Gold_Win] + dependsOn: [Crossgen_amd_lin2] condition: succeededOrFailed() steps: - template: proxies-custom-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus - serviceBusQueueName: citrine2 + serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile intel-db-db " - + arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load --profile gold-db-db " + diff --git a/build/benchmarks-ci-02.yml b/build/benchmarks-ci-02.yml index a833e5cbe..f9ee4c37b 100644 --- a/build/benchmarks-ci-02.yml +++ b/build/benchmarks-ci-02.yml @@ -1,16 +1,14 @@ -# Do not change this file, it is generated using these steps: -# - The file benchmarks*.json defines how each pipeline set of jobs is run in parallel -# - Update the associated benchmarks*.json file with machine and scenario updates -# - Install python and install the requirements for the crank-scheduler in benchmarks/scripts/crank-scheduler/requirements.txt -# - Run the scheduler specifying the desired benchmarks*.json file, this template, and benchmarks/output to automatically overwrite the current pipeline. -# - Ex. python ./scripts/crank-scheduler/main.py --config ./build/benchmarks_ci.json --template ./build/benchmarks.template.liquid --yaml-output ./build +# Do not change this file, it is generated by the pod-scheduler. +# Source of truth: see ../scripts/pod-scheduler/README.md +# To regenerate, run from the repo root: +# python ./scripts/pod-scheduler/main.py --config ./build/benchmarks_ci_pods.json --base-name benchmarks-ci --yaml-output ./build trigger: none pr: none schedules: - cron: "0 9/12 * * *" - always: true # always run the pipeline even if there have not been source code changes since the last successful scheduled run. + always: true branches: include: - main @@ -32,10 +30,10 @@ jobs: # GROUP 1 -- job: Containers_Gold_Lin - displayName: 1- Containers Gold Lin +- job: Containers_gold_lin + displayName: 1- Containers gold-lin pool: server - timeoutInMinutes: 120 + timeoutInMinutes: 180 dependsOn: [] condition: succeededOrFailed() steps: @@ -45,11 +43,11 @@ jobs: serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load --profile gold-db-db " - -- job: Containers_Intel_Lin - displayName: 1- Containers Intel Lin + +- job: Containers_intel_lin + displayName: 1- Containers intel-lin pool: server - timeoutInMinutes: 120 + timeoutInMinutes: 180 dependsOn: [] condition: succeededOrFailed() steps: @@ -59,9 +57,9 @@ jobs: serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile intel-db-db " - -- job: Blazor_Intel_Perflin - displayName: 1- Blazor Intel Perflin + +- job: Blazor_intel_perflin + displayName: 1- Blazor intel-perflin pool: server timeoutInMinutes: 120 dependsOn: [] @@ -73,9 +71,9 @@ jobs: serviceBusQueueName: citrine3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-perflin-app --profile intel-perfload-load " - -- job: HttpClient_Gold_Win - displayName: 1- HttpClient Gold Win + +- job: HttpClient_gold_win + displayName: 1- HttpClient gold-win pool: server timeoutInMinutes: 120 dependsOn: [] @@ -87,14 +85,14 @@ jobs: serviceBusQueueName: mono serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load " - + # GROUP 2 -- job: PGO_Gold_Lin - displayName: 2- PGO Gold Lin +- job: PGO_gold_lin + displayName: 2- PGO gold-lin pool: server - timeoutInMinutes: 120 - dependsOn: [Containers_Gold_Lin, Containers_Intel_Lin, Blazor_Intel_Perflin, HttpClient_Gold_Win] + timeoutInMinutes: 180 + dependsOn: [Containers_gold_lin, Containers_intel_lin, Blazor_intel_perflin, HttpClient_gold_win] condition: succeededOrFailed() steps: - template: pgo-scenarios.yml @@ -103,12 +101,12 @@ jobs: serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load --profile gold-db-db " - -- job: PGO_Intel_Lin - displayName: 2- PGO Intel Lin + +- job: PGO_intel_lin + displayName: 2- PGO intel-lin pool: server - timeoutInMinutes: 120 - dependsOn: [Containers_Gold_Lin, Containers_Intel_Lin, Blazor_Intel_Perflin, HttpClient_Gold_Win] + timeoutInMinutes: 180 + dependsOn: [Containers_gold_lin, Containers_intel_lin, Blazor_intel_perflin, HttpClient_gold_win] condition: succeededOrFailed() steps: - template: pgo-scenarios.yml @@ -117,12 +115,12 @@ jobs: serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile intel-db-db " - -- job: HttpClient_Intel_Win - displayName: 2- HttpClient Intel Win + +- job: HttpClient_intel_win + displayName: 2- HttpClient intel-win pool: server timeoutInMinutes: 120 - dependsOn: [Containers_Gold_Lin, Containers_Intel_Lin, Blazor_Intel_Perflin, HttpClient_Gold_Win] + dependsOn: [Containers_gold_lin, Containers_intel_lin, Blazor_intel_perflin, HttpClient_gold_win] condition: succeededOrFailed() steps: - template: httpclient-scenarios.yml @@ -131,12 +129,12 @@ jobs: serviceBusQueueName: citrine3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load " - -- job: Baselines_Gold_Win - displayName: 2- Baselines Gold Win + +- job: Baselines_gold_win + displayName: 2- Baselines gold-win pool: server timeoutInMinutes: 120 - dependsOn: [Containers_Gold_Lin, Containers_Intel_Lin, Blazor_Intel_Perflin, HttpClient_Gold_Win] + dependsOn: [Containers_gold_lin, Containers_intel_lin, Blazor_intel_perflin, HttpClient_gold_win] condition: succeededOrFailed() steps: - template: baselines-scenarios.yml @@ -145,14 +143,14 @@ jobs: serviceBusQueueName: mono serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load " - + # GROUP 3 -- job: Baselines_Database_Gold_Lin - displayName: 3- Baselines Database Gold Lin +- job: Baselines_Database_amd_lin2 + displayName: 3- Baselines Database amd-lin2 pool: server timeoutInMinutes: 120 - dependsOn: [PGO_Gold_Lin, PGO_Intel_Lin, HttpClient_Intel_Win, Baselines_Gold_Win] + dependsOn: [PGO_gold_lin, PGO_intel_lin, HttpClient_intel_win, Baselines_gold_win] condition: succeededOrFailed() steps: - template: baselines-database-scenarios.yml @@ -160,13 +158,13 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load --profile gold-db-db " - -- job: Baselines_Database_Intel_Lin - displayName: 3- Baselines Database Intel Lin + arguments: "$(ciProfile) --profile amd-lin2-app --profile gold-load-load --profile gold-db-db " + +- job: Baselines_Database_intel_lin + displayName: 3- Baselines Database intel-lin pool: server timeoutInMinutes: 120 - dependsOn: [PGO_Gold_Lin, PGO_Intel_Lin, HttpClient_Intel_Win, Baselines_Gold_Win] + dependsOn: [PGO_gold_lin, PGO_intel_lin, HttpClient_intel_win, Baselines_gold_win] condition: succeededOrFailed() steps: - template: baselines-database-scenarios.yml @@ -175,12 +173,12 @@ jobs: serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile intel-db-db " - -- job: Baselines_Intel_Win - displayName: 3- Baselines Intel Win + +- job: Baselines_intel_win + displayName: 3- Baselines intel-win pool: server timeoutInMinutes: 120 - dependsOn: [PGO_Gold_Lin, PGO_Intel_Lin, HttpClient_Intel_Win, Baselines_Gold_Win] + dependsOn: [PGO_gold_lin, PGO_intel_lin, HttpClient_intel_win, Baselines_gold_win] condition: succeededOrFailed() steps: - template: baselines-scenarios.yml @@ -189,28 +187,28 @@ jobs: serviceBusQueueName: citrine3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load " - -- job: Baselines_Amd_Lin2 - displayName: 3- Baselines Amd Lin2 + +- job: SignalR_gold_win + displayName: 3- SignalR gold-win pool: server timeoutInMinutes: 120 - dependsOn: [PGO_Gold_Lin, PGO_Intel_Lin, HttpClient_Intel_Win, Baselines_Gold_Win] + dependsOn: [PGO_gold_lin, PGO_intel_lin, HttpClient_intel_win, Baselines_gold_win] condition: succeededOrFailed() steps: - - template: baselines-scenarios.yml + - template: signalr-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: mono serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile amd-lin2-app --profile gold-load2-load " - + arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load " + # GROUP 4 -- job: Baselines_Database_Gold_Win - displayName: 4- Baselines Database Gold Win +- job: Baselines_Database_gold_lin + displayName: 4- Baselines Database gold-lin pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Gold_Lin, Baselines_Database_Intel_Lin, Baselines_Intel_Win, Baselines_Amd_Lin2] + dependsOn: [Baselines_Database_amd_lin2, Baselines_Database_intel_lin, Baselines_intel_win, SignalR_gold_win] condition: succeededOrFailed() steps: - template: baselines-database-scenarios.yml @@ -218,13 +216,13 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load --profile gold-db-db " - -- job: Baselines_Database_Intel_Win - displayName: 4- Baselines Database Intel Win + arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load --profile gold-db-db " + +- job: Baselines_Database_intel_win + displayName: 4- Baselines Database intel-win pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Gold_Lin, Baselines_Database_Intel_Lin, Baselines_Intel_Win, Baselines_Amd_Lin2] + dependsOn: [Baselines_Database_amd_lin2, Baselines_Database_intel_lin, Baselines_intel_win, SignalR_gold_win] condition: succeededOrFailed() steps: - template: baselines-database-scenarios.yml @@ -233,12 +231,12 @@ jobs: serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load --profile intel-db-db " - -- job: Blazor_Gold_Lin - displayName: 4- Blazor Gold Lin + +- job: Blazor_intel_lin + displayName: 4- Blazor intel-lin pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Gold_Lin, Baselines_Database_Intel_Lin, Baselines_Intel_Win, Baselines_Amd_Lin2] + dependsOn: [Baselines_Database_amd_lin2, Baselines_Database_intel_lin, Baselines_intel_win, SignalR_gold_win] condition: succeededOrFailed() steps: - template: blazor-scenarios.yml @@ -246,43 +244,43 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine3 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load " - -- job: Blazor_Intel_Lin - displayName: 4- Blazor Intel Lin + arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load " + +- job: Trends_gold_win + displayName: 4- Trends gold-win pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Gold_Lin, Baselines_Database_Intel_Lin, Baselines_Intel_Win, Baselines_Amd_Lin2] + dependsOn: [Baselines_Database_amd_lin2, Baselines_Database_intel_lin, Baselines_intel_win, SignalR_gold_win] condition: succeededOrFailed() steps: - - template: blazor-scenarios.yml + - template: trend-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: mono serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load " - + arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load " + # GROUP 5 -- job: SslStream_Gold_Lin - displayName: 5- SslStream Gold Lin +- job: HttpClient_gold_lin + displayName: 5- HttpClient gold-lin pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Gold_Win, Baselines_Database_Intel_Win, Blazor_Gold_Lin, Blazor_Intel_Lin] + dependsOn: [Baselines_Database_gold_lin, Baselines_Database_intel_win, Blazor_intel_lin, Trends_gold_win] condition: succeededOrFailed() steps: - - template: sslstream-scenarios.yml + - template: httpclient-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load " - -- job: SslStream_Intel_Lin - displayName: 5- SslStream Intel Lin + +- job: SslStream_intel_lin + displayName: 5- SslStream intel-lin pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Gold_Win, Baselines_Database_Intel_Win, Blazor_Gold_Lin, Blazor_Intel_Lin] + dependsOn: [Baselines_Database_gold_lin, Baselines_Database_intel_win, Blazor_intel_lin, Trends_gold_win] condition: succeededOrFailed() steps: - template: sslstream-scenarios.yml @@ -291,26 +289,26 @@ jobs: serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load " - -- job: SignalR_Gold_Win - displayName: 5- SignalR Gold Win + +- job: MVC_gold_win + displayName: 5- MVC gold-win pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Gold_Win, Baselines_Database_Intel_Win, Blazor_Gold_Lin, Blazor_Intel_Lin] + dependsOn: [Baselines_Database_gold_lin, Baselines_Database_intel_win, Blazor_intel_lin, Trends_gold_win] condition: succeededOrFailed() steps: - - template: signalr-scenarios.yml + - template: mvc-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine3 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load " - -- job: MVC_Intel_Win - displayName: 5- MVC Intel Win + arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load --profile gold-db-db " + +- job: MVC_intel_win + displayName: 5- MVC intel-win pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Gold_Win, Baselines_Database_Intel_Win, Blazor_Gold_Lin, Blazor_Intel_Lin] + dependsOn: [Baselines_Database_gold_lin, Baselines_Database_intel_win, Blazor_intel_lin, Trends_gold_win] condition: succeededOrFailed() steps: - template: mvc-scenarios.yml @@ -319,42 +317,42 @@ jobs: serviceBusQueueName: mono serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load --profile intel-db-db " - + # GROUP 6 -- job: Frameworks_Database_Gold_Lin - displayName: 6- Frameworks Database Gold Lin +- job: Baselines_amd_lin2 + displayName: 6- Baselines amd-lin2 pool: server timeoutInMinutes: 120 - dependsOn: [SslStream_Gold_Lin, SslStream_Intel_Lin, SignalR_Gold_Win, MVC_Intel_Win] + dependsOn: [HttpClient_gold_lin, SslStream_intel_lin, MVC_gold_win, MVC_intel_win] condition: succeededOrFailed() steps: - - template: frameworks-database-scenarios.yml + - template: baselines-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load --profile gold-db-db " - -- job: Frameworks_Database_Intel_Lin - displayName: 6- Frameworks Database Intel Lin + arguments: "$(ciProfile) --profile amd-lin2-app --profile gold-load-load " + +- job: SignalR_intel_lin + displayName: 6- SignalR intel-lin pool: server timeoutInMinutes: 120 - dependsOn: [SslStream_Gold_Lin, SslStream_Intel_Lin, SignalR_Gold_Win, MVC_Intel_Win] + dependsOn: [HttpClient_gold_lin, SslStream_intel_lin, MVC_gold_win, MVC_intel_win] condition: succeededOrFailed() steps: - - template: frameworks-database-scenarios.yml + - template: signalr-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile intel-db-db " - -- job: Trends_Gold_Win - displayName: 6- Trends Gold Win + arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load " + +- job: Trends_intel_win + displayName: 6- Trends intel-win pool: server timeoutInMinutes: 120 - dependsOn: [SslStream_Gold_Lin, SslStream_Intel_Lin, SignalR_Gold_Win, MVC_Intel_Win] + dependsOn: [HttpClient_gold_lin, SslStream_intel_lin, MVC_gold_win, MVC_intel_win] condition: succeededOrFailed() steps: - template: trend-scenarios.yml @@ -362,13 +360,13 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine3 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load " - -- job: EF_Core_Intel_Win - displayName: 6- EF Core Intel Win + arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load " + +- job: EF_Core_gold_win + displayName: 6- EF Core gold-win pool: server timeoutInMinutes: 120 - dependsOn: [SslStream_Gold_Lin, SslStream_Intel_Lin, SignalR_Gold_Win, MVC_Intel_Win] + dependsOn: [HttpClient_gold_lin, SslStream_intel_lin, MVC_gold_win, MVC_intel_win] condition: succeededOrFailed() steps: - template: efcore-scenarios.yml @@ -376,29 +374,29 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: mono serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load " - + arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load " + # GROUP 7 -- job: MVC_Gold_Lin - displayName: 7- MVC Gold Lin +- job: SignalR_gold_lin + displayName: 7- SignalR gold-lin pool: server timeoutInMinutes: 120 - dependsOn: [Frameworks_Database_Gold_Lin, Frameworks_Database_Intel_Lin, Trends_Gold_Win, EF_Core_Intel_Win] + dependsOn: [Baselines_amd_lin2, SignalR_intel_lin, Trends_intel_win, EF_Core_gold_win] condition: succeededOrFailed() steps: - - template: mvc-scenarios.yml + - template: signalr-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load --profile gold-db-db " - -- job: MVC_Intel_Lin - displayName: 7- MVC Intel Lin + arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load " + +- job: MVC_intel_lin + displayName: 7- MVC intel-lin pool: server timeoutInMinutes: 120 - dependsOn: [Frameworks_Database_Gold_Lin, Frameworks_Database_Intel_Lin, Trends_Gold_Win, EF_Core_Intel_Win] + dependsOn: [Baselines_amd_lin2, SignalR_intel_lin, Trends_intel_win, EF_Core_gold_win] condition: succeededOrFailed() steps: - template: mvc-scenarios.yml @@ -407,56 +405,56 @@ jobs: serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile intel-db-db " - -- job: Trends_Amd_Lin2 - displayName: 7- Trends Amd Lin2 + +- job: GC_intel_win + displayName: 7- GC intel-win pool: server timeoutInMinutes: 120 - dependsOn: [Frameworks_Database_Gold_Lin, Frameworks_Database_Intel_Lin, Trends_Gold_Win, EF_Core_Intel_Win] + dependsOn: [Baselines_amd_lin2, SignalR_intel_lin, Trends_intel_win, EF_Core_gold_win] condition: succeededOrFailed() steps: - - template: trend-scenarios.yml + - template: gc-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine3 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile amd-lin2-app --profile gold-load2-load " - -- job: GC_Gold_Win - displayName: 7- GC Gold Win + arguments: "$(ciProfile) --profile intel-win-app " + +- job: Trends_Database_gold_win + displayName: 7- Trends Database gold-win pool: server timeoutInMinutes: 120 - dependsOn: [Frameworks_Database_Gold_Lin, Frameworks_Database_Intel_Lin, Trends_Gold_Win, EF_Core_Intel_Win] + dependsOn: [Baselines_amd_lin2, SignalR_intel_lin, Trends_intel_win, EF_Core_gold_win] condition: succeededOrFailed() steps: - - template: gc-scenarios.yml + - template: trend-database-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: mono serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-win-app " - + arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load --profile gold-db-db " + # GROUP 8 -- job: NativeAOT_Gold_Win - displayName: 8- NativeAOT Gold Win +- job: Frameworks_Database_gold_lin + displayName: 8- Frameworks Database gold-lin pool: server timeoutInMinutes: 120 - dependsOn: [MVC_Gold_Lin, MVC_Intel_Lin, Trends_Amd_Lin2, GC_Gold_Win] + dependsOn: [SignalR_gold_lin, MVC_intel_lin, GC_intel_win, Trends_Database_gold_win] condition: succeededOrFailed() steps: - - template: nativeaot-scenarios.yml + - template: frameworks-database-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load --profile gold-db-db " - -- job: Trends_Gold_Lin - displayName: 8- Trends Gold Lin + arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load --profile gold-db-db " + +- job: Trends_intel_lin + displayName: 8- Trends intel-lin pool: server timeoutInMinutes: 120 - dependsOn: [MVC_Gold_Lin, MVC_Intel_Lin, Trends_Amd_Lin2, GC_Gold_Win] + dependsOn: [SignalR_gold_lin, MVC_intel_lin, GC_intel_win, Trends_Database_gold_win] condition: succeededOrFailed() steps: - template: trend-scenarios.yml @@ -464,57 +462,57 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load " - -- job: Trends_Intel_Lin - displayName: 8- Trends Intel Lin + arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load " + +- job: Trends_Database_intel_win + displayName: 8- Trends Database intel-win pool: server timeoutInMinutes: 120 - dependsOn: [MVC_Gold_Lin, MVC_Intel_Lin, Trends_Amd_Lin2, GC_Gold_Win] + dependsOn: [SignalR_gold_lin, MVC_intel_lin, GC_intel_win, Trends_Database_gold_win] condition: succeededOrFailed() steps: - - template: trend-scenarios.yml + - template: trend-database-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine3 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load " - -- job: Trends_Database_Intel_Win - displayName: 8- Trends Database Intel Win + arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load --profile intel-db-db " + +- job: WebSockets_gold_win + displayName: 8- WebSockets gold-win pool: server timeoutInMinutes: 120 - dependsOn: [MVC_Gold_Lin, MVC_Intel_Lin, Trends_Amd_Lin2, GC_Gold_Win] + dependsOn: [SignalR_gold_lin, MVC_intel_lin, GC_intel_win, Trends_Database_gold_win] condition: succeededOrFailed() steps: - - template: trend-database-scenarios.yml + - template: websocket-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: mono serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load --profile intel-db-db " - + arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load " + # GROUP 9 -- job: Frameworks_Amd_Lin2 - displayName: 9- Frameworks Amd Lin2 +- job: NativeAOT_gold_lin + displayName: 9- NativeAOT gold-lin pool: server timeoutInMinutes: 120 - dependsOn: [NativeAOT_Gold_Win, Trends_Gold_Lin, Trends_Intel_Lin, Trends_Database_Intel_Win] + dependsOn: [Frameworks_Database_gold_lin, Trends_intel_lin, Trends_Database_intel_win, WebSockets_gold_win] condition: succeededOrFailed() steps: - - template: frameworks-scenarios.yml + - template: nativeaot-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile amd-lin2-app --profile gold-load-load --profile gold-db-db " - -- job: GC_Gold_Lin - displayName: 9- GC Gold Lin + arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load --profile gold-db-db " + +- job: GC_intel_lin + displayName: 9- GC intel-lin pool: server timeoutInMinutes: 120 - dependsOn: [NativeAOT_Gold_Win, Trends_Gold_Lin, Trends_Intel_Lin, Trends_Database_Intel_Win] + dependsOn: [Frameworks_Database_gold_lin, Trends_intel_lin, Trends_Database_intel_win, WebSockets_gold_win] condition: succeededOrFailed() steps: - template: gc-scenarios.yml @@ -522,179 +520,155 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-lin-app " - -- job: GC_Intel_Lin - displayName: 9- GC Intel Lin + arguments: "$(ciProfile) --profile intel-lin-app " + +- job: WebSockets_intel_win + displayName: 9- WebSockets intel-win pool: server timeoutInMinutes: 120 - dependsOn: [NativeAOT_Gold_Win, Trends_Gold_Lin, Trends_Intel_Lin, Trends_Database_Intel_Win] + dependsOn: [Frameworks_Database_gold_lin, Trends_intel_lin, Trends_Database_intel_win, WebSockets_gold_win] condition: succeededOrFailed() steps: - - template: gc-scenarios.yml + - template: websocket-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine3 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile intel-lin-app " - -- job: WebSockets_Gold_Win - displayName: 9- WebSockets Gold Win + arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load " + +- job: Build_gold_win + displayName: 9- Build gold-win pool: server timeoutInMinutes: 120 - dependsOn: [NativeAOT_Gold_Win, Trends_Gold_Lin, Trends_Intel_Lin, Trends_Database_Intel_Win] + dependsOn: [Frameworks_Database_gold_lin, Trends_intel_lin, Trends_Database_intel_win, WebSockets_gold_win] condition: succeededOrFailed() steps: - - template: websocket-scenarios.yml + - template: build-perf-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: mono serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load " - + arguments: "$(ciProfile) --profile gold-win-app " + # GROUP 10 -- job: Trends_Database_Gold_Win - displayName: 10- Trends Database Gold Win +- job: Trends_gold_lin + displayName: 10- Trends gold-lin pool: server timeoutInMinutes: 120 - dependsOn: [Frameworks_Amd_Lin2, GC_Gold_Lin, GC_Intel_Lin, WebSockets_Gold_Win] + dependsOn: [NativeAOT_gold_lin, GC_intel_lin, WebSockets_intel_win, Build_gold_win] condition: succeededOrFailed() steps: - - template: trend-database-scenarios.yml + - template: trend-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load --profile gold-db-db " - -- job: Single_File_Gold_Lin - displayName: 10- Single File Gold Lin - pool: server - timeoutInMinutes: 120 - dependsOn: [Frameworks_Amd_Lin2, GC_Gold_Lin, GC_Intel_Lin, WebSockets_Gold_Win] - condition: succeededOrFailed() - steps: - - template: singlefile-scenarios.yml - parameters: - connection: ASPNET Benchmarks Service Bus - serviceBusQueueName: citrine2 - serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load " - -- job: Single_File_Intel_Lin - displayName: 10- Single File Intel Lin + +- job: Single_File_intel_lin + displayName: 10- Single File intel-lin pool: server timeoutInMinutes: 120 - dependsOn: [Frameworks_Amd_Lin2, GC_Gold_Lin, GC_Intel_Lin, WebSockets_Gold_Win] + dependsOn: [NativeAOT_gold_lin, GC_intel_lin, WebSockets_intel_win, Build_gold_win] condition: succeededOrFailed() steps: - template: singlefile-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus - serviceBusQueueName: citrine3 + serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load " - -- job: Crossgen_Intel_Win - displayName: 10- Crossgen Intel Win + +- job: Build_intel_win + displayName: 10- Build intel-win pool: server timeoutInMinutes: 120 - dependsOn: [Frameworks_Amd_Lin2, GC_Gold_Lin, GC_Intel_Lin, WebSockets_Gold_Win] + dependsOn: [NativeAOT_gold_lin, GC_intel_lin, WebSockets_intel_win, Build_gold_win] condition: succeededOrFailed() steps: - - template: crossgen2-scenarios.yml + - template: build-perf-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus - serviceBusQueueName: mono + serviceBusQueueName: citrine3 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load " - + arguments: "$(ciProfile) --profile intel-win-app " + # GROUP 11 -- job: Crossgen_Gold_Lin - displayName: 11- Crossgen Gold Lin +- job: Trends_Database_amd_lin2 + displayName: 11- Trends Database amd-lin2 pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Database_Gold_Win, Single_File_Gold_Lin, Single_File_Intel_Lin, Crossgen_Intel_Win] + dependsOn: [Trends_gold_lin, Single_File_intel_lin, Build_intel_win] condition: succeededOrFailed() steps: - - template: crossgen2-scenarios.yml + - template: trend-database-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load " - -- job: Crossgen_Intel_Lin - displayName: 11- Crossgen Intel Lin + arguments: "$(ciProfile) --profile amd-lin2-app --profile gold-load-load --profile gold-db-db " + +- job: Custom_Proxies_intel_lin + displayName: 11- Custom Proxies intel-lin pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Database_Gold_Win, Single_File_Gold_Lin, Single_File_Intel_Lin, Crossgen_Intel_Win] + dependsOn: [Trends_gold_lin, Single_File_intel_lin, Build_intel_win] condition: succeededOrFailed() steps: - - template: crossgen2-scenarios.yml + - template: proxies-custom-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load " - -- job: Crossgen_Amd_Lin2 - displayName: 11- Crossgen Amd Lin2 - pool: server - timeoutInMinutes: 120 - dependsOn: [Trends_Database_Gold_Win, Single_File_Gold_Lin, Single_File_Intel_Lin, Crossgen_Intel_Win] - condition: succeededOrFailed() - steps: - - template: crossgen2-scenarios.yml - parameters: - connection: ASPNET Benchmarks Service Bus - serviceBusQueueName: citrine3 - serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile amd-lin2-app --profile gold-load2-load " - -- job: Build_Intel_Win - displayName: 11- Build Intel Win + arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile intel-db-db " + +# GROUP 12 + +- job: Single_File_gold_lin + displayName: 12- Single File gold-lin pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Database_Gold_Win, Single_File_Gold_Lin, Single_File_Intel_Lin, Crossgen_Intel_Win] + dependsOn: [Trends_Database_amd_lin2, Custom_Proxies_intel_lin] condition: succeededOrFailed() steps: - - template: build-perf-scenarios.yml + - template: singlefile-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus - serviceBusQueueName: mono + serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile intel-win-app " - -# GROUP 12 + arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load " + +# GROUP 13 -- job: Build_Gold_Lin - displayName: 12- Build Gold Lin +- job: WebSockets_gold_lin + displayName: 13- WebSockets gold-lin pool: server timeoutInMinutes: 120 - dependsOn: [Crossgen_Gold_Lin, Crossgen_Intel_Lin, Crossgen_Amd_Lin2, Build_Intel_Win] + dependsOn: [Single_File_gold_lin] condition: succeededOrFailed() steps: - - template: build-perf-scenarios.yml + - template: websocket-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-lin-app " - -- job: Build_Intel_Lin - displayName: 12- Build Intel Lin + arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load " + +# GROUP 14 + +- job: Crossgen_gold_lin + displayName: 14- Crossgen gold-lin pool: server timeoutInMinutes: 120 - dependsOn: [Crossgen_Gold_Lin, Crossgen_Intel_Lin, Crossgen_Amd_Lin2, Build_Intel_Win] + dependsOn: [WebSockets_gold_lin] condition: succeededOrFailed() steps: - - template: build-perf-scenarios.yml + - template: crossgen2-scenarios.yml parameters: connection: ASPNET Benchmarks Service Bus - serviceBusQueueName: citrine2 + serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile intel-lin-app " - + arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load " + diff --git a/build/benchmarks-ci-azure.yml b/build/benchmarks-ci-azure.yml index 868e76e43..2406d43af 100644 --- a/build/benchmarks-ci-azure.yml +++ b/build/benchmarks-ci-azure.yml @@ -1,16 +1,14 @@ -# Do not change this file, it is generated using these steps: -# - The file benchmarks*.json defines how each pipeline set of jobs is run in parallel -# - Update the associated benchmarks*.json file with machine and scenario updates -# - Install python and install the requirements for the crank-scheduler in benchmarks/scripts/crank-scheduler/requirements.txt -# - Run the scheduler specifying the desired benchmarks*.json file, this template, and benchmarks/output to automatically overwrite the current pipeline. -# - Ex. python ./scripts/crank-scheduler/main.py --config ./build/benchmarks_ci.json --template ./build/benchmarks.template.liquid --yaml-output ./build +# Do not change this file, it is generated by the pod-scheduler. +# Source of truth: see ../scripts/pod-scheduler/README.md +# To regenerate, run from the repo root: +# python ./scripts/pod-scheduler/main.py --config ./build/benchmarks_ci_azure_pods.json --base-name benchmarks-ci-azure --yaml-output ./build trigger: none pr: none schedules: - cron: "0 2/12 * * *" - always: true # always run the pipeline even if there have not been source code changes since the last successful scheduled run. + always: true branches: include: - main @@ -32,10 +30,10 @@ jobs: # GROUP 1 -- job: Containers_Azure_Server_Arm64 - displayName: 1- Containers Azure Server Arm64 +- job: Containers_azure_arm64 + displayName: 1- Containers azure-arm64 pool: server - timeoutInMinutes: 120 + timeoutInMinutes: 180 dependsOn: [] condition: succeededOrFailed() steps: @@ -45,11 +43,11 @@ jobs: serviceBusQueueName: azure serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile azure-server-arm64-app --profile azure-client-load --profile azure-db-db " - -- job: Containers_Azure2_Server_Amd64 - displayName: 1- Containers Azure2 Server Amd64 + +- job: Containers_azure2_amd64 + displayName: 1- Containers azure2-amd64 pool: server - timeoutInMinutes: 120 + timeoutInMinutes: 180 dependsOn: [] condition: succeededOrFailed() steps: @@ -59,14 +57,14 @@ jobs: serviceBusQueueName: azurearm64 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile azure2-server-amd64-app --profile azure2-client-load --profile azure2-db-db " - + # GROUP 2 -- job: Containers_Cobalt_Cloud_Lin_Server - displayName: 2- Containers Cobalt Cloud Lin Server +- job: Containers_cobalt_cloud_lin + displayName: 2- Containers cobalt-cloud-lin pool: server - timeoutInMinutes: 120 - dependsOn: [Containers_Azure_Server_Arm64, Containers_Azure2_Server_Amd64] + timeoutInMinutes: 180 + dependsOn: [Containers_azure_arm64, Containers_azure2_amd64] condition: succeededOrFailed() steps: - template: containers-scenarios.yml @@ -75,12 +73,12 @@ jobs: serviceBusQueueName: azure serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-app --profile cobalt-cloud-lin-client-load --profile cobalt-cloud-lin-db-db " - -- job: Baselines_Database_Azure_Server_Arm64 - displayName: 2- Baselines Database Azure Server Arm64 + +- job: Baselines_Database_azure_arm64 + displayName: 2- Baselines Database azure-arm64 pool: server timeoutInMinutes: 120 - dependsOn: [Containers_Azure_Server_Arm64, Containers_Azure2_Server_Amd64] + dependsOn: [Containers_azure_arm64, Containers_azure2_amd64] condition: succeededOrFailed() steps: - template: baselines-database-scenarios.yml @@ -89,14 +87,14 @@ jobs: serviceBusQueueName: azurearm64 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile azure-server-arm64-app --profile azure-client-load --profile azure-db-db " - + # GROUP 3 -- job: Containers_Cobalt_Cloud_Lin_Server_Azure_Linux3 - displayName: 3- Containers Cobalt Cloud Lin Server Azure Linux3 +- job: Containers_cobalt_cloud_lin_azl3 + displayName: 3- Containers cobalt-cloud-lin-azl3 pool: server - timeoutInMinutes: 120 - dependsOn: [Containers_Cobalt_Cloud_Lin_Server, Baselines_Database_Azure_Server_Arm64] + timeoutInMinutes: 180 + dependsOn: [Containers_cobalt_cloud_lin, Baselines_Database_azure_arm64] condition: succeededOrFailed() steps: - template: containers-scenarios.yml @@ -105,12 +103,12 @@ jobs: serviceBusQueueName: azure serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-azure-linux3-app --profile cobalt-cloud-lin-client-load --profile cobalt-cloud-lin-db-db " - -- job: Baselines_Database_Azure2_Server_Amd64 - displayName: 3- Baselines Database Azure2 Server Amd64 + +- job: Baselines_Database_azure2_amd64 + displayName: 3- Baselines Database azure2-amd64 pool: server timeoutInMinutes: 120 - dependsOn: [Containers_Cobalt_Cloud_Lin_Server, Baselines_Database_Azure_Server_Arm64] + dependsOn: [Containers_cobalt_cloud_lin, Baselines_Database_azure_arm64] condition: succeededOrFailed() steps: - template: baselines-database-scenarios.yml @@ -119,14 +117,14 @@ jobs: serviceBusQueueName: azurearm64 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile azure2-server-amd64-app --profile azure2-client-load --profile azure2-db-db " - + # GROUP 4 -- job: Baselines_Database_Cobalt_Cloud_Lin_Server - displayName: 4- Baselines Database Cobalt Cloud Lin Server +- job: Baselines_Database_cobalt_cloud_lin + displayName: 4- Baselines Database cobalt-cloud-lin pool: server timeoutInMinutes: 120 - dependsOn: [Containers_Cobalt_Cloud_Lin_Server_Azure_Linux3, Baselines_Database_Azure2_Server_Amd64] + dependsOn: [Containers_cobalt_cloud_lin_azl3, Baselines_Database_azure2_amd64] condition: succeededOrFailed() steps: - template: baselines-database-scenarios.yml @@ -135,12 +133,12 @@ jobs: serviceBusQueueName: azure serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-app --profile cobalt-cloud-lin-client-load --profile cobalt-cloud-lin-db-db " - -- job: Baselines_Azure_Server_Arm64 - displayName: 4- Baselines Azure Server Arm64 + +- job: Baselines_azure_arm64 + displayName: 4- Baselines azure-arm64 pool: server timeoutInMinutes: 120 - dependsOn: [Containers_Cobalt_Cloud_Lin_Server_Azure_Linux3, Baselines_Database_Azure2_Server_Amd64] + dependsOn: [Containers_cobalt_cloud_lin_azl3, Baselines_Database_azure2_amd64] condition: succeededOrFailed() steps: - template: baselines-scenarios.yml @@ -149,14 +147,14 @@ jobs: serviceBusQueueName: azurearm64 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile azure-server-arm64-app --profile azure-client-load " - + # GROUP 5 -- job: Baselines_Database_Cobalt_Cloud_Lin_Server_Azure_Linux3 - displayName: 5- Baselines Database Cobalt Cloud Lin Server Azure Linux3 +- job: Baselines_Database_cobalt_cloud_lin_azl3 + displayName: 5- Baselines Database cobalt-cloud-lin-azl3 pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Cobalt_Cloud_Lin_Server, Baselines_Azure_Server_Arm64] + dependsOn: [Baselines_Database_cobalt_cloud_lin, Baselines_azure_arm64] condition: succeededOrFailed() steps: - template: baselines-database-scenarios.yml @@ -165,12 +163,12 @@ jobs: serviceBusQueueName: azure serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-azure-linux3-app --profile cobalt-cloud-lin-client-load --profile cobalt-cloud-lin-db-db " - -- job: Baselines_Azure2_Server_Amd64 - displayName: 5- Baselines Azure2 Server Amd64 + +- job: Baselines_azure2_amd64 + displayName: 5- Baselines azure2-amd64 pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Cobalt_Cloud_Lin_Server, Baselines_Azure_Server_Arm64] + dependsOn: [Baselines_Database_cobalt_cloud_lin, Baselines_azure_arm64] condition: succeededOrFailed() steps: - template: baselines-scenarios.yml @@ -179,14 +177,14 @@ jobs: serviceBusQueueName: azurearm64 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile azure2-server-amd64-app --profile azure2-client-load " - + # GROUP 6 -- job: Baselines_Cobalt_Cloud_Lin_Server - displayName: 6- Baselines Cobalt Cloud Lin Server +- job: Baselines_cobalt_cloud_lin + displayName: 6- Baselines cobalt-cloud-lin pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Cobalt_Cloud_Lin_Server_Azure_Linux3, Baselines_Azure2_Server_Amd64] + dependsOn: [Baselines_Database_cobalt_cloud_lin_azl3, Baselines_azure2_amd64] condition: succeededOrFailed() steps: - template: baselines-scenarios.yml @@ -195,12 +193,12 @@ jobs: serviceBusQueueName: azure serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-app --profile cobalt-cloud-lin-client-load " - -- job: Baselines_Cobalt_Cloud_Lin_Server_Azure_Linux3 - displayName: 6- Baselines Cobalt Cloud Lin Server Azure Linux3 + +- job: Baselines_cobalt_cloud_lin_azl3_dual + displayName: 6- Baselines cobalt-cloud-lin-azl3-dual pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Cobalt_Cloud_Lin_Server_Azure_Linux3, Baselines_Azure2_Server_Amd64] + dependsOn: [Baselines_Database_cobalt_cloud_lin_azl3, Baselines_azure2_amd64] condition: succeededOrFailed() steps: - template: baselines-scenarios.yml @@ -209,14 +207,14 @@ jobs: serviceBusQueueName: azurearm64 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-azure-linux3-app --profile cobalt-cloud-lin-db-load " - + # GROUP 7 -- job: MVC_Cobalt_Cloud_Lin_Server - displayName: 7- MVC Cobalt Cloud Lin Server +- job: MVC_cobalt_cloud_lin + displayName: 7- MVC cobalt-cloud-lin pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Cobalt_Cloud_Lin_Server, Baselines_Cobalt_Cloud_Lin_Server_Azure_Linux3] + dependsOn: [Baselines_cobalt_cloud_lin, Baselines_cobalt_cloud_lin_azl3_dual] condition: succeededOrFailed() steps: - template: mvc-scenarios.yml @@ -225,12 +223,12 @@ jobs: serviceBusQueueName: azure serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-app --profile cobalt-cloud-lin-client-load --profile cobalt-cloud-lin-db-db " - -- job: Trends_Azure_Server_Arm64 - displayName: 7- Trends Azure Server Arm64 + +- job: Trends_azure_arm64 + displayName: 7- Trends azure-arm64 pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Cobalt_Cloud_Lin_Server, Baselines_Cobalt_Cloud_Lin_Server_Azure_Linux3] + dependsOn: [Baselines_cobalt_cloud_lin, Baselines_cobalt_cloud_lin_azl3_dual] condition: succeededOrFailed() steps: - template: trend-scenarios.yml @@ -239,14 +237,14 @@ jobs: serviceBusQueueName: azurearm64 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile azure-server-arm64-app --profile azure-client-load " - + # GROUP 8 -- job: MVC_Cobalt_Cloud_Lin_Server_Azure_Linux3 - displayName: 8- MVC Cobalt Cloud Lin Server Azure Linux3 +- job: MVC_cobalt_cloud_lin_azl3 + displayName: 8- MVC cobalt-cloud-lin-azl3 pool: server timeoutInMinutes: 120 - dependsOn: [MVC_Cobalt_Cloud_Lin_Server, Trends_Azure_Server_Arm64] + dependsOn: [MVC_cobalt_cloud_lin, Trends_azure_arm64] condition: succeededOrFailed() steps: - template: mvc-scenarios.yml @@ -255,12 +253,12 @@ jobs: serviceBusQueueName: azure serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-azure-linux3-app --profile cobalt-cloud-lin-client-load --profile cobalt-cloud-lin-db-db " - -- job: Trends_Azure2_Server_Amd64 - displayName: 8- Trends Azure2 Server Amd64 + +- job: Trends_azure2_amd64 + displayName: 8- Trends azure2-amd64 pool: server timeoutInMinutes: 120 - dependsOn: [MVC_Cobalt_Cloud_Lin_Server, Trends_Azure_Server_Arm64] + dependsOn: [MVC_cobalt_cloud_lin, Trends_azure_arm64] condition: succeededOrFailed() steps: - template: trend-scenarios.yml @@ -269,14 +267,14 @@ jobs: serviceBusQueueName: azurearm64 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile azure2-server-amd64-app --profile azure2-client-load " - + # GROUP 9 -- job: Trends_Cobalt_Cloud_Lin_Server - displayName: 9- Trends Cobalt Cloud Lin Server +- job: Trends_cobalt_cloud_lin + displayName: 9- Trends cobalt-cloud-lin pool: server timeoutInMinutes: 120 - dependsOn: [MVC_Cobalt_Cloud_Lin_Server_Azure_Linux3, Trends_Azure2_Server_Amd64] + dependsOn: [MVC_cobalt_cloud_lin_azl3, Trends_azure2_amd64] condition: succeededOrFailed() steps: - template: trend-scenarios.yml @@ -285,12 +283,12 @@ jobs: serviceBusQueueName: azure serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-app --profile cobalt-cloud-lin-client-load " - -- job: Trends_Cobalt_Cloud_Lin_Server_Azure_Linux3 - displayName: 9- Trends Cobalt Cloud Lin Server Azure Linux3 + +- job: Trends_cobalt_cloud_lin_azl3_dual + displayName: 9- Trends cobalt-cloud-lin-azl3-dual pool: server timeoutInMinutes: 120 - dependsOn: [MVC_Cobalt_Cloud_Lin_Server_Azure_Linux3, Trends_Azure2_Server_Amd64] + dependsOn: [MVC_cobalt_cloud_lin_azl3, Trends_azure2_amd64] condition: succeededOrFailed() steps: - template: trend-scenarios.yml @@ -299,14 +297,14 @@ jobs: serviceBusQueueName: azurearm64 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-azure-linux3-app --profile cobalt-cloud-lin-db-load " - + # GROUP 10 -- job: Trends_Idna_Amd_Lin - displayName: 10- Trends Idna Amd Lin +- job: Trends_idna_amd_lin + displayName: 10- Trends idna-amd-lin pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Cobalt_Cloud_Lin_Server, Trends_Cobalt_Cloud_Lin_Server_Azure_Linux3] + dependsOn: [Trends_cobalt_cloud_lin, Trends_cobalt_cloud_lin_azl3_dual] condition: succeededOrFailed() steps: - template: trend-scenarios.yml @@ -314,13 +312,13 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: azure serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile idna-amd-lin-app --profile idna-amd-win-load " - -- job: Trends_Idna_Intel_Lin - displayName: 10- Trends Idna Intel Lin + arguments: "$(ciProfile) --profile idna-amd-lin-app --profile idna-intel-lin-load " + +- job: Trends_idna_amd_win + displayName: 10- Trends idna-amd-win pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Cobalt_Cloud_Lin_Server, Trends_Cobalt_Cloud_Lin_Server_Azure_Linux3] + dependsOn: [Trends_cobalt_cloud_lin, Trends_cobalt_cloud_lin_azl3_dual] condition: succeededOrFailed() steps: - template: trend-scenarios.yml @@ -328,15 +326,15 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: azurearm64 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile idna-intel-lin-app --profile idna-intel-win-load " - + arguments: "$(ciProfile) --profile idna-amd-win-app --profile azure2-db-load " + # GROUP 11 -- job: Trends_Idna_Amd_Win - displayName: 11- Trends Idna Amd Win +- job: Trends_idna_intel_lin + displayName: 11- Trends idna-intel-lin pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Idna_Amd_Lin, Trends_Idna_Intel_Lin] + dependsOn: [Trends_idna_amd_lin, Trends_idna_amd_win] condition: succeededOrFailed() steps: - template: trend-scenarios.yml @@ -344,13 +342,13 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: azure serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile idna-amd-win-app --profile idna-intel-lin-load " - -- job: Trends_Idna_Intel_Win - displayName: 11- Trends Idna Intel Win + arguments: "$(ciProfile) --profile idna-intel-lin-app --profile idna-amd-lin-load " + +- job: Trends_idna_intel_win + displayName: 11- Trends idna-intel-win pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Idna_Amd_Lin, Trends_Idna_Intel_Lin] + dependsOn: [Trends_idna_amd_lin, Trends_idna_amd_win] condition: succeededOrFailed() steps: - template: trend-scenarios.yml @@ -358,15 +356,15 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: azurearm64 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile idna-intel-win-app --profile idna-amd-lin-load " - + arguments: "$(ciProfile) --profile idna-intel-win-app --profile azure2-db-load " + # GROUP 12 -- job: Trends_Database_Azure_Server_Arm64 - displayName: 12- Trends Database Azure Server Arm64 +- job: Trends_Database_azure_arm64 + displayName: 12- Trends Database azure-arm64 pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Idna_Amd_Win, Trends_Idna_Intel_Win] + dependsOn: [Trends_idna_intel_lin, Trends_idna_intel_win] condition: succeededOrFailed() steps: - template: trend-database-scenarios.yml @@ -375,12 +373,12 @@ jobs: serviceBusQueueName: azure serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile azure-server-arm64-app --profile azure-client-load --profile azure-db-db " - -- job: Trends_Database_Azure2_Server_Amd64 - displayName: 12- Trends Database Azure2 Server Amd64 + +- job: Trends_Database_azure2_amd64 + displayName: 12- Trends Database azure2-amd64 pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Idna_Amd_Win, Trends_Idna_Intel_Win] + dependsOn: [Trends_idna_intel_lin, Trends_idna_intel_win] condition: succeededOrFailed() steps: - template: trend-database-scenarios.yml @@ -389,14 +387,14 @@ jobs: serviceBusQueueName: azurearm64 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile azure2-server-amd64-app --profile azure2-client-load --profile azure2-db-db " - + # GROUP 13 -- job: Trends_Database_Cobalt_Cloud_Lin_Server - displayName: 13- Trends Database Cobalt Cloud Lin Server +- job: Trends_Database_cobalt_cloud_lin + displayName: 13- Trends Database cobalt-cloud-lin pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Database_Azure_Server_Arm64, Trends_Database_Azure2_Server_Amd64] + dependsOn: [Trends_Database_azure_arm64, Trends_Database_azure2_amd64] condition: succeededOrFailed() steps: - template: trend-database-scenarios.yml @@ -405,14 +403,14 @@ jobs: serviceBusQueueName: azure serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-app --profile cobalt-cloud-lin-client-load --profile cobalt-cloud-lin-db-db " - + # GROUP 14 -- job: Trends_Database_Cobalt_Cloud_Lin_Server_Azure_Linux3 - displayName: 14- Trends Database Cobalt Cloud Lin Server Azure Linux3 +- job: Trends_Database_cobalt_cloud_lin_azl3 + displayName: 14- Trends Database cobalt-cloud-lin-azl3 pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Database_Cobalt_Cloud_Lin_Server] + dependsOn: [Trends_Database_cobalt_cloud_lin] condition: succeededOrFailed() steps: - template: trend-database-scenarios.yml @@ -421,4 +419,4 @@ jobs: serviceBusQueueName: azure serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-azure-linux3-app --profile cobalt-cloud-lin-client-load --profile cobalt-cloud-lin-db-db " - + diff --git a/build/benchmarks-ci-cobalt.yml b/build/benchmarks-ci-cobalt.yml index 571c66798..1d0c2d0b5 100644 --- a/build/benchmarks-ci-cobalt.yml +++ b/build/benchmarks-ci-cobalt.yml @@ -1,15 +1,14 @@ -# Do not change this file, it is generated using these steps: -# - The file benchmarks.matrix.yml defines how each job is run in parallel -# - Convert its content to json using https://jsonformatter.org/yaml-to-json -# - Use the template in benchmarks.template.liquid and the converted json using https://liquidjs.com/playground.html -# - Update this file with the result of the template generation +# Do not change this file, it is generated by the pod-scheduler. +# Source of truth: see ../scripts/pod-scheduler/README.md +# To regenerate, run from the repo root: +# python ./scripts/pod-scheduler/main.py --config ./build/benchmarks_ci_cobalt_pods.json --base-name benchmarks-ci-cobalt --yaml-output ./build trigger: none pr: none schedules: - cron: "0 3/12 * * *" - always: true # always run the pipeline even if there have not been source code changes since the last successful scheduled run. + always: true branches: include: - main @@ -31,10 +30,10 @@ jobs: # GROUP 1 -- job: Containers_Cobalt_Hosted_Lin_Server - displayName: 1- Containers Cobalt Hosted Lin Server +- job: Containers_cobalt_hosted_lin + displayName: 1- Containers cobalt-hosted-lin pool: server - timeoutInMinutes: 120 + timeoutInMinutes: 180 dependsOn: [] condition: succeededOrFailed() steps: @@ -44,11 +43,11 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-app --profile cobalt-hosted-lin-client-load --profile cobalt-hosted-lin-db-db " - -- job: Containers_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 1- Containers Cobalt Hosted Lin Server Azure Linux3 + +- job: Containers_cobalt_hosted_lin_azl3 + displayName: 1- Containers cobalt-hosted-lin-azl3 pool: server - timeoutInMinutes: 120 + timeoutInMinutes: 180 dependsOn: [] condition: succeededOrFailed() steps: @@ -58,14 +57,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-app --profile cobalt-hosted-lin-client-azure-linux3-load --profile cobalt-hosted-lin-db-azure-linux3-db " - + # GROUP 2 -- job: PGO_Cobalt_Hosted_Lin_Server - displayName: 2- PGO Cobalt Hosted Lin Server +- job: PGO_cobalt_hosted_lin + displayName: 2- PGO cobalt-hosted-lin pool: server - timeoutInMinutes: 120 - dependsOn: [Containers_Cobalt_Hosted_Lin_Server, Containers_Cobalt_Hosted_Lin_Server_Azure_Linux3] + timeoutInMinutes: 180 + dependsOn: [Containers_cobalt_hosted_lin, Containers_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: pgo-scenarios.yml @@ -74,12 +73,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-app --profile cobalt-hosted-lin-client-load --profile cobalt-hosted-lin-db-db " - -- job: PGO_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 2- PGO Cobalt Hosted Lin Server Azure Linux3 + +- job: PGO_cobalt_hosted_lin_azl3 + displayName: 2- PGO cobalt-hosted-lin-azl3 pool: server - timeoutInMinutes: 120 - dependsOn: [Containers_Cobalt_Hosted_Lin_Server, Containers_Cobalt_Hosted_Lin_Server_Azure_Linux3] + timeoutInMinutes: 180 + dependsOn: [Containers_cobalt_hosted_lin, Containers_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: pgo-scenarios.yml @@ -88,14 +87,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-app --profile cobalt-hosted-lin-client-azure-linux3-load --profile cobalt-hosted-lin-db-azure-linux3-db " - + # GROUP 3 -- job: PGO_28_Core_Cobalt_Hosted_Lin_Server - displayName: 3- PGO 28 Core Cobalt Hosted Lin Server +- job: PGO_cobalt_hosted_lin_28 + displayName: 3- PGO cobalt-hosted-lin-28 pool: server - timeoutInMinutes: 120 - dependsOn: [PGO_Cobalt_Hosted_Lin_Server, PGO_Cobalt_Hosted_Lin_Server_Azure_Linux3] + timeoutInMinutes: 180 + dependsOn: [PGO_cobalt_hosted_lin, PGO_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: pgo-scenarios.yml @@ -104,12 +103,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-28-app --profile cobalt-hosted-lin-client-load --profile cobalt-hosted-lin-db-db " - -- job: PGO_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 3- PGO 28 Core Cobalt Hosted Lin Server Azure Linux3 + +- job: PGO_cobalt_hosted_lin_azl3_28 + displayName: 3- PGO cobalt-hosted-lin-azl3-28 pool: server - timeoutInMinutes: 120 - dependsOn: [PGO_Cobalt_Hosted_Lin_Server, PGO_Cobalt_Hosted_Lin_Server_Azure_Linux3] + timeoutInMinutes: 180 + dependsOn: [PGO_cobalt_hosted_lin, PGO_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: pgo-scenarios.yml @@ -118,14 +117,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-28-app --profile cobalt-hosted-lin-client-azure-linux3-load --profile cobalt-hosted-lin-db-azure-linux3-db " - + # GROUP 4 -- job: Baselines_Database_Cobalt_Hosted_Lin_Server - displayName: 4- Baselines Database Cobalt Hosted Lin Server +- job: Baselines_Database_cobalt_hosted_lin + displayName: 4- Baselines Database cobalt-hosted-lin pool: server timeoutInMinutes: 120 - dependsOn: [PGO_28_Core_Cobalt_Hosted_Lin_Server, PGO_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [PGO_cobalt_hosted_lin_28, PGO_cobalt_hosted_lin_azl3_28] condition: succeededOrFailed() steps: - template: baselines-database-scenarios.yml @@ -134,12 +133,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-app --profile cobalt-hosted-lin-client-load --profile cobalt-hosted-lin-db-db " - -- job: Baselines_Database_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 4- Baselines Database Cobalt Hosted Lin Server Azure Linux3 + +- job: Baselines_Database_cobalt_hosted_lin_azl3 + displayName: 4- Baselines Database cobalt-hosted-lin-azl3 pool: server timeoutInMinutes: 120 - dependsOn: [PGO_28_Core_Cobalt_Hosted_Lin_Server, PGO_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [PGO_cobalt_hosted_lin_28, PGO_cobalt_hosted_lin_azl3_28] condition: succeededOrFailed() steps: - template: baselines-database-scenarios.yml @@ -148,14 +147,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-app --profile cobalt-hosted-lin-client-azure-linux3-load --profile cobalt-hosted-lin-db-azure-linux3-db " - + # GROUP 5 -- job: Baselines_Database_28_Core_Cobalt_Hosted_Lin_Server - displayName: 5- Baselines Database 28 Core Cobalt Hosted Lin Server +- job: Baselines_Database_cobalt_hosted_lin_28 + displayName: 5- Baselines Database cobalt-hosted-lin-28 pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Cobalt_Hosted_Lin_Server, Baselines_Database_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Baselines_Database_cobalt_hosted_lin, Baselines_Database_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: baselines-database-scenarios.yml @@ -164,12 +163,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-28-app --profile cobalt-hosted-lin-client-load --profile cobalt-hosted-lin-db-db " - -- job: Baselines_Database_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 5- Baselines Database 28 Core Cobalt Hosted Lin Server Azure Linux3 + +- job: Baselines_Database_cobalt_hosted_lin_azl3_28 + displayName: 5- Baselines Database cobalt-hosted-lin-azl3-28 pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Cobalt_Hosted_Lin_Server, Baselines_Database_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Baselines_Database_cobalt_hosted_lin, Baselines_Database_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: baselines-database-scenarios.yml @@ -178,14 +177,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-28-app --profile cobalt-hosted-lin-client-azure-linux3-load --profile cobalt-hosted-lin-db-azure-linux3-db " - + # GROUP 6 -- job: Baselines_Cobalt_Hosted_Lin_Server - displayName: 6- Baselines Cobalt Hosted Lin Server +- job: Baselines_cobalt_hosted_lin + displayName: 6- Baselines cobalt-hosted-lin pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_28_Core_Cobalt_Hosted_Lin_Server, Baselines_Database_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Baselines_Database_cobalt_hosted_lin_28, Baselines_Database_cobalt_hosted_lin_azl3_28] condition: succeededOrFailed() steps: - template: baselines-scenarios.yml @@ -194,12 +193,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-app --profile cobalt-hosted-lin-client-load " - -- job: Baselines_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 6- Baselines Cobalt Hosted Lin Server Azure Linux3 + +- job: Baselines_cobalt_hosted_lin_azl3 + displayName: 6- Baselines cobalt-hosted-lin-azl3 pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Database_28_Core_Cobalt_Hosted_Lin_Server, Baselines_Database_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Baselines_Database_cobalt_hosted_lin_28, Baselines_Database_cobalt_hosted_lin_azl3_28] condition: succeededOrFailed() steps: - template: baselines-scenarios.yml @@ -208,14 +207,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-app --profile cobalt-hosted-lin-client-azure-linux3-load " - + # GROUP 7 -- job: Baselines_28_Core_Cobalt_Hosted_Lin_Server - displayName: 7- Baselines 28 Core Cobalt Hosted Lin Server +- job: Baselines_cobalt_hosted_lin_28 + displayName: 7- Baselines cobalt-hosted-lin-28 pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Cobalt_Hosted_Lin_Server, Baselines_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Baselines_cobalt_hosted_lin, Baselines_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: baselines-scenarios.yml @@ -224,12 +223,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-28-app --profile cobalt-hosted-lin-client-load " - -- job: Baselines_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 7- Baselines 28 Core Cobalt Hosted Lin Server Azure Linux3 + +- job: Baselines_cobalt_hosted_lin_azl3_28 + displayName: 7- Baselines cobalt-hosted-lin-azl3-28 pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_Cobalt_Hosted_Lin_Server, Baselines_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Baselines_cobalt_hosted_lin, Baselines_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: baselines-scenarios.yml @@ -238,14 +237,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-28-app --profile cobalt-hosted-lin-client-azure-linux3-load " - + # GROUP 8 -- job: Frameworks_Database_Cobalt_Hosted_Lin_Server - displayName: 8- Frameworks Database Cobalt Hosted Lin Server +- job: Frameworks_Database_cobalt_hosted_lin + displayName: 8- Frameworks Database cobalt-hosted-lin pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_28_Core_Cobalt_Hosted_Lin_Server, Baselines_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Baselines_cobalt_hosted_lin_28, Baselines_cobalt_hosted_lin_azl3_28] condition: succeededOrFailed() steps: - template: frameworks-database-scenarios.yml @@ -254,12 +253,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-app --profile cobalt-hosted-lin-client-load --profile cobalt-hosted-lin-db-db " - -- job: Frameworks_Database_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 8- Frameworks Database Cobalt Hosted Lin Server Azure Linux3 + +- job: Frameworks_Database_cobalt_hosted_lin_azl3 + displayName: 8- Frameworks Database cobalt-hosted-lin-azl3 pool: server timeoutInMinutes: 120 - dependsOn: [Baselines_28_Core_Cobalt_Hosted_Lin_Server, Baselines_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Baselines_cobalt_hosted_lin_28, Baselines_cobalt_hosted_lin_azl3_28] condition: succeededOrFailed() steps: - template: frameworks-database-scenarios.yml @@ -268,14 +267,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-app --profile cobalt-hosted-lin-client-azure-linux3-load --profile cobalt-hosted-lin-db-azure-linux3-db " - + # GROUP 9 -- job: Frameworks_Database_28_Core_Cobalt_Hosted_Lin_Server - displayName: 9- Frameworks Database 28 Core Cobalt Hosted Lin Server +- job: Frameworks_Database_cobalt_hosted_lin_28 + displayName: 9- Frameworks Database cobalt-hosted-lin-28 pool: server timeoutInMinutes: 120 - dependsOn: [Frameworks_Database_Cobalt_Hosted_Lin_Server, Frameworks_Database_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Frameworks_Database_cobalt_hosted_lin, Frameworks_Database_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: frameworks-database-scenarios.yml @@ -284,12 +283,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-28-app --profile cobalt-hosted-lin-client-load --profile cobalt-hosted-lin-db-db " - -- job: Frameworks_Database_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 9- Frameworks Database 28 Core Cobalt Hosted Lin Server Azure Linux3 + +- job: Frameworks_Database_cobalt_hosted_lin_azl3_28 + displayName: 9- Frameworks Database cobalt-hosted-lin-azl3-28 pool: server timeoutInMinutes: 120 - dependsOn: [Frameworks_Database_Cobalt_Hosted_Lin_Server, Frameworks_Database_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Frameworks_Database_cobalt_hosted_lin, Frameworks_Database_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: frameworks-database-scenarios.yml @@ -298,14 +297,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-28-app --profile cobalt-hosted-lin-client-azure-linux3-load --profile cobalt-hosted-lin-db-azure-linux3-db " - + # GROUP 10 -- job: MVC_Cobalt_Hosted_Lin_Server - displayName: 10- MVC Cobalt Hosted Lin Server +- job: MVC_cobalt_hosted_lin + displayName: 10- MVC cobalt-hosted-lin pool: server timeoutInMinutes: 120 - dependsOn: [Frameworks_Database_28_Core_Cobalt_Hosted_Lin_Server, Frameworks_Database_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Frameworks_Database_cobalt_hosted_lin_28, Frameworks_Database_cobalt_hosted_lin_azl3_28] condition: succeededOrFailed() steps: - template: mvc-scenarios.yml @@ -314,12 +313,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-app --profile cobalt-hosted-lin-client-load --profile cobalt-hosted-lin-db-db " - -- job: MVC_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 10- MVC Cobalt Hosted Lin Server Azure Linux3 + +- job: MVC_cobalt_hosted_lin_azl3 + displayName: 10- MVC cobalt-hosted-lin-azl3 pool: server timeoutInMinutes: 120 - dependsOn: [Frameworks_Database_28_Core_Cobalt_Hosted_Lin_Server, Frameworks_Database_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Frameworks_Database_cobalt_hosted_lin_28, Frameworks_Database_cobalt_hosted_lin_azl3_28] condition: succeededOrFailed() steps: - template: mvc-scenarios.yml @@ -328,14 +327,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-app --profile cobalt-hosted-lin-client-azure-linux3-load --profile cobalt-hosted-lin-db-azure-linux3-db " - + # GROUP 11 -- job: NativeAOT_Cobalt_Hosted_Lin_Server - displayName: 11- NativeAOT Cobalt Hosted Lin Server +- job: NativeAOT_cobalt_hosted_lin + displayName: 11- NativeAOT cobalt-hosted-lin pool: server timeoutInMinutes: 120 - dependsOn: [MVC_Cobalt_Hosted_Lin_Server, MVC_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [MVC_cobalt_hosted_lin, MVC_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: nativeaot-scenarios.yml @@ -344,12 +343,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-app --profile cobalt-hosted-lin-client-load --profile cobalt-hosted-lin-db-db " - -- job: NativeAOT_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 11- NativeAOT Cobalt Hosted Lin Server Azure Linux3 + +- job: NativeAOT_cobalt_hosted_lin_azl3 + displayName: 11- NativeAOT cobalt-hosted-lin-azl3 pool: server timeoutInMinutes: 120 - dependsOn: [MVC_Cobalt_Hosted_Lin_Server, MVC_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [MVC_cobalt_hosted_lin, MVC_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: nativeaot-scenarios.yml @@ -358,14 +357,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-app --profile cobalt-hosted-lin-client-azure-linux3-load --profile cobalt-hosted-lin-db-azure-linux3-db " - + # GROUP 12 -- job: NativeAOT_28_Core_Cobalt_Hosted_Lin_Server - displayName: 12- NativeAOT 28 Core Cobalt Hosted Lin Server +- job: NativeAOT_cobalt_hosted_lin_28 + displayName: 12- NativeAOT cobalt-hosted-lin-28 pool: server timeoutInMinutes: 120 - dependsOn: [NativeAOT_Cobalt_Hosted_Lin_Server, NativeAOT_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [NativeAOT_cobalt_hosted_lin, NativeAOT_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: nativeaot-scenarios.yml @@ -374,12 +373,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-28-app --profile cobalt-hosted-lin-client-load --profile cobalt-hosted-lin-db-db " - -- job: NativeAOT_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 12- NativeAOT 28 Core Cobalt Hosted Lin Server Azure Linux3 + +- job: NativeAOT_cobalt_hosted_lin_azl3_28 + displayName: 12- NativeAOT cobalt-hosted-lin-azl3-28 pool: server timeoutInMinutes: 120 - dependsOn: [NativeAOT_Cobalt_Hosted_Lin_Server, NativeAOT_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [NativeAOT_cobalt_hosted_lin, NativeAOT_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: nativeaot-scenarios.yml @@ -388,14 +387,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-28-app --profile cobalt-hosted-lin-client-azure-linux3-load --profile cobalt-hosted-lin-db-azure-linux3-db " - + # GROUP 13 -- job: Trends_Cobalt_Hosted_Lin_Server - displayName: 13- Trends Cobalt Hosted Lin Server +- job: Trends_cobalt_hosted_lin + displayName: 13- Trends cobalt-hosted-lin pool: server timeoutInMinutes: 120 - dependsOn: [NativeAOT_28_Core_Cobalt_Hosted_Lin_Server, NativeAOT_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [NativeAOT_cobalt_hosted_lin_28, NativeAOT_cobalt_hosted_lin_azl3_28] condition: succeededOrFailed() steps: - template: trend-scenarios.yml @@ -404,12 +403,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-app --profile cobalt-hosted-lin-client-load " - -- job: Trends_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 13- Trends Cobalt Hosted Lin Server Azure Linux3 + +- job: Trends_cobalt_hosted_lin_azl3 + displayName: 13- Trends cobalt-hosted-lin-azl3 pool: server timeoutInMinutes: 120 - dependsOn: [NativeAOT_28_Core_Cobalt_Hosted_Lin_Server, NativeAOT_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [NativeAOT_cobalt_hosted_lin_28, NativeAOT_cobalt_hosted_lin_azl3_28] condition: succeededOrFailed() steps: - template: trend-scenarios.yml @@ -418,14 +417,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-app --profile cobalt-hosted-lin-client-azure-linux3-load " - + # GROUP 14 -- job: Trends_28_Core_Cobalt_Hosted_Lin_Server - displayName: 14- Trends 28 Core Cobalt Hosted Lin Server +- job: Trends_cobalt_hosted_lin_28 + displayName: 14- Trends cobalt-hosted-lin-28 pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Cobalt_Hosted_Lin_Server, Trends_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Trends_cobalt_hosted_lin, Trends_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: trend-scenarios.yml @@ -434,12 +433,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-28-app --profile cobalt-hosted-lin-client-load " - -- job: Trends_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 14- Trends 28 Core Cobalt Hosted Lin Server Azure Linux3 + +- job: Trends_cobalt_hosted_lin_azl3_28 + displayName: 14- Trends cobalt-hosted-lin-azl3-28 pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Cobalt_Hosted_Lin_Server, Trends_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Trends_cobalt_hosted_lin, Trends_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: trend-scenarios.yml @@ -448,14 +447,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-28-app --profile cobalt-hosted-lin-client-azure-linux3-load " - + # GROUP 15 -- job: Frameworks_Cobalt_Hosted_Lin_Server - displayName: 15- Frameworks Cobalt Hosted Lin Server +- job: Frameworks_cobalt_hosted_lin + displayName: 15- Frameworks cobalt-hosted-lin pool: server timeoutInMinutes: 120 - dependsOn: [Trends_28_Core_Cobalt_Hosted_Lin_Server, Trends_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Trends_cobalt_hosted_lin_28, Trends_cobalt_hosted_lin_azl3_28] condition: succeededOrFailed() steps: - template: frameworks-scenarios.yml @@ -464,12 +463,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-app --profile cobalt-hosted-lin-client-load --profile cobalt-hosted-lin-db-db " - -- job: Frameworks_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 15- Frameworks Cobalt Hosted Lin Server Azure Linux3 + +- job: Frameworks_cobalt_hosted_lin_azl3 + displayName: 15- Frameworks cobalt-hosted-lin-azl3 pool: server timeoutInMinutes: 120 - dependsOn: [Trends_28_Core_Cobalt_Hosted_Lin_Server, Trends_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Trends_cobalt_hosted_lin_28, Trends_cobalt_hosted_lin_azl3_28] condition: succeededOrFailed() steps: - template: frameworks-scenarios.yml @@ -478,14 +477,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-app --profile cobalt-hosted-lin-client-azure-linux3-load --profile cobalt-hosted-lin-db-azure-linux3-db " - + # GROUP 16 -- job: Frameworks_28_Core_Cobalt_Hosted_Lin_Server - displayName: 16- Frameworks 28 Core Cobalt Hosted Lin Server +- job: Frameworks_cobalt_hosted_lin_28 + displayName: 16- Frameworks cobalt-hosted-lin-28 pool: server timeoutInMinutes: 120 - dependsOn: [Frameworks_Cobalt_Hosted_Lin_Server, Frameworks_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Frameworks_cobalt_hosted_lin, Frameworks_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: frameworks-scenarios.yml @@ -494,12 +493,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-28-app --profile cobalt-hosted-lin-client-load --profile cobalt-hosted-lin-db-db " - -- job: Frameworks_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 16- Frameworks 28 Core Cobalt Hosted Lin Server Azure Linux3 + +- job: Frameworks_cobalt_hosted_lin_azl3_28 + displayName: 16- Frameworks cobalt-hosted-lin-azl3-28 pool: server timeoutInMinutes: 120 - dependsOn: [Frameworks_Cobalt_Hosted_Lin_Server, Frameworks_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Frameworks_cobalt_hosted_lin, Frameworks_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: frameworks-scenarios.yml @@ -508,14 +507,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-28-app --profile cobalt-hosted-lin-client-azure-linux3-load --profile cobalt-hosted-lin-db-azure-linux3-db " - + # GROUP 17 -- job: GC_Cobalt_Hosted_Lin_Server - displayName: 17- GC Cobalt Hosted Lin Server +- job: GC_cobalt_hosted_lin + displayName: 17- GC cobalt-hosted-lin pool: server timeoutInMinutes: 120 - dependsOn: [Frameworks_28_Core_Cobalt_Hosted_Lin_Server, Frameworks_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Frameworks_cobalt_hosted_lin_28, Frameworks_cobalt_hosted_lin_azl3_28] condition: succeededOrFailed() steps: - template: gc-scenarios.yml @@ -524,12 +523,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-app " - -- job: GC_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 17- GC Cobalt Hosted Lin Server Azure Linux3 + +- job: GC_cobalt_hosted_lin_azl3 + displayName: 17- GC cobalt-hosted-lin-azl3 pool: server timeoutInMinutes: 120 - dependsOn: [Frameworks_28_Core_Cobalt_Hosted_Lin_Server, Frameworks_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Frameworks_cobalt_hosted_lin_28, Frameworks_cobalt_hosted_lin_azl3_28] condition: succeededOrFailed() steps: - template: gc-scenarios.yml @@ -538,14 +537,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-app " - + # GROUP 18 -- job: GC_28_Core_Cobalt_Hosted_Lin_Server - displayName: 18- GC 28 Core Cobalt Hosted Lin Server +- job: GC_cobalt_hosted_lin_28 + displayName: 18- GC cobalt-hosted-lin-28 pool: server timeoutInMinutes: 120 - dependsOn: [GC_Cobalt_Hosted_Lin_Server, GC_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [GC_cobalt_hosted_lin, GC_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: gc-scenarios.yml @@ -554,12 +553,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-28-app " - -- job: GC_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 18- GC 28 Core Cobalt Hosted Lin Server Azure Linux3 + +- job: GC_cobalt_hosted_lin_azl3_28 + displayName: 18- GC cobalt-hosted-lin-azl3-28 pool: server timeoutInMinutes: 120 - dependsOn: [GC_Cobalt_Hosted_Lin_Server, GC_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [GC_cobalt_hosted_lin, GC_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: gc-scenarios.yml @@ -568,14 +567,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-28-app " - + # GROUP 19 -- job: Trends_Database_Cobalt_Hosted_Lin_Server - displayName: 19- Trends Database Cobalt Hosted Lin Server +- job: Trends_Database_cobalt_hosted_lin + displayName: 19- Trends Database cobalt-hosted-lin pool: server timeoutInMinutes: 120 - dependsOn: [GC_28_Core_Cobalt_Hosted_Lin_Server, GC_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [GC_cobalt_hosted_lin_28, GC_cobalt_hosted_lin_azl3_28] condition: succeededOrFailed() steps: - template: trend-database-scenarios.yml @@ -584,12 +583,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-app --profile cobalt-hosted-lin-client-load --profile cobalt-hosted-lin-db-db " - -- job: Trends_Database_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 19- Trends Database Cobalt Hosted Lin Server Azure Linux3 + +- job: Trends_Database_cobalt_hosted_lin_azl3 + displayName: 19- Trends Database cobalt-hosted-lin-azl3 pool: server timeoutInMinutes: 120 - dependsOn: [GC_28_Core_Cobalt_Hosted_Lin_Server, GC_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [GC_cobalt_hosted_lin_28, GC_cobalt_hosted_lin_azl3_28] condition: succeededOrFailed() steps: - template: trend-database-scenarios.yml @@ -598,14 +597,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-app --profile cobalt-hosted-lin-client-azure-linux3-load --profile cobalt-hosted-lin-db-azure-linux3-db " - + # GROUP 20 -- job: Trends_Database_28_Core_Cobalt_Hosted_Lin_Server - displayName: 20- Trends Database 28 Core Cobalt Hosted Lin Server +- job: Trends_Database_cobalt_hosted_lin_28 + displayName: 20- Trends Database cobalt-hosted-lin-28 pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Database_Cobalt_Hosted_Lin_Server, Trends_Database_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Trends_Database_cobalt_hosted_lin, Trends_Database_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: trend-database-scenarios.yml @@ -614,12 +613,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-28-app --profile cobalt-hosted-lin-client-load --profile cobalt-hosted-lin-db-db " - -- job: Trends_Database_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 20- Trends Database 28 Core Cobalt Hosted Lin Server Azure Linux3 + +- job: Trends_Database_cobalt_hosted_lin_azl3_28 + displayName: 20- Trends Database cobalt-hosted-lin-azl3-28 pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Database_Cobalt_Hosted_Lin_Server, Trends_Database_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Trends_Database_cobalt_hosted_lin, Trends_Database_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: trend-database-scenarios.yml @@ -628,14 +627,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-28-app --profile cobalt-hosted-lin-client-azure-linux3-load --profile cobalt-hosted-lin-db-azure-linux3-db " - + # GROUP 21 -- job: Crossgen_Cobalt_Hosted_Lin_Server - displayName: 21- Crossgen Cobalt Hosted Lin Server +- job: Crossgen_cobalt_hosted_lin + displayName: 21- Crossgen cobalt-hosted-lin pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Database_28_Core_Cobalt_Hosted_Lin_Server, Trends_Database_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Trends_Database_cobalt_hosted_lin_28, Trends_Database_cobalt_hosted_lin_azl3_28] condition: succeededOrFailed() steps: - template: crossgen2-scenarios.yml @@ -644,12 +643,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-app --profile cobalt-hosted-lin-client-load " - -- job: Crossgen_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 21- Crossgen Cobalt Hosted Lin Server Azure Linux3 + +- job: Crossgen_cobalt_hosted_lin_azl3 + displayName: 21- Crossgen cobalt-hosted-lin-azl3 pool: server timeoutInMinutes: 120 - dependsOn: [Trends_Database_28_Core_Cobalt_Hosted_Lin_Server, Trends_Database_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Trends_Database_cobalt_hosted_lin_28, Trends_Database_cobalt_hosted_lin_azl3_28] condition: succeededOrFailed() steps: - template: crossgen2-scenarios.yml @@ -658,14 +657,14 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-app --profile cobalt-hosted-lin-client-azure-linux3-load " - + # GROUP 22 -- job: Crossgen_28_Core_Cobalt_Hosted_Lin_Server - displayName: 22- Crossgen 28 Core Cobalt Hosted Lin Server +- job: Crossgen_cobalt_hosted_lin_28 + displayName: 22- Crossgen cobalt-hosted-lin-28 pool: server timeoutInMinutes: 120 - dependsOn: [Crossgen_Cobalt_Hosted_Lin_Server, Crossgen_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Crossgen_cobalt_hosted_lin, Crossgen_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: crossgen2-scenarios.yml @@ -674,12 +673,12 @@ jobs: serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-28-app --profile cobalt-hosted-lin-client-load " - -- job: Crossgen_28_Core_Cobalt_Hosted_Lin_Server_Azure_Linux3 - displayName: 22- Crossgen 28 Core Cobalt Hosted Lin Server Azure Linux3 + +- job: Crossgen_cobalt_hosted_lin_azl3_28 + displayName: 22- Crossgen cobalt-hosted-lin-azl3-28 pool: server timeoutInMinutes: 120 - dependsOn: [Crossgen_Cobalt_Hosted_Lin_Server, Crossgen_Cobalt_Hosted_Lin_Server_Azure_Linux3] + dependsOn: [Crossgen_cobalt_hosted_lin, Crossgen_cobalt_hosted_lin_azl3] condition: succeededOrFailed() steps: - template: crossgen2-scenarios.yml @@ -688,4 +687,4 @@ jobs: serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-28-app --profile cobalt-hosted-lin-client-azure-linux3-load " - + diff --git a/build/benchmarks.matrix.01.yml b/build/benchmarks.matrix.01.yml deleted file mode 100644 index 1540e7b61..000000000 --- a/build/benchmarks.matrix.01.yml +++ /dev/null @@ -1,299 +0,0 @@ -# This file describes all the scenarios which are run continuously on AzDo. -# It generates the file benchmarks.yml. See this file for instructions. - -queues: - - citrine1 - - citrine2 - - citrine3 - - mono - -schedule: "0 9/12 * * *" - -groups: - - - jobs: - - - name: Trends Database Intel Linux - template: trend-database-scenarios.yml - profiles: - - intel-lin-app - - intel-load-load - - intel-db-db - - - name: Trends Intel Windows - template: trend-scenarios.yml - profiles: - - intel-win-app - - intel-load2-load - - - name: Trends Database Arm 28 Linux - template: trend-database-scenarios.yml - profiles: - - arm-lin-28-app - - amd-lin-load - - amd-lin2-db - - - name: Trends Database Gold Linux - template: trend-database-scenarios.yml - profiles: - - aspnet-gold-lin - - - jobs: - - - name: Trends Database Intel Windows - template: trend-database-scenarios.yml - profiles: - - intel-win-app - - intel-load-load - - intel-db-db - - - name: Trends Intel Linux - template: trend-scenarios.yml - profiles: - - intel-lin-app - - intel-load2-load - - - name: Native Aot Arm 28 Linux - template: nativeaot-scenarios.yml - profiles: - - arm-lin-28-app - - amd-lin-load - - amd-lin2-db - - - name: Trends Gold Windows - template: trend-scenarios.yml - profiles: - - aspnet-gold-win - - - jobs: - - - name: Trends Database Amd Linux - template: trend-database-scenarios.yml - profiles: - - amd-lin-app - - intel-load-load - - intel-db-db - - - name: Trends Arm 28 Linux - template: trend-scenarios.yml - profiles: - - arm-lin-28-app - - intel-load2-load - - - name: GC Intel Windows - template: gc-scenarios.yml - profiles: - - intel-win-app - - - name: Trends Gold Linux - template: trend-scenarios.yml - profiles: - - aspnet-gold-lin - - - jobs: - - - name: Trends Amd Linux - template: trend-scenarios.yml - profiles: - - amd-lin-app - - intel-load2-load - - - name: Containers Intel Linux - template: containers-scenarios.yml - profiles: - - intel-lin-app - - intel-load-load - - intel-db-db - - - name: EF Core Intel Windows - template: efcore-scenarios.yml - profiles: - - intel-win-app - - - name: Trends Database Gold Windows - template: trend-database-scenarios.yml - profiles: - - aspnet-gold-win - - - jobs: - - - name: Baselines Database Intel Linux - template: baselines-database-scenarios.yml - profiles: - - intel-lin-app - - intel-load-load - - intel-db-db - - - name: Baselines Intel Windows - template: baselines-scenarios.yml - profiles: - - intel-win-app - - intel-load2-load - - - name: Baselines Database Arm 28 Linux - template: baselines-database-scenarios.yml - profiles: - - arm-lin-28-app - - amd-lin-load - - amd-lin2-db - - - name: Baselines Database Gold Linux - template: baselines-database-scenarios.yml - profiles: - - aspnet-gold-lin - - - jobs: - - - name: Baselines Database Intel Windows - template: baselines-database-scenarios.yml - profiles: - - intel-win-app - - intel-load-load - - intel-db-db - - - name: Baselines Intel Linux - template: baselines-scenarios.yml - profiles: - - intel-lin-app - - intel-load2-load - - - name: PGO Arm 28 Linux - template: pgo-scenarios.yml - profiles: - - arm-lin-28-app - - amd-lin-load - - amd-lin2-db - - - name: Baselines Gold Windows - template: baselines-scenarios.yml - profiles: - - aspnet-gold-win - - - jobs: - - - name: Baselines Database Amd Linux - template: baselines-database-scenarios.yml - profiles: - - amd-lin-app - - intel-load-load - - intel-db-db - - - name: Baselines Arm 28 Linux - template: baselines-scenarios.yml - profiles: - - arm-lin-28-app - - intel-load2-load - - - name: Native Aot Intel Windows - template: nativeaot-scenarios.yml - profiles: - - intel-win-app - - intel-lin-load - - amd-lin2-db - - - name: Baselines Gold Linux - template: baselines-scenarios.yml - profiles: - - aspnet-gold-lin - - - jobs: - - - name: Baselines Amd Linux - template: baselines-scenarios.yml - profiles: - - amd-lin-app - - intel-lin-load - - - name: Frameworks Intel Linux - template: frameworks-scenarios.yml - profiles: - - intel-lin-app - - intel-load-load - - amd-lin2-db # required by nodejs and tfb-database argument - - - name: Grpc Intel Windows - template: grpc-scenarios.yml - profiles: - - intel-win-app - - intel-db-load - - - name: Baselines Database Gold Windows - template: baselines-database-scenarios.yml - profiles: - - aspnet-gold-win - - - jobs: - - - name: Frameworks Database Intel Linux - template: frameworks-database-scenarios.yml - profiles: - - intel-lin-app - - intel-load-load - - intel-db-db - - - name: WebSockets Intel Windows - template: websocket-scenarios.yml - profiles: - - intel-win-app - - intel-load2-load - - - name: Frameworks Database Arm 28 Linux - template: frameworks-database-scenarios.yml - profiles: - - arm-lin-28-app - - amd-lin-load - - amd-lin2-db - - - name: Frameworks Database Intel Gold Linux - template: frameworks-database-scenarios.yml - profiles: - - aspnet-gold-lin - - - jobs: - - - name: Frameworks Database Amd Linux - template: frameworks-database-scenarios.yml - profiles: - - amd-lin-app - - intel-load-load - - intel-db-db - - - name: Frameworks Arm 80 Linux - template: frameworks-scenarios.yml - profiles: - - arm-lin-app - - amd-lin-load - - amd-lin2-db # required by nodejs and tfb-database argument - - - name: Trends Siryn Linux - template: trend-scenarios.yml - profiles: - - siryn-lin-app - - intel-load2-load - - - name: Frameworks Intel Gold Linux - template: frameworks-scenarios.yml - profiles: - - aspnet-gold-lin - - - jobs: - - - name: Frameworks Amd Linux - template: frameworks-scenarios.yml - profiles: - - amd-lin-app - - amd-lin2-load - - intel-load2-db # required by nodejs and tfb-database argument - - - name: Native Aot Intel Linux - template: nativeaot-scenarios.yml - profiles: - - intel-lin-app - - intel-load-load - - intel-db-db - - - name: Baselines Siryn Linux - template: baselines-scenarios.yml - profiles: - - siryn-lin-app - - arm-lin-load diff --git a/build/benchmarks.matrix.02.yml b/build/benchmarks.matrix.02.yml deleted file mode 100644 index 725ba9640..000000000 --- a/build/benchmarks.matrix.02.yml +++ /dev/null @@ -1,231 +0,0 @@ -# This file describes all the scenarios which are run continuously on AzDo. -# It generates the file benchmarks.yml. See this file for instructions. - -queues: - - citrine1 - - citrine2 - - citrine3 - - mono - -schedule: "0 3/12 * * *" - -groups: - - - jobs: - - - name: Frameworks Arm 28 Linux - template: frameworks-scenarios.yml - profiles: - - arm-lin-28-app - - intel-load2-load - - amd-lin2-db # required by nodejs and tfb-database argument - - - name: GC Intel Linux - template: gc-scenarios.yml - profiles: - - intel-lin-app - - - name: SslStream Intel Linux - template: sslstream-scenarios.yml - profiles: - - intel-load-app - - amd-lin-load - - - jobs: - - - name: PGO Intel Windows - template: pgo-scenarios.yml - profiles: - - intel-win-app - - amd-lin2-load - - intel-db-db - - - name: PGO Intel Linux - template: pgo-scenarios.yml - profiles: - - intel-lin-app - - amd-lin-load - - intel-load-db - - - name: Mvc Siryn Linux - template: mvc-scenarios.yml - profiles: - - siryn-lin-app - - arm-lin-load - - intel-load2-db - - - jobs: - - - name: SignalR Intel Linux - template: signalr-scenarios.yml - profiles: - - intel-lin-app - - intel-load-load - - - name: SignalR Intel Windows - template: signalr-scenarios.yml - profiles: - - intel-win-app - - intel-load2-load - - - name: Blazor Intel Linux - template: blazor-scenarios.yml - profiles: - - intel-perflin-app - - intel-perfload-load - - - jobs: - - - name: WebSockets Intel Linux - template: websocket-scenarios.yml - profiles: - - intel-lin-app - - intel-load-load - - - name: Build Intel Linux - template: build-perf-scenarios.yml - profiles: - - intel-load2-app - - intel-db-load - - - name: Trends Database Arm 80 Linux - template: trend-database-scenarios.yml - profiles: - - arm-lin-app - - amd-lin-load - - amd-lin2-db - - - jobs: - - - name: Single File Intel Linux - template: singlefile-scenarios.yml - profiles: - - intel-lin-app - - intel-load-load - - - name: Mvc Intel Windows - template: mvc-scenarios.yml - profiles: - - intel-win-app - - intel-load2-load - - intel-db-db - - - name: Frameworks Database Arm 80 Linux - template: frameworks-database-scenarios.yml - profiles: - - arm-lin-app - - amd-lin-load - - amd-lin2-db - - - jobs: - - - name: Mvc Intel Linux - template: mvc-scenarios.yml - profiles: - - intel-lin-app - - intel-load-load - - intel-db-db - - - name: Single File Intel Windows - template: singlefile-scenarios.yml - profiles: - - intel-win-app - - intel-load2-load - - - name: GC Arm 28 Linux - template: gc-scenarios.yml - profiles: - - arm-lin-28-app - - - jobs: - - - name: Custom Proxies Intel Linux - template: proxies-custom-scenarios.yml - profiles: - - intel-lin-app - - intel-load-load - - intel-db-db - - - name: Build Intel Windows - template: build-perf-scenarios.yml - profiles: - - intel-win-app - - intel-load2-load - - - name: Trends Arm 80 Linux - template: trend-scenarios.yml - profiles: - - arm-lin-app - - amd-lin-load - - - jobs: - - - name: Proxies Intel Linux - template: proxies-scenarios.yml - timeout: 180 - profiles: - - intel-lin-app - - intel-load-load - - intel-db-db - - - name: Grpc Intel Linux - template: grpc-scenarios.yml - profiles: - - intel-load2-app - - amd-lin-load - - - name: Baselines Database Siryn Linux - template: baselines-database-scenarios.yml - profiles: - - siryn-lin-app - - arm-lin-load - - amd-lin2-db - - - jobs: - - - name: Crossgen Arm 28 Linux - template: crossgen2-scenarios.yml - profiles: - - arm-lin-28-app - - amd-lin-load - - - name: Crossgen Intel Linux - template: crossgen2-scenarios.yml - profiles: - - intel-lin-app - - intel-load-load - - - name: Trends Database Siryn Linux - template: trend-database-scenarios.yml - profiles: - - siryn-lin-app - - intel-load2-load - - intel-db-db - - - jobs: - - - name: Crossgen Intel Windows - template: crossgen2-scenarios.yml - profiles: - - intel-win-app - - intel-load-load - - - name: Crossgen Amd Linux - template: crossgen2-scenarios.yml - profiles: - - amd-lin-app - - intel-load2-load - - - name: HttpClient Intel Linux - template: httpclient-scenarios.yml - profiles: - - intel-lin-app - - amd-lin2-load - - - jobs: - - - name: HttpClient Intel Windows - template: httpclient-scenarios.yml - profiles: - - intel-win-app - - amd-lin2-load diff --git a/build/benchmarks.template.liquid b/build/benchmarks.template.liquid deleted file mode 100644 index 2aa230237..000000000 --- a/build/benchmarks.template.liquid +++ /dev/null @@ -1,56 +0,0 @@ -# Do not change this file, it is generated using these steps: -# - The file benchmarks*.json defines how each pipeline set of jobs is run in parallel -# - Update the associated benchmarks*.json file with machine and scenario updates -# - Install python and install the requirements for the crank-scheduler in benchmarks/scripts/crank-scheduler/requirements.txt -# - Run the scheduler specifying the desired benchmarks*.json file, this template, and benchmarks/output to automatically overwrite the current pipeline. -# - Ex. python ./scripts/crank-scheduler/main.py --config ./build/benchmarks_ci.json --template ./build/benchmarks.template.liquid --yaml-output ./build - -{%- assign defaultTimeout = 120 %} - -trigger: none -pr: none - -schedules: -- cron: "{{ schedule }}" - always: true # always run the pipeline even if there have not been source code changes since the last successful scheduled run. - branches: - include: - - main - -variables: - - template: job-variables.yml - - name: session - value: $(Build.BuildNumber) - - name: buildId - value: $(Build.BuildId) - - name: buildNumber - value: $(Build.BuildNumber) - - name: am - value: $[lt(format('{0:HH}', pipeline.startTime), 12)] - - name: pm - value: $[ge(format('{0:HH}', pipeline.startTime), 12)] - -jobs: - -{% for group in groups -%} -{% assign n = forloop.index -%} -# GROUP {{ n }} - -{% for job in group.jobs -%} -- job: {{job.name | replace: ' ', '_'}} - displayName: {{ n }}- {{job.name}} - pool: server - timeoutInMinutes: {{ job.timeout | default: defaultTimeout }} - dependsOn: [{%for previousJob in previousGroup.jobs%}{{previousJob.name | replace: ' ', '_'}}{%if forloop.last == false%}, {%endif%}{%endfor%}] - condition: succeededOrFailed() - steps: - - template: {{job.template}} - parameters: - connection: ASPNET Benchmarks Service Bus - serviceBusQueueName: {{queues[forloop.index0]}} - serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) {%for p in job.profiles%}--profile {{p}} {% endfor %}" - -{% endfor %} -{%-assign previousGroup = group -%} -{% endfor %} \ No newline at end of file diff --git a/build/benchmarks.yml b/build/benchmarks.yml deleted file mode 100644 index 39149baff..000000000 --- a/build/benchmarks.yml +++ /dev/null @@ -1,847 +0,0 @@ - -# Do not change this file, it is generated using these steps: -# - The file benchmarks.matrix.yml defines how each job is run in parallel -# - Convert its content to json using https://jsonformatter.org/yaml-to-json -# - Use the template in benchmarks.template.liquid and the converted json using https://liquidjs.com/playground.html -# - Update this file with the result of the template generation - -variables: - - template: job-variables.yml - - name: session - value: $(Build.BuildNumber) - - name: buildId - value: $(Build.BuildId) - - name: buildNumber - value: $(Build.BuildNumber) - - name: am - value: $[lt(format('{0:HH}', pipeline.startTime), 12)] - - name: pm - value: $[ge(format('{0:HH}', pipeline.startTime), 12)] - -jobs: - -# GROUP 1 - -- job: Trends_Database_Intel_Linux - displayName: 1- Trends Database Intel Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [] - condition: succeededOrFailed() - steps: - - template: trend-database-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile intel-db-db " - -- job: Trends_Intel_Windows - displayName: 1- Trends Intel Windows - pool: server - timeoutInMinutes: 120 - dependsOn: [] - condition: succeededOrFailed() - steps: - - template: trend-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load " - -- job: Trends_Database_Arm_28_Linux - displayName: 1- Trends Database Arm 28 Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [] - condition: succeededOrFailed() - steps: - - template: trend-database-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 3 - arguments: "$(ciProfile) --profile arm-lin-28-app --profile amd-lin-load --profile amd-lin2-db " - -- job: Trends_Database_Azure_Linux - displayName: 1- Trends Database Azure Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [] - condition: succeededOrFailed() - steps: - - template: trend-database-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Azure Queue - arguments: "$(ciProfile) --profile aspnet-azure-lin " - -# GROUP 2 - -- job: Trends_Database_Intel_Windows - displayName: 2- Trends Database Intel Windows - pool: server - timeoutInMinutes: 120 - dependsOn: [Trends_Database_Intel_Linux, Trends_Intel_Windows, Trends_Database_Arm_28_Linux, Trends_Database_Azure_Linux] - condition: succeededOrFailed() - steps: - - template: trend-database-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile intel-win-app --profile intel-load-load --profile intel-db-db " - -- job: Trends_Intel_Linux - displayName: 2- Trends Intel Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Trends_Database_Intel_Linux, Trends_Intel_Windows, Trends_Database_Arm_28_Linux, Trends_Database_Azure_Linux] - condition: succeededOrFailed() - steps: - - template: trend-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load2-load " - -- job: Trends_Database_Arm_28_Windows - displayName: 2- Trends Database Arm 28 Windows - pool: server - timeoutInMinutes: 120 - dependsOn: [Trends_Database_Intel_Linux, Trends_Intel_Windows, Trends_Database_Arm_28_Linux, Trends_Database_Azure_Linux] - condition: succeededOrFailed() - steps: - - template: trend-database-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 3 - arguments: "$(ciProfile) --profile arm-win-28-app --profile amd-lin-load --profile amd-lin2-db " - -- job: Trends_Azure_Linux - displayName: 2- Trends Azure Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Trends_Database_Intel_Linux, Trends_Intel_Windows, Trends_Database_Arm_28_Linux, Trends_Database_Azure_Linux] - condition: succeededOrFailed() - steps: - - template: trend-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Azure Queue - arguments: "$(ciProfile) --profile aspnet-azure-lin " - -# GROUP 3 - -- job: Trends_Database_Amd_Linux - displayName: 3- Trends Database Amd Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Trends_Database_Intel_Windows, Trends_Intel_Linux, Trends_Database_Arm_28_Windows, Trends_Azure_Linux] - condition: succeededOrFailed() - steps: - - template: trend-database-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile amd-lin-app --profile intel-load-load --profile intel-db-db " - -- job: Trends_Arm_28_Linux - displayName: 3- Trends Arm 28 Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Trends_Database_Intel_Windows, Trends_Intel_Linux, Trends_Database_Arm_28_Windows, Trends_Azure_Linux] - condition: succeededOrFailed() - steps: - - template: trend-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile arm-lin-28-app --profile intel-load2-load " - -- job: Trends_Arm_28_Windows - displayName: 3- Trends Arm 28 Windows - pool: server - timeoutInMinutes: 120 - dependsOn: [Trends_Database_Intel_Windows, Trends_Intel_Linux, Trends_Database_Arm_28_Windows, Trends_Azure_Linux] - condition: succeededOrFailed() - steps: - - template: trend-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 3 - arguments: "$(ciProfile) --profile arm-win-28-app --profile intel-lin-load " - -- job: Baselines_Database_Azure_Linux - displayName: 3- Baselines Database Azure Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Trends_Database_Intel_Windows, Trends_Intel_Linux, Trends_Database_Arm_28_Windows, Trends_Azure_Linux] - condition: succeededOrFailed() - steps: - - template: baselines-database-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Azure Queue - arguments: "$(ciProfile) --profile aspnet-azure-lin " - -# GROUP 4 - -- job: Trends_Amd_Linux - displayName: 4- Trends Amd Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Trends_Database_Amd_Linux, Trends_Arm_28_Linux, Trends_Arm_28_Windows, Baselines_Database_Azure_Linux] - condition: succeededOrFailed() - steps: - - template: trend-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile amd-lin-app --profile intel-load2-load " - -- job: Containers_Intel_Linux - displayName: 4- Containers Intel Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Trends_Database_Amd_Linux, Trends_Arm_28_Linux, Trends_Arm_28_Windows, Baselines_Database_Azure_Linux] - condition: succeededOrFailed() - steps: - - template: containers-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile intel-db-db " - -- job: EF_Core_Intel_Windows - displayName: 4- EF Core Intel Windows - pool: server - timeoutInMinutes: 120 - dependsOn: [Trends_Database_Amd_Linux, Trends_Arm_28_Linux, Trends_Arm_28_Windows, Baselines_Database_Azure_Linux] - condition: succeededOrFailed() - steps: - - template: efcore-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 3 - arguments: "$(ciProfile) --profile intel-win-app " - -- job: Baselines_Azure_Linux - displayName: 4- Baselines Azure Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Trends_Database_Amd_Linux, Trends_Arm_28_Linux, Trends_Arm_28_Windows, Baselines_Database_Azure_Linux] - condition: succeededOrFailed() - steps: - - template: baselines-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Azure Queue - arguments: "$(ciProfile) --profile aspnet-azure-lin " - -# GROUP 5 - -- job: Baselines_Database_Intel_Linux - displayName: 5- Baselines Database Intel Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Trends_Amd_Linux, Containers_Intel_Linux, EF_Core_Intel_Windows, Baselines_Azure_Linux] - condition: succeededOrFailed() - steps: - - template: baselines-database-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile intel-db-db " - -- job: Baselines_Intel_Windows - displayName: 5- Baselines Intel Windows - pool: server - timeoutInMinutes: 120 - dependsOn: [Trends_Amd_Linux, Containers_Intel_Linux, EF_Core_Intel_Windows, Baselines_Azure_Linux] - condition: succeededOrFailed() - steps: - - template: baselines-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load " - -- job: Baselines_Database_Arm_28_Linux - displayName: 5- Baselines Database Arm 28 Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Trends_Amd_Linux, Containers_Intel_Linux, EF_Core_Intel_Windows, Baselines_Azure_Linux] - condition: succeededOrFailed() - steps: - - template: baselines-database-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 3 - arguments: "$(ciProfile) --profile arm-lin-28-app --profile amd-lin-load --profile amd-lin2-db " - -# GROUP 6 - -- job: Baselines_Database_Intel_Windows - displayName: 6- Baselines Database Intel Windows - pool: server - timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Intel_Linux, Baselines_Intel_Windows, Baselines_Database_Arm_28_Linux] - condition: succeededOrFailed() - steps: - - template: baselines-database-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile intel-win-app --profile intel-load-load --profile intel-db-db " - -- job: Baselines_Intel_Linux - displayName: 6- Baselines Intel Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Intel_Linux, Baselines_Intel_Windows, Baselines_Database_Arm_28_Linux] - condition: succeededOrFailed() - steps: - - template: baselines-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load2-load " - -- job: Baselines_Database_Arm_28_Windows - displayName: 6- Baselines Database Arm 28 Windows - pool: server - timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Intel_Linux, Baselines_Intel_Windows, Baselines_Database_Arm_28_Linux] - condition: succeededOrFailed() - steps: - - template: baselines-database-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 3 - arguments: "$(ciProfile) --profile arm-win-28-app --profile amd-lin-load --profile amd-lin2-db " - -# GROUP 7 - -- job: Baselines_Database_Amd_Linux - displayName: 7- Baselines Database Amd Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Intel_Windows, Baselines_Intel_Linux, Baselines_Database_Arm_28_Windows] - condition: succeededOrFailed() - steps: - - template: baselines-database-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile amd-lin-app --profile intel-load-load --profile intel-db-db " - -- job: Baselines_Arm_28_Linux - displayName: 7- Baselines Arm 28 Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Intel_Windows, Baselines_Intel_Linux, Baselines_Database_Arm_28_Windows] - condition: succeededOrFailed() - steps: - - template: baselines-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile arm-lin-28-app --profile intel-load2-load " - -- job: Baselines_Arm_28_Windows - displayName: 7- Baselines Arm 28 Windows - pool: server - timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Intel_Windows, Baselines_Intel_Linux, Baselines_Database_Arm_28_Windows] - condition: succeededOrFailed() - steps: - - template: baselines-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 3 - arguments: "$(ciProfile) --profile arm-win-28-app --profile intel-lin-load " - -# GROUP 8 - -- job: Baselines_Amd_Linux - displayName: 8- Baselines Amd Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Amd_Linux, Baselines_Arm_28_Linux, Baselines_Arm_28_Windows] - condition: succeededOrFailed() - steps: - - template: baselines-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile amd-lin-app --profile intel-load2-load " - -- job: Frameworks_Intel_Linux - displayName: 8- Frameworks Intel Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Amd_Linux, Baselines_Arm_28_Linux, Baselines_Arm_28_Windows] - condition: succeededOrFailed() - steps: - - template: frameworks-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile amd-lin2-db " - -- job: Grpc_Intel_Windows - displayName: 8- Grpc Intel Windows - pool: server - timeoutInMinutes: 120 - dependsOn: [Baselines_Database_Amd_Linux, Baselines_Arm_28_Linux, Baselines_Arm_28_Windows] - condition: succeededOrFailed() - steps: - - template: grpc-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 3 - arguments: "$(ciProfile) --profile intel-win-app --profile intel-db-load " - -# GROUP 9 - -- job: Frameworks_Database_Intel_Linux - displayName: 9- Frameworks Database Intel Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Baselines_Amd_Linux, Frameworks_Intel_Linux, Grpc_Intel_Windows] - condition: succeededOrFailed() - steps: - - template: frameworks-database-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile intel-db-db " - -- job: WebSockets_Intel_Windows - displayName: 9- WebSockets Intel Windows - pool: server - timeoutInMinutes: 120 - dependsOn: [Baselines_Amd_Linux, Frameworks_Intel_Linux, Grpc_Intel_Windows] - condition: succeededOrFailed() - steps: - - template: websocket-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load " - -- job: Frameworks_Database_Arm_28_Linux - displayName: 9- Frameworks Database Arm 28 Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Baselines_Amd_Linux, Frameworks_Intel_Linux, Grpc_Intel_Windows] - condition: succeededOrFailed() - steps: - - template: frameworks-database-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 3 - arguments: "$(ciProfile) --profile arm-lin-28-app --profile amd-lin-load --profile amd-lin2-db " - -# GROUP 10 - -- job: Frameworks_Database_Amd_Linux - displayName: 10- Frameworks Database Amd Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Frameworks_Database_Intel_Linux, WebSockets_Intel_Windows, Frameworks_Database_Arm_28_Linux] - condition: succeededOrFailed() - steps: - - template: frameworks-database-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile amd-lin-app --profile intel-load-load --profile intel-db-db " - -- job: PGO_Intel_Windows - displayName: 10- PGO Intel Windows - pool: server - timeoutInMinutes: 120 - dependsOn: [Frameworks_Database_Intel_Linux, WebSockets_Intel_Windows, Frameworks_Database_Arm_28_Linux] - condition: succeededOrFailed() - steps: - - template: pgo-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile intel-win-app --profile intel-lin-load --profile intel-load2-db " - -- job: Frameworks_Arm_80_Linux - displayName: 10- Frameworks Arm 80 Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Frameworks_Database_Intel_Linux, WebSockets_Intel_Windows, Frameworks_Database_Arm_28_Linux] - condition: succeededOrFailed() - steps: - - template: frameworks-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 3 - arguments: "$(ciProfile) --profile arm-lin-app --profile amd-lin-load --profile amd-lin2-db " - -# GROUP 11 - -- job: Frameworks_Amd_Linux - displayName: 11- Frameworks Amd Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Frameworks_Database_Amd_Linux, PGO_Intel_Windows, Frameworks_Arm_80_Linux] - condition: succeededOrFailed() - steps: - - template: frameworks-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile amd-lin-app --profile intel-load2-load --profile amd-lin2-db " - -- job: PGO_Intel_Linux - displayName: 11- PGO Intel Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Frameworks_Database_Amd_Linux, PGO_Intel_Windows, Frameworks_Arm_80_Linux] - condition: succeededOrFailed() - steps: - - template: pgo-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile intel-db-db " - -# GROUP 12 - -- job: Mono_Database_Intel_Linux - displayName: 12- Mono Database Intel Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Frameworks_Amd_Linux, PGO_Intel_Linux] - condition: succeededOrFailed() - steps: - - template: mono-database-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile intel-lin-app --profile amd-lin-load --profile intel-db-db " - -- job: Mono_Database_Arm_28_Linux - displayName: 12- Mono Database Arm 28 Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Frameworks_Amd_Linux, PGO_Intel_Linux] - condition: succeededOrFailed() - steps: - - template: mono-database-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile arm-lin-28-app --profile amd-lin2-load --profile intel-load2-db " - -# GROUP 13 - -- job: Mono_Arm_28_Linux - displayName: 13- Mono Arm 28 Linux - pool: server - timeoutInMinutes: 180 - dependsOn: [Mono_Database_Intel_Linux, Mono_Database_Arm_28_Linux] - condition: succeededOrFailed() - steps: - - template: mono-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile arm-lin-28-app --profile intel-load2-load " - -- job: Mono_Intel_Linux - displayName: 13- Mono Intel Linux - pool: server - timeoutInMinutes: 180 - dependsOn: [Mono_Database_Intel_Linux, Mono_Database_Arm_28_Linux] - condition: succeededOrFailed() - steps: - - template: mono-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load " - -# GROUP 14 - -- job: PGO_Arm_28_Linux - displayName: 14- PGO Arm 28 Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Mono_Arm_28_Linux, Mono_Intel_Linux] - condition: succeededOrFailed() - steps: - - template: pgo-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile arm-lin-28-app --profile intel-load2-load --profile intel-db-db " - -- job: Grpc_Intel_Linux - displayName: 14- Grpc Intel Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Mono_Arm_28_Linux, Mono_Intel_Linux] - condition: succeededOrFailed() - steps: - - template: grpc-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load " - -- job: Trends_Arm_80_Windows - displayName: 14- Trends Arm 80 Windows - pool: server - timeoutInMinutes: 120 - dependsOn: [Mono_Arm_28_Linux, Mono_Intel_Linux] - condition: succeededOrFailed() - steps: - - template: trend-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 3 - arguments: "$(ciProfile) --profile arm-win-app --profile amd-lin2-load " - -# GROUP 15 - -- job: SignalR_Intel_Linux - displayName: 15- SignalR Intel Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [PGO_Arm_28_Linux, Grpc_Intel_Linux, Trends_Arm_80_Windows] - condition: succeededOrFailed() - steps: - - template: signalr-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load " - -- job: SignalR_Intel_Windows - displayName: 15- SignalR Intel Windows - pool: server - timeoutInMinutes: 120 - dependsOn: [PGO_Arm_28_Linux, Grpc_Intel_Linux, Trends_Arm_80_Windows] - condition: succeededOrFailed() - steps: - - template: signalr-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile intel-win-app --profile intel-load-load " - -- job: Blazor_Intel_Linux - displayName: 15- Blazor Intel Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [PGO_Arm_28_Linux, Grpc_Intel_Linux, Trends_Arm_80_Windows] - condition: succeededOrFailed() - steps: - - template: blazor-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 3 - arguments: "$(ciProfile) --profile intel-perflin-app " - -# GROUP 16 - -- job: WebSockets_Intel_Linux - displayName: 16- WebSockets Intel Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [SignalR_Intel_Linux, SignalR_Intel_Windows, Blazor_Intel_Linux] - condition: succeededOrFailed() - steps: - - template: websocket-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load " - -- job: Build_Intel_Linux - displayName: 16- Build Intel Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [SignalR_Intel_Linux, SignalR_Intel_Windows, Blazor_Intel_Linux] - condition: succeededOrFailed() - steps: - - template: build-perf-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile intel-load2-app --profile intel-db-load " - -- job: Trends_Database_Arm_80_Linux - displayName: 16- Trends Database Arm 80 Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [SignalR_Intel_Linux, SignalR_Intel_Windows, Blazor_Intel_Linux] - condition: succeededOrFailed() - steps: - - template: trend-database-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 3 - arguments: "$(ciProfile) --profile arm-lin-app --profile amd-lin-load --profile amd-lin2-db " - -# GROUP 17 - -- job: Single_File_Intel_Linux - displayName: 17- Single File Intel Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [WebSockets_Intel_Linux, Build_Intel_Linux, Trends_Database_Arm_80_Linux] - condition: succeededOrFailed() - steps: - - template: singlefile-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load " - -- job: Mvc_Intel_Windows - displayName: 17- Mvc Intel Windows - pool: server - timeoutInMinutes: 120 - dependsOn: [WebSockets_Intel_Linux, Build_Intel_Linux, Trends_Database_Arm_80_Linux] - condition: succeededOrFailed() - steps: - - template: mvc-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load --profile intel-db-db " - -- job: Frameworks_Database_Arm_80_Linux - displayName: 17- Frameworks Database Arm 80 Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [WebSockets_Intel_Linux, Build_Intel_Linux, Trends_Database_Arm_80_Linux] - condition: succeededOrFailed() - steps: - - template: frameworks-database-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 3 - arguments: "$(ciProfile) --profile arm-lin-app --profile amd-lin-load --profile amd-lin2-db " - -# GROUP 18 - -- job: Mvc_Intel_Linux - displayName: 18- Mvc Intel Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Single_File_Intel_Linux, Mvc_Intel_Windows, Frameworks_Database_Arm_80_Linux] - condition: succeededOrFailed() - steps: - - template: mvc-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile intel-db-db " - -- job: Single_File_Intel_Windows - displayName: 18- Single File Intel Windows - pool: server - timeoutInMinutes: 120 - dependsOn: [Single_File_Intel_Linux, Mvc_Intel_Windows, Frameworks_Database_Arm_80_Linux] - condition: succeededOrFailed() - steps: - - template: singlefile-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load " - -- job: Trends_Database_Arm_80_Windows - displayName: 18- Trends Database Arm 80 Windows - pool: server - timeoutInMinutes: 120 - dependsOn: [Single_File_Intel_Linux, Mvc_Intel_Windows, Frameworks_Database_Arm_80_Linux] - condition: succeededOrFailed() - steps: - - template: trend-database-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 3 - arguments: "$(ciProfile) --profile arm-win-app --profile amd-lin-load --profile amd-lin2-db " - -# GROUP 19 - -- job: Custom_Proxies_Intel_Linux - displayName: 19- Custom Proxies Intel Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Mvc_Intel_Linux, Single_File_Intel_Windows, Trends_Database_Arm_80_Windows] - condition: succeededOrFailed() - steps: - - template: proxies-custom-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile intel-db-db " - -- job: Build_Intel_Windows - displayName: 19- Build Intel Windows - pool: server - timeoutInMinutes: 120 - dependsOn: [Mvc_Intel_Linux, Single_File_Intel_Windows, Trends_Database_Arm_80_Windows] - condition: succeededOrFailed() - steps: - - template: build-perf-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile intel-win-app --profile intel-load2-load " - -- job: Trends_Arm_80_Linux - displayName: 19- Trends Arm 80 Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Mvc_Intel_Linux, Single_File_Intel_Windows, Trends_Database_Arm_80_Windows] - condition: succeededOrFailed() - steps: - - template: trend-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 3 - arguments: "$(ciProfile) --profile arm-lin-app --profile amd-lin-load " - -# GROUP 20 - -- job: Proxies_Intel_Linux - displayName: 20- Proxies Intel Linux - pool: server - timeoutInMinutes: 180 - dependsOn: [Custom_Proxies_Intel_Linux, Build_Intel_Windows, Trends_Arm_80_Linux] - condition: succeededOrFailed() - steps: - - template: proxies-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load --profile intel-db-db " - -- job: Frameworks_Arm_28_Linux - displayName: 20- Frameworks Arm 28 Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Custom_Proxies_Intel_Linux, Build_Intel_Windows, Trends_Arm_80_Linux] - condition: succeededOrFailed() - steps: - - template: frameworks-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile arm-lin-28-app --profile intel-load2-load --profile amd-lin2-db " - -# GROUP 21 - -- job: Crossgen_Arm_28_Linux - displayName: 21- Crossgen Arm 28 Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Proxies_Intel_Linux, Frameworks_Arm_28_Linux] - condition: succeededOrFailed() - steps: - - template: crossgen2-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile arm-lin-28-app --profile amd-lin-load --profile intel-db-db " - -- job: Crossgen_Arm_28_Windows - displayName: 21- Crossgen Arm 28 Windows - pool: server - timeoutInMinutes: 120 - dependsOn: [Proxies_Intel_Linux, Frameworks_Arm_28_Linux] - condition: succeededOrFailed() - steps: - - template: crossgen2-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile arm-win-28-app --profile amd-lin2-load " - -- job: Crossgen_Intel_Linux - displayName: 21- Crossgen Intel Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Proxies_Intel_Linux, Frameworks_Arm_28_Linux] - condition: succeededOrFailed() - steps: - - template: crossgen2-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 3 - arguments: "$(ciProfile) --profile intel-lin-app --profile intel-load-load " - -# GROUP 22 - -- job: Crossgen_Intel_Windows - displayName: 22- Crossgen Intel Windows - pool: server - timeoutInMinutes: 120 - dependsOn: [Crossgen_Arm_28_Linux, Crossgen_Arm_28_Windows, Crossgen_Intel_Linux] - condition: succeededOrFailed() - steps: - - template: crossgen2-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 1 - arguments: "$(ciProfile) --profile intel-win-app --profile intel-load-load " - -- job: Crossgen_Amd_Linux - displayName: 22- Crossgen Amd Linux - pool: server - timeoutInMinutes: 120 - dependsOn: [Crossgen_Arm_28_Linux, Crossgen_Arm_28_Windows, Crossgen_Intel_Linux] - condition: succeededOrFailed() - steps: - - template: crossgen2-scenarios.yml - parameters: - connection: ASP.NET Benchmarks Queue 2 - arguments: "$(ciProfile) --profile amd-lin-app --profile intel-load2-load " - - diff --git a/build/benchmarks_ci.json b/build/benchmarks_ci.json deleted file mode 100644 index 2c1fdec3b..000000000 --- a/build/benchmarks_ci.json +++ /dev/null @@ -1,522 +0,0 @@ -{ - "metadata": { - "name": "CI Benchmarks Configuration", - "description": "Combined machines and scenarios for continuous integration benchmarks", - "version": "1.0", - "schedule": "0 3/12 * * *", - "queues": [ - "citrine1", - "citrine2", - "citrine3", - "mono" - ], - "yaml_generation": { - "target_yaml_count": 2, - "schedule_offset_hours": 6 - } - }, - "machines": [ - { - "name": "intel-lin", - "machine_group": "intel", - "capabilities": { - "sut": { - "priority": 1, - "profiles": [ - "intel-lin-app" - ] - }, - "load": { - "priority": 2, - "profiles": [ - "intel-lin-load" - ] - } - }, - "preferred_partners": [ - "intel-load", - "intel-load2", - "intel-db" - ] - }, - { - "name": "intel-win", - "machine_group": "intel", - "capabilities": { - "sut": { - "priority": 1, - "profiles": [ - "intel-win-app" - ] - }, - "load": { - "priority": 2, - "profiles": [ - "intel-win-load" - ] - } - }, - "preferred_partners": [ - "intel-load2", - "intel-load", - "intel-db" - ] - }, - { - "name": "intel-load", - "machine_group": "intel", - "capabilities": { - "load": { - "priority": 1, - "profiles": [ - "intel-load-load" - ] - }, - "sut": { - "priority": 2, - "profiles": [ - "intel-load-app" - ] - }, - "db": { - "priority": 3, - "profiles": [ - "intel-load-db" - ] - } - } - }, - { - "name": "intel-load2", - "machine_group": "intel", - "capabilities": { - "load": { - "priority": 1, - "profiles": [ - "intel-load2-load" - ] - }, - "sut": { - "priority": 2, - "profiles": [ - "intel-load2-app" - ] - }, - "db": { - "priority": 3, - "profiles": [ - "intel-load2-db" - ] - } - } - }, - { - "name": "intel-db", - "machine_group": "intel", - "capabilities": { - "db": { - "priority": 1, - "profiles": [ - "intel-db-db" - ] - }, - "sut": { - "priority": 2, - "profiles": [ - "intel-db-app" - ] - }, - "load": { - "priority": 3, - "profiles": [ - "intel-db-load" - ] - } - } - }, - { - "name": "intel-perflin", - "machine_group": "intel-perf", - "capabilities": { - "sut": { - "priority": 1, - "profiles": [ - "intel-perflin-app" - ] - } - } - }, - { - "name": "intel-perfload", - "machine_group": "intel-perf", - "capabilities": { - "load": { - "priority": 1, - "profiles": [ - "intel-perfload-load" - ] - } - } - }, - { - "name": "amd-lin2", - "machine_group": "gold", - "capabilities": { - "sut": { - "priority": 1, - "profiles": [ - "amd-lin2-app" - ] - } - } - }, - { - "name": "gold-lin", - "machine_group": "gold", - "capabilities": { - "sut": { - "priority": 1, - "profiles": [ - "gold-lin-app", - "gold-lin-28-app" - ] - } - }, - "preferred_partners": [ - "gold-load", - "gold-load2", - "gold-db" - ] - }, - { - "name": "gold-win", - "machine_group": "gold", - "capabilities": { - "sut": { - "priority": 1, - "profiles": [ - "gold-win-app", - "gold-win-28-app" - ] - }, - "load": { - "priority": 2, - "profiles": [ - "gold-win-load" - ] - } - }, - "preferred_partners": [ - "gold-load2", - "gold-load", - "gold-db" - ] - }, - { - "name": "gold-load", - "machine_group": "gold", - "capabilities": { - "load": { - "priority": 1, - "profiles": [ - "gold-load-load" - ] - } - } - }, - { - "name": "gold-load2", - "machine_group": "gold", - "capabilities": { - "load": { - "priority": 1, - "profiles": [ - "gold-load2-load" - ] - } - } - }, - { - "name": "gold-db", - "machine_group": "gold", - "capabilities": { - "db": { - "priority": 1, - "profiles": [ - "gold-db-db" - ] - } - } - } - ], - "scenarios": [ - { - "name": "Baselines", - "template": "baselines-scenarios.yml", - "type": 2, - "target_machines": [ - "gold-lin", - "gold-win", - "intel-lin", - "intel-win", - "amd-lin2" - ], - "estimated_runtime": 30.0 - }, - { - "name": "Baselines Database", - "template": "baselines-database-scenarios.yml", - "type": 3, - "target_machines": [ - "gold-lin", - "gold-win", - "intel-lin", - "intel-win", - "amd-lin2" - ], - "estimated_runtime": 45.0 - }, - { - "name": "Blazor", - "template": "blazor-scenarios.yml", - "type": 2, - "target_machines": [ - "gold-lin", - "intel-lin", - "intel-perflin" - ], - "estimated_runtime": null - }, - { - "name": "Build", - "template": "build-perf-scenarios.yml", - "type": 1, - "target_machines": [ - "gold-lin", - "gold-win", - "intel-lin", - "intel-win" - ], - "estimated_runtime": 1.0 - }, - { - "name": "Containers", - "template": "containers-scenarios.yml", - "type": 3, - "target_machines": [ - "gold-lin", - "intel-lin" - ], - "estimated_runtime": 90.0 - }, - { - "name": "Crossgen", - "template": "crossgen2-scenarios.yml", - "type": 2, - "target_machines": [ - "gold-lin", - "gold-win", - "intel-lin", - "intel-win", - "amd-lin2" - ], - "estimated_runtime": 5.0 - }, - { - "name": "Custom Proxies", - "template": "proxies-custom-scenarios.yml", - "type": 3, - "target_machines": [ - "gold-lin", - "intel-lin" - ], - "estimated_runtime": 5.0 - }, - { - "name": "EF Core", - "template": "efcore-scenarios.yml", - "type": 2, - "target_machines": [ - "gold-win", - "intel-win" - ], - "estimated_runtime": 15.0 - }, - { - "name": "Frameworks", - "template": "frameworks-scenarios.yml", - "type": 3, - "target_machines": [ - "gold-lin", - "intel-lin", - "amd-lin2" - ], - "estimated_runtime": 15.0 - }, - { - "name": "Frameworks Database", - "template": "frameworks-database-scenarios.yml", - "type": 3, - "target_machines": [ - "gold-lin", - "intel-lin", - "amd-lin2" - ], - "estimated_runtime": 20.0 - }, - { - "name": "GC", - "template": "gc-scenarios.yml", - "type": 1, - "target_machines": [ - "gold-lin", - "gold-win", - "intel-lin", - "intel-win" - ], - "estimated_runtime": 15.0 - }, - { - "name": "Grpc", - "template": "grpc-scenarios.yml", - "type": 2, - "target_machines": [ - "gold-lin", - "gold-win", - "intel-lin", - "intel-win" - ], - "estimated_runtime": 70.0 - }, - { - "name": "HttpClient", - "template": "httpclient-scenarios.yml", - "type": 2, - "target_machines": [ - "gold-lin", - "gold-win", - "intel-lin", - "intel-win" - ], - "estimated_runtime": null - }, - { - "name": "MVC", - "template": "mvc-scenarios.yml", - "type": 3, - "target_machines": [ - "gold-lin", - "gold-win", - "intel-lin", - "intel-win" - ], - "estimated_runtime": 20.0 - }, - { - "name": "NativeAOT", - "template": "nativeaot-scenarios.yml", - "type": 3, - "target_machines": [ - "gold-lin", - "gold-win", - "intel-lin", - "intel-win" - ], - "estimated_runtime": 20.0 - }, - { - "name": "PGO", - "template": "pgo-scenarios.yml", - "type": 3, - "target_machines": [ - "gold-lin", - "gold-win", - "intel-lin", - "intel-win" - ], - "estimated_runtime": 90.0 - }, - { - "name": "Proxies", - "template": "proxies-scenarios.yml", - "type": 3, - "target_machines": [ - "gold-lin", - "intel-lin" - ], - "estimated_runtime": 150.0 - }, - { - "name": "SignalR", - "template": "signalr-scenarios.yml", - "type": 2, - "target_machines": [ - "gold-lin", - "gold-win", - "intel-lin", - "intel-win" - ], - "estimated_runtime": 30.0 - }, - { - "name": "Single File", - "template": "singlefile-scenarios.yml", - "type": 2, - "target_machines": [ - "gold-lin", - "gold-win", - "intel-lin", - "intel-win" - ], - "estimated_runtime": 10.0 - }, - { - "name": "SslStream", - "template": "sslstream-scenarios.yml", - "type": 2, - "target_machines": [ - "gold-lin", - "intel-lin" - ], - "estimated_runtime": null - }, - { - "name": "Trends", - "template": "trend-scenarios.yml", - "type": 2, - "target_machines": [ - "gold-lin", - "gold-win", - "intel-lin", - "intel-win", - "amd-lin2" - ], - "estimated_runtime": 20.0 - }, - { - "name": "Trends Database", - "template": "trend-database-scenarios.yml", - "type": 3, - "target_machines": [ - "gold-lin", - "gold-win", - "intel-lin", - "intel-win", - "amd-lin2" - ], - "estimated_runtime": 15.0 - }, - { - "name": "WebSockets", - "template": "websocket-scenarios.yml", - "type": 2, - "target_machines": [ - "gold-lin", - "gold-win", - "intel-lin", - "intel-win" - ], - "estimated_runtime": 6.0 - } - ] -} \ No newline at end of file diff --git a/build/benchmarks_ci_azure.json b/build/benchmarks_ci_azure.json deleted file mode 100644 index 20160bf17..000000000 --- a/build/benchmarks_ci_azure.json +++ /dev/null @@ -1,462 +0,0 @@ -{ - "metadata": { - "name": "Azure CI Benchmarks Configuration", - "description": "Combined machines and scenarios for continuous integration benchmarks on Azure", - "version": "1.0", - "schedule": "0 2/12 * * *", - "queues": [ - "azure", - "azurearm64" - ], - "yaml_generation": { - "target_yaml_count": 1, - "schedule_offset_hours": 6 - } - }, - "machines": [ - { - "name": "azure-db", - "machine_group": "azure", - "capabilities": { - "db": { - "priority": 1, - "profiles": [ - "azure-db-db" - ] - }, - "load": { - "priority": 2, - "profiles": [ - "azure-db-load" - ] - }, - "sut": { - "priority": 3, - "profiles": [ - "azure-db-app" - ] - } - } - }, - { - "name": "azure-client", - "machine_group": "azure", - "capabilities": { - "load": { - "priority": 1, - "profiles": [ - "azure-client-load" - ] - }, - "sut": { - "priority": 2, - "profiles": [ - "azure-client-app" - ] - }, - "db": { - "priority": 3, - "profiles": [ - "azure-client-db" - ] - } - } - }, - { - "name": "azure-server-arm64", - "machine_group": "azure", - "capabilities": { - "sut": { - "priority": 1, - "profiles": [ - "azure-server-arm64-app" - ] - }, - "load": { - "priority": 2, - "profiles": [ - "azure-server-arm64-load" - ] - }, - "db": { - "priority": 3, - "profiles": [ - "azure-server-arm64-db" - ] - } - }, - "preferred_partners": [ - "azure-client", - "azure-db" - ] - }, - { - "name": "azure2-client", - "machine_group": "azure2", - "capabilities": { - "load": { - "priority": 1, - "profiles": [ - "azure2-client-load" - ] - }, - "sut": { - "priority": 2, - "profiles": [ - "azure2-client-app" - ] - }, - "db": { - "priority": 3, - "profiles": [ - "azure2-client-db" - ] - } - } - }, - { - "name": "azure2-db", - "machine_group": "azure2", - "capabilities": { - "db": { - "priority": 1, - "profiles": [ - "azure2-db-db" - ] - }, - "sut": { - "priority": 2, - "profiles": [ - "azure2-db-app" - ] - }, - "load": { - "priority": 3, - "profiles": [ - "azure2-db-load" - ] - } - } - }, - { - "name": "azure2-server-amd64", - "machine_group": "azure2", - "capabilities": { - "sut": { - "priority": 1, - "profiles": [ - "azure2-server-amd64-app" - ] - }, - "load": { - "priority": 2, - "profiles": [ - "azure2-server-amd64-load" - ] - }, - "db": { - "priority": 3, - "profiles": [ - "azure2-server-amd64-db" - ] - } - }, - "preferred_partners": [ - "azure2-client", - "azure2-db" - ] - }, - { - "name": "idna-amd-lin", - "machine_group": "idna", - "capabilities": { - "sut": { - "priority": 1, - "profiles": [ - "idna-amd-lin-app" - ] - }, - "load": { - "priority": 2, - "profiles": [ - "idna-amd-lin-load" - ] - }, - "db": { - "priority": 3, - "profiles": [ - "idna-amd-lin-db" - ] - } - }, - "preferred_partners": [ - "idna-intel-lin", - "azure2-db" - ] - }, - { - "name": "idna-amd-win", - "machine_group": "idna", - "capabilities": { - "sut": { - "priority": 1, - "profiles": [ - "idna-amd-win-app" - ] - }, - "load": { - "priority": 2, - "profiles": [ - "idna-amd-win-load" - ] - }, - "db": { - "priority": 3, - "profiles": [ - "idna-amd-win-db" - ] - } - }, - "preferred_partners": [ - "idna-amd-lin", - "azure2-db" - ] - }, - { - "name": "idna-intel-lin", - "machine_group": "idna", - "capabilities": { - "sut": { - "priority": 1, - "profiles": [ - "idna-intel-lin-app" - ] - }, - "load": { - "priority": 2, - "profiles": [ - "idna-intel-lin-load" - ] - }, - "db": { - "priority": 3, - "profiles": [ - "idna-intel-lin-db" - ] - } - }, - "preferred_partners": [ - "idna-amd-lin", - "azure2-db" - ] - }, - { - "name": "idna-intel-win", - "machine_group": "idna", - "capabilities": { - "sut": { - "priority": 1, - "profiles": [ - "idna-intel-win-app" - ] - }, - "load": { - "priority": 2, - "profiles": [ - "idna-intel-win-load" - ] - }, - "db": { - "priority": 3, - "profiles": [ - "idna-intel-win-db" - ] - } - }, - "preferred_partners": [ - "idna-amd-lin", - "azure2-db" - ] - }, - { - "name": "cobalt-cloud-lin-server", - "machine_group": "cobalt-cloud", - "capabilities": { - "sut": { - "priority": 1, - "profiles": [ - "cobalt-cloud-lin-server-app" - ] - }, - "load": { - "priority": 2, - "profiles": [ - "cobalt-cloud-lin-server-load" - ] - }, - "db": { - "priority": 3, - "profiles": [ - "cobalt-cloud-lin-server-db" - ] - } - }, - "preferred_partners": [ - "cobalt-cloud-lin-client", - "cobalt-cloud-lin-db" - ] - }, - { - "name": "cobalt-cloud-lin-client", - "machine_group": "cobalt-cloud", - "capabilities": { - "load": { - "priority": 1, - "profiles": [ - "cobalt-cloud-lin-client-load" - ] - }, - "db": { - "priority": 2, - "profiles": [ - "cobalt-cloud-lin-client-db" - ] - }, - "sut": { - "priority": 3, - "profiles": [ - "cobalt-cloud-lin-client-app" - ] - } - } - }, - { - "name": "cobalt-cloud-lin-db", - "machine_group": "cobalt-cloud", - "capabilities": { - "db": { - "priority": 1, - "profiles": [ - "cobalt-cloud-lin-db-db" - ] - }, - "load": { - "priority": 2, - "profiles": [ - "cobalt-cloud-lin-db-load" - ] - }, - "sut": { - "priority": 3, - "profiles": [ - "cobalt-cloud-lin-db-app" - ] - } - } - }, - { - "name": "cobalt-cloud-lin-server-azure-linux3", - "machine_group": "cobalt-cloud", - "capabilities": { - "sut": { - "priority": 1, - "profiles": [ - "cobalt-cloud-lin-server-azure-linux3-app" - ] - }, - "load": { - "priority": 2, - "profiles": [ - "cobalt-cloud-lin-server-azure-linux3-load" - ] - }, - "db": { - "priority": 3, - "profiles": [ - "cobalt-cloud-lin-server-azure-linux3-db" - ] - } - }, - "preferred_partners": [ - "cobalt-cloud-lin-client", - "cobalt-cloud-lin-db" - ] - } - ], - "scenarios": [ - { - "name": "Baselines", - "template": "baselines-scenarios.yml", - "type": 2, - "target_machines": [ - "azure-server-arm64", - "azure2-server-amd64", - "cobalt-cloud-lin-server", - "cobalt-cloud-lin-server-azure-linux3" - ], - "estimated_runtime": 30.0 - }, - { - "name": "Baselines Database", - "template": "baselines-database-scenarios.yml", - "type": 3, - "target_machines": [ - "azure-server-arm64", - "azure2-server-amd64", - "cobalt-cloud-lin-server", - "cobalt-cloud-lin-server-azure-linux3" - ], - "estimated_runtime": 45.0 - }, - { - "name": "Containers", - "template": "containers-scenarios.yml", - "type": 3, - "target_machines": [ - "azure-server-arm64", - "azure2-server-amd64", - "cobalt-cloud-lin-server", - "cobalt-cloud-lin-server-azure-linux3" - ], - "estimated_runtime": 90.0 - }, - { - "name": "MVC", - "template": "mvc-scenarios.yml", - "type": 3, - "target_machines": [ - "cobalt-cloud-lin-server", - "cobalt-cloud-lin-server-azure-linux3" - ], - "estimated_runtime": 20.0 - }, - { - "name": "Trends", - "template": "trend-scenarios.yml", - "type": 2, - "target_machines": [ - "azure-server-arm64", - "azure2-server-amd64", - "cobalt-cloud-lin-server", - "cobalt-cloud-lin-server-azure-linux3", - "idna-amd-lin", - "idna-amd-win", - "idna-intel-lin", - "idna-intel-win" - ], - "estimated_runtime": 20.0 - }, - { - "name": "Trends Database", - "template": "trend-database-scenarios.yml", - "type": 3, - "target_machines": [ - "azure-server-arm64", - "azure2-server-amd64", - "cobalt-cloud-lin-server", - "cobalt-cloud-lin-server-azure-linux3" - ], - "estimated_runtime": 15.0 - } - ] -} diff --git a/build/benchmarks_ci_azure_pods.json b/build/benchmarks_ci_azure_pods.json new file mode 100644 index 000000000..fead30052 --- /dev/null +++ b/build/benchmarks_ci_azure_pods.json @@ -0,0 +1,107 @@ +{ + "metadata": { + "name": "Azure CI Benchmarks Configuration", + "description": "Pod-based scheduling for Azure CI benchmarks (includes cobalt cloud from eastus2)", + "version": "2.1", + "schedule": "0 2/12 * * *", + "queues": [ + "azure", + "azurearm64" + ], + "yaml_generation": { + "target_yaml_count": 1, + "schedule_offset_hours": 0 + } + }, + "pods": [ + { + "name": "azure-arm64", + "machines": { "sut": "azure-server-arm64", "load": "azure-client", "db": "azure-db" }, + "profiles": { "sut": "azure-server-arm64-app", "load": "azure-client-load", "db": "azure-db-db" } + }, + { + "name": "azure2-amd64", + "machines": { "sut": "azure2-server-amd64", "load": "azure2-client", "db": "azure2-db" }, + "profiles": { "sut": "azure2-server-amd64-app", "load": "azure2-client-load", "db": "azure2-db-db" } + }, + { + "name": "cobalt-cloud-lin", + "machines": { "sut": "cobalt-cloud-lin-server", "load": "cobalt-cloud-lin-client", "db": "cobalt-cloud-lin-db" }, + "profiles": { "sut": "cobalt-cloud-lin-server-app", "load": "cobalt-cloud-lin-client-load", "db": "cobalt-cloud-lin-db-db" } + }, + { + "name": "cobalt-cloud-lin-azl3", + "machines": { "sut": "cobalt-cloud-lin-server-azl3", "load": "cobalt-cloud-lin-client", "db": "cobalt-cloud-lin-db" }, + "profiles": { "sut": "cobalt-cloud-lin-server-azure-linux3-app", "load": "cobalt-cloud-lin-client-load", "db": "cobalt-cloud-lin-db-db" } + }, + { + "name": "cobalt-cloud-lin-azl3-dual", + "machines": { "sut": "cobalt-cloud-lin-server-azl3", "load": "cobalt-cloud-lin-db" }, + "profiles": { "sut": "cobalt-cloud-lin-server-azure-linux3-app", "load": "cobalt-cloud-lin-db-load" } + }, + { + "name": "idna-amd-lin", + "machines": { "sut": "idna-amd-lin", "load": "idna-intel-lin" }, + "profiles": { "sut": "idna-amd-lin-app", "load": "idna-intel-lin-load" } + }, + { + "name": "idna-amd-win", + "machines": { "sut": "idna-amd-win", "load": "azure2-db" }, + "profiles": { "sut": "idna-amd-win-app", "load": "azure2-db-load" } + }, + { + "name": "idna-intel-lin", + "machines": { "sut": "idna-intel-lin", "load": "idna-amd-lin" }, + "profiles": { "sut": "idna-intel-lin-app", "load": "idna-amd-lin-load" } + }, + { + "name": "idna-intel-win", + "machines": { "sut": "idna-intel-win", "load": "azure2-db" }, + "profiles": { "sut": "idna-intel-win-app", "load": "azure2-db-load" } + } + ], + "scenarios": [ + { + "name": "Containers", + "template": "containers-scenarios.yml", + "type": 3, + "pods": ["azure-arm64", "azure2-amd64", "cobalt-cloud-lin", "cobalt-cloud-lin-azl3"], + "estimated_runtime": 90.0 + }, + { + "name": "Baselines Database", + "template": "baselines-database-scenarios.yml", + "type": 3, + "pods": ["azure-arm64", "azure2-amd64", "cobalt-cloud-lin", "cobalt-cloud-lin-azl3"], + "estimated_runtime": 45.0 + }, + { + "name": "Baselines", + "template": "baselines-scenarios.yml", + "type": 2, + "pods": ["azure-arm64", "azure2-amd64", "cobalt-cloud-lin", "cobalt-cloud-lin-azl3-dual"], + "estimated_runtime": 30.0 + }, + { + "name": "MVC", + "template": "mvc-scenarios.yml", + "type": 3, + "pods": ["cobalt-cloud-lin", "cobalt-cloud-lin-azl3"], + "estimated_runtime": 20.0 + }, + { + "name": "Trends", + "template": "trend-scenarios.yml", + "type": 2, + "pods": ["azure-arm64", "azure2-amd64", "cobalt-cloud-lin", "cobalt-cloud-lin-azl3-dual", "idna-amd-lin", "idna-amd-win", "idna-intel-lin", "idna-intel-win"], + "estimated_runtime": 20.0 + }, + { + "name": "Trends Database", + "template": "trend-database-scenarios.yml", + "type": 3, + "pods": ["azure-arm64", "azure2-amd64", "cobalt-cloud-lin", "cobalt-cloud-lin-azl3"], + "estimated_runtime": 15.0 + } + ] +} diff --git a/build/benchmarks_ci_cobalt.json b/build/benchmarks_ci_cobalt.json deleted file mode 100644 index acf0b9677..000000000 --- a/build/benchmarks_ci_cobalt.json +++ /dev/null @@ -1,396 +0,0 @@ -{ - "metadata": { - "name": "CI Benchmarks Configuration", - "description": "Combined machines and scenarios for continuous integration benchmarks", - "version": "1.0", - "schedule": "0 3/12 * * *", - "queues": [ - "cobalthosted", - "cobalthosted_azurelinux3" - ], - "yaml_generation": { - "target_yaml_count": 1, - "schedule_offset_hours": 6 - } - }, - "machines": [ - { - "name": "cobalt-hosted-lin-server", - "capabilities": { - "sut": { - "priority": 1, - "profiles": [ - "cobalt-hosted-lin-server-app", - "cobalt-hosted-lin-server-28-app" - ] - } - }, - "preferred_partners": [ - "cobalt-hosted-lin-client", - "cobalt-hosted-lin-db" - ] - }, - { - "name": "cobalt-hosted-lin-client", - "capabilities": { - "load": { - "priority": 1, - "profiles": [ - "cobalt-hosted-lin-client-load" - ] - } - } - }, - { - "name": "cobalt-hosted-lin-db", - "capabilities": { - "db": { - "priority": 1, - "profiles": [ - "cobalt-hosted-lin-db-db" - ] - } - } - }, - { - "name": "cobalt-hosted-lin-server-azure-linux3", - "capabilities": { - "sut": { - "priority": 1, - "profiles": [ - "cobalt-hosted-lin-server-azure-linux3-app", - "cobalt-hosted-lin-server-azure-linux3-28-app" - ] - } - }, - "preferred_partners": [ - "cobalt-hosted-lin-client-azure-linux3", - "cobalt-hosted-lin-db-azure-linux3" - ] - }, - { - "name": "cobalt-hosted-lin-client-azure-linux3", - "capabilities": { - "load": { - "priority": 1, - "profiles": [ - "cobalt-hosted-lin-client-azure-linux3-load" - ] - } - } - }, - { - "name": "cobalt-hosted-lin-db-azure-linux3", - "capabilities": { - "db": { - "priority": 1, - "profiles": [ - "cobalt-hosted-lin-db-azure-linux3-db" - ] - } - } - } - ], - "scenarios": [ - { - "name": "Baselines", - "template": "baselines-scenarios.yml", - "type": 2, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 30.0 - }, - { - "name": "Baselines 28 Core", - "template": "baselines-scenarios.yml", - "type": 2, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 30.0, - "profile_overrides": { - "cobalt-hosted-lin-server": { - "sut": "cobalt-hosted-lin-server-28-app" - }, - "cobalt-hosted-lin-server-azure-linux3": { - "sut": "cobalt-hosted-lin-server-azure-linux3-28-app" - } - } - }, - { - "name": "Baselines Database", - "template": "baselines-database-scenarios.yml", - "type": 3, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 45.0 - }, - { - "name": "Baselines Database 28 Core", - "template": "baselines-database-scenarios.yml", - "type": 3, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 45.0, - "profile_overrides": { - "cobalt-hosted-lin-server": { - "sut": "cobalt-hosted-lin-server-28-app" - }, - "cobalt-hosted-lin-server-azure-linux3": { - "sut": "cobalt-hosted-lin-server-azure-linux3-28-app" - } - } - }, - { - "name": "Containers", - "template": "containers-scenarios.yml", - "type": 3, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 90.0 - }, - { - "name": "Crossgen", - "template": "crossgen2-scenarios.yml", - "type": 2, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 5.0 - }, - { - "name": "Crossgen 28 Core", - "template": "crossgen2-scenarios.yml", - "type": 2, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 5.0, - "profile_overrides": { - "cobalt-hosted-lin-server": { - "sut": "cobalt-hosted-lin-server-28-app" - }, - "cobalt-hosted-lin-server-azure-linux3": { - "sut": "cobalt-hosted-lin-server-azure-linux3-28-app" - } - } - }, - { - "name": "Frameworks", - "template": "frameworks-scenarios.yml", - "type": 3, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 15.0 - }, - { - "name": "Frameworks 28 Core", - "template": "frameworks-scenarios.yml", - "type": 3, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 15.0, - "profile_overrides": { - "cobalt-hosted-lin-server": { - "sut": "cobalt-hosted-lin-server-28-app" - }, - "cobalt-hosted-lin-server-azure-linux3": { - "sut": "cobalt-hosted-lin-server-azure-linux3-28-app" - } - } - }, - { - "name": "Frameworks Database", - "template": "frameworks-database-scenarios.yml", - "type": 3, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 20.0 - }, - { - "name": "Frameworks Database 28 Core", - "template": "frameworks-database-scenarios.yml", - "type": 3, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 20.0, - "profile_overrides": { - "cobalt-hosted-lin-server": { - "sut": "cobalt-hosted-lin-server-28-app" - }, - "cobalt-hosted-lin-server-azure-linux3": { - "sut": "cobalt-hosted-lin-server-azure-linux3-28-app" - } - } - }, - { - "name": "GC", - "template": "gc-scenarios.yml", - "type": 1, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 15.0 - }, - { - "name": "GC 28 Core", - "template": "gc-scenarios.yml", - "type": 1, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 15.0, - "profile_overrides": { - "cobalt-hosted-lin-server": { - "sut": "cobalt-hosted-lin-server-28-app" - }, - "cobalt-hosted-lin-server-azure-linux3": { - "sut": "cobalt-hosted-lin-server-azure-linux3-28-app" - } - } - }, - { - "name": "MVC", - "template": "mvc-scenarios.yml", - "type": 3, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 20.0 - }, - { - "name": "NativeAOT", - "template": "nativeaot-scenarios.yml", - "type": 3, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 20.0 - }, - { - "name": "NativeAOT 28 Core", - "template": "nativeaot-scenarios.yml", - "type": 3, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 20.0, - "profile_overrides": { - "cobalt-hosted-lin-server": { - "sut": "cobalt-hosted-lin-server-28-app" - }, - "cobalt-hosted-lin-server-azure-linux3": { - "sut": "cobalt-hosted-lin-server-azure-linux3-28-app" - } - } - }, - { - "name": "PGO", - "template": "pgo-scenarios.yml", - "type": 3, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 90.0 - }, - { - "name": "PGO 28 Core", - "template": "pgo-scenarios.yml", - "type": 3, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 90.0, - "profile_overrides": { - "cobalt-hosted-lin-server": { - "sut": "cobalt-hosted-lin-server-28-app" - }, - "cobalt-hosted-lin-server-azure-linux3": { - "sut": "cobalt-hosted-lin-server-azure-linux3-28-app" - } - } - }, - { - "name": "Trends", - "template": "trend-scenarios.yml", - "type": 2, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 20.0 - }, - { - "name": "Trends 28 Core", - "template": "trend-scenarios.yml", - "type": 2, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 20.0, - "profile_overrides": { - "cobalt-hosted-lin-server": { - "sut": "cobalt-hosted-lin-server-28-app" - }, - "cobalt-hosted-lin-server-azure-linux3": { - "sut": "cobalt-hosted-lin-server-azure-linux3-28-app" - } - } - }, - { - "name": "Trends Database", - "template": "trend-database-scenarios.yml", - "type": 3, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 15.0 - }, - { - "name": "Trends Database 28 Core", - "template": "trend-database-scenarios.yml", - "type": 3, - "target_machines": [ - "cobalt-hosted-lin-server", - "cobalt-hosted-lin-server-azure-linux3" - ], - "estimated_runtime": 15.0, - "profile_overrides": { - "cobalt-hosted-lin-server": { - "sut": "cobalt-hosted-lin-server-28-app" - }, - "cobalt-hosted-lin-server-azure-linux3": { - "sut": "cobalt-hosted-lin-server-azure-linux3-28-app" - } - } - } - ] -} \ No newline at end of file diff --git a/build/benchmarks_ci_cobalt_pods.json b/build/benchmarks_ci_cobalt_pods.json new file mode 100644 index 000000000..aed674790 --- /dev/null +++ b/build/benchmarks_ci_cobalt_pods.json @@ -0,0 +1,124 @@ +{ + "metadata": { + "name": "Cobalt Hosted CI Benchmarks Configuration", + "description": "Pod-based scheduling for cobalt hosted CI benchmarks", + "version": "2.0", + "schedule": "0 3/12 * * *", + "queues": [ + "cobalthosted", + "cobalthosted_azurelinux3" + ], + "yaml_generation": { + "target_yaml_count": 1, + "schedule_offset_hours": 0 + } + }, + "pods": [ + { + "name": "cobalt-hosted-lin", + "machines": { "sut": "cobalt-hosted-lin-server", "load": "cobalt-hosted-lin-client", "db": "cobalt-hosted-lin-db" }, + "profiles": { "sut": "cobalt-hosted-lin-server-app", "load": "cobalt-hosted-lin-client-load", "db": "cobalt-hosted-lin-db-db" } + }, + { + "name": "cobalt-hosted-lin-azl3", + "machines": { "sut": "cobalt-hosted-lin-server-azl3", "load": "cobalt-hosted-lin-client-azl3", "db": "cobalt-hosted-lin-db-azl3" }, + "profiles": { "sut": "cobalt-hosted-lin-server-azure-linux3-app", "load": "cobalt-hosted-lin-client-azure-linux3-load", "db": "cobalt-hosted-lin-db-azure-linux3-db" } + }, + { + "name": "cobalt-hosted-lin-28", + "machines": { "sut": "cobalt-hosted-lin-server", "load": "cobalt-hosted-lin-client", "db": "cobalt-hosted-lin-db" }, + "profiles": { "sut": "cobalt-hosted-lin-server-28-app", "load": "cobalt-hosted-lin-client-load", "db": "cobalt-hosted-lin-db-db" } + }, + { + "name": "cobalt-hosted-lin-azl3-28", + "machines": { "sut": "cobalt-hosted-lin-server-azl3", "load": "cobalt-hosted-lin-client-azl3", "db": "cobalt-hosted-lin-db-azl3" }, + "profiles": { "sut": "cobalt-hosted-lin-server-azure-linux3-28-app", "load": "cobalt-hosted-lin-client-azure-linux3-load", "db": "cobalt-hosted-lin-db-azure-linux3-db" } + } + ], + "scenarios": [ + { + "name": "Containers", + "template": "containers-scenarios.yml", + "type": 3, + "pods": ["cobalt-hosted-lin", "cobalt-hosted-lin-azl3"], + "estimated_runtime": 90.0 + }, + { + "name": "PGO", + "template": "pgo-scenarios.yml", + "type": 3, + "pods": ["cobalt-hosted-lin", "cobalt-hosted-lin-azl3", "cobalt-hosted-lin-28", "cobalt-hosted-lin-azl3-28"], + "estimated_runtime": 90.0 + }, + { + "name": "Baselines Database", + "template": "baselines-database-scenarios.yml", + "type": 3, + "pods": ["cobalt-hosted-lin", "cobalt-hosted-lin-azl3", "cobalt-hosted-lin-28", "cobalt-hosted-lin-azl3-28"], + "estimated_runtime": 45.0 + }, + { + "name": "Baselines", + "template": "baselines-scenarios.yml", + "type": 2, + "pods": ["cobalt-hosted-lin", "cobalt-hosted-lin-azl3", "cobalt-hosted-lin-28", "cobalt-hosted-lin-azl3-28"], + "estimated_runtime": 30.0 + }, + { + "name": "Frameworks Database", + "template": "frameworks-database-scenarios.yml", + "type": 3, + "pods": ["cobalt-hosted-lin", "cobalt-hosted-lin-azl3", "cobalt-hosted-lin-28", "cobalt-hosted-lin-azl3-28"], + "estimated_runtime": 20.0 + }, + { + "name": "MVC", + "template": "mvc-scenarios.yml", + "type": 3, + "pods": ["cobalt-hosted-lin", "cobalt-hosted-lin-azl3"], + "estimated_runtime": 20.0 + }, + { + "name": "NativeAOT", + "template": "nativeaot-scenarios.yml", + "type": 3, + "pods": ["cobalt-hosted-lin", "cobalt-hosted-lin-azl3", "cobalt-hosted-lin-28", "cobalt-hosted-lin-azl3-28"], + "estimated_runtime": 20.0 + }, + { + "name": "Trends", + "template": "trend-scenarios.yml", + "type": 2, + "pods": ["cobalt-hosted-lin", "cobalt-hosted-lin-azl3", "cobalt-hosted-lin-28", "cobalt-hosted-lin-azl3-28"], + "estimated_runtime": 20.0 + }, + { + "name": "Trends Database", + "template": "trend-database-scenarios.yml", + "type": 3, + "pods": ["cobalt-hosted-lin", "cobalt-hosted-lin-azl3", "cobalt-hosted-lin-28", "cobalt-hosted-lin-azl3-28"], + "estimated_runtime": 15.0 + }, + { + "name": "Crossgen", + "template": "crossgen2-scenarios.yml", + "type": 2, + "pods": ["cobalt-hosted-lin", "cobalt-hosted-lin-azl3", "cobalt-hosted-lin-28", "cobalt-hosted-lin-azl3-28"], + "estimated_runtime": 5.0 + }, + { + "name": "Frameworks", + "template": "frameworks-scenarios.yml", + "type": 3, + "pods": ["cobalt-hosted-lin", "cobalt-hosted-lin-azl3", "cobalt-hosted-lin-28", "cobalt-hosted-lin-azl3-28"], + "estimated_runtime": 15.0 + }, + { + "name": "GC", + "template": "gc-scenarios.yml", + "type": 1, + "pods": ["cobalt-hosted-lin", "cobalt-hosted-lin-azl3", "cobalt-hosted-lin-28", "cobalt-hosted-lin-azl3-28"], + "estimated_runtime": 15.0 + } + ] +} diff --git a/build/benchmarks_ci_pods.json b/build/benchmarks_ci_pods.json new file mode 100644 index 000000000..ad4e75104 --- /dev/null +++ b/build/benchmarks_ci_pods.json @@ -0,0 +1,213 @@ +{ + "metadata": { + "name": "CI Benchmarks Configuration", + "description": "Pod-based scheduling for continuous integration benchmarks", + "version": "2.0", + "schedule": "0 3/12 * * *", + "queues": [ + "citrine1", + "citrine2", + "citrine3", + "mono" + ], + "yaml_generation": { + "target_yaml_count": 2, + "schedule_offset_hours": 6 + } + }, + "pods": [ + { + "name": "intel-lin", + "machines": { "sut": "intel-lin", "load": "intel-load", "db": "intel-db" }, + "profiles": { "sut": "intel-lin-app", "load": "intel-load-load", "db": "intel-db-db" } + }, + { + "name": "intel-win", + "machines": { "sut": "intel-win", "load": "intel-load2", "db": "intel-db" }, + "profiles": { "sut": "intel-win-app", "load": "intel-load2-load", "db": "intel-db-db" } + }, + { + "name": "gold-lin", + "machines": { "sut": "gold-lin", "load": "gold-load", "db": "gold-db" }, + "profiles": { "sut": "gold-lin-app", "load": "gold-load-load", "db": "gold-db-db" } + }, + { + "name": "gold-win", + "machines": { "sut": "gold-win", "load": "gold-load2", "db": "gold-db" }, + "profiles": { "sut": "gold-win-app", "load": "gold-load2-load", "db": "gold-db-db" } + }, + { + "name": "amd-lin2", + "machines": { "sut": "amd-lin2", "load": "gold-load", "db": "gold-db" }, + "profiles": { "sut": "amd-lin2-app", "load": "gold-load-load", "db": "gold-db-db" } + }, + { + "name": "intel-perflin", + "machines": { "sut": "intel-perflin", "load": "intel-perfload" }, + "profiles": { "sut": "intel-perflin-app", "load": "intel-perfload-load" } + } + ], + "scenarios": [ + { + "name": "Baselines", + "template": "baselines-scenarios.yml", + "type": 2, + "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win", "amd-lin2"], + "estimated_runtime": 30.0 + }, + { + "name": "Baselines Database", + "template": "baselines-database-scenarios.yml", + "type": 3, + "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win", "amd-lin2"], + "estimated_runtime": 45.0 + }, + { + "name": "Blazor", + "template": "blazor-scenarios.yml", + "type": 2, + "pods": ["gold-lin", "intel-lin", "intel-perflin"], + "estimated_runtime": 45.0 + }, + { + "name": "Build", + "template": "build-perf-scenarios.yml", + "type": 1, + "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win"], + "estimated_runtime": 1.0 + }, + { + "name": "Containers", + "template": "containers-scenarios.yml", + "type": 3, + "pods": ["gold-lin", "intel-lin"], + "estimated_runtime": 90.0 + }, + { + "name": "Crossgen", + "template": "crossgen2-scenarios.yml", + "type": 2, + "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win", "amd-lin2"], + "estimated_runtime": 5.0 + }, + { + "name": "Custom Proxies", + "template": "proxies-custom-scenarios.yml", + "type": 3, + "pods": ["gold-lin", "intel-lin"], + "estimated_runtime": 5.0 + }, + { + "name": "EF Core", + "template": "efcore-scenarios.yml", + "type": 2, + "pods": ["gold-win", "intel-win"], + "estimated_runtime": 15.0 + }, + { + "name": "Frameworks", + "template": "frameworks-scenarios.yml", + "type": 3, + "pods": ["gold-lin", "intel-lin", "amd-lin2"], + "estimated_runtime": 15.0 + }, + { + "name": "Frameworks Database", + "template": "frameworks-database-scenarios.yml", + "type": 3, + "pods": ["gold-lin", "intel-lin", "amd-lin2"], + "estimated_runtime": 20.0 + }, + { + "name": "GC", + "template": "gc-scenarios.yml", + "type": 1, + "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win"], + "estimated_runtime": 15.0 + }, + { + "name": "Grpc", + "template": "grpc-scenarios.yml", + "type": 2, + "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win"], + "estimated_runtime": 70.0 + }, + { + "name": "HttpClient", + "template": "httpclient-scenarios.yml", + "type": 2, + "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win"], + "estimated_runtime": 45.0 + }, + { + "name": "MVC", + "template": "mvc-scenarios.yml", + "type": 3, + "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win"], + "estimated_runtime": 20.0 + }, + { + "name": "NativeAOT", + "template": "nativeaot-scenarios.yml", + "type": 3, + "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win"], + "estimated_runtime": 20.0 + }, + { + "name": "PGO", + "template": "pgo-scenarios.yml", + "type": 3, + "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win"], + "estimated_runtime": 90.0 + }, + { + "name": "Proxies", + "template": "proxies-scenarios.yml", + "type": 3, + "pods": ["gold-lin", "intel-lin"], + "estimated_runtime": 150.0 + }, + { + "name": "SignalR", + "template": "signalr-scenarios.yml", + "type": 2, + "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win"], + "estimated_runtime": 30.0 + }, + { + "name": "Single File", + "template": "singlefile-scenarios.yml", + "type": 2, + "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win"], + "estimated_runtime": 10.0 + }, + { + "name": "SslStream", + "template": "sslstream-scenarios.yml", + "type": 2, + "pods": ["gold-lin", "intel-lin"], + "estimated_runtime": 45.0 + }, + { + "name": "Trends", + "template": "trend-scenarios.yml", + "type": 2, + "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win", "amd-lin2"], + "estimated_runtime": 20.0 + }, + { + "name": "Trends Database", + "template": "trend-database-scenarios.yml", + "type": 3, + "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win", "amd-lin2"], + "estimated_runtime": 15.0 + }, + { + "name": "WebSockets", + "template": "websocket-scenarios.yml", + "type": 2, + "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win"], + "estimated_runtime": 6.0 + } + ] +} diff --git a/scripts/pod-scheduler/README.md b/scripts/pod-scheduler/README.md new file mode 100644 index 000000000..4e46f0b62 --- /dev/null +++ b/scripts/pod-scheduler/README.md @@ -0,0 +1,197 @@ +# Pod-Based Crank Scheduler + +A simplified scheduler for assigning benchmark scenarios to machine "pods" and +generating Azure DevOps pipeline YAML files. + +## Concept + +A **pod** is a fixed group of machines that always run together: +- **SUT** (System Under Test) — required +- **Load** generator — optional, for dual/triple scenarios +- **DB** (Database) — optional, for triple scenarios + +Pods that share physical machines (e.g., two pods using the same DB machine) +cannot run in the same stage. The scheduler handles this automatically. + +## Quick Start + +Run from the repository root: + +```bash +# Show schedule summary +python scripts/pod-scheduler/main.py --config build/benchmarks_ci_pods.json + +# Generate pipeline YAML files +python scripts/pod-scheduler/main.py \ + --config build/benchmarks_ci_pods.json \ + --yaml-output build + +# Regenerate the Azure or Cobalt pipelines +python scripts/pod-scheduler/main.py \ + --config build/benchmarks_ci_azure_pods.json \ + --base-name benchmarks-ci-azure --yaml-output build + +python scripts/pod-scheduler/main.py \ + --config build/benchmarks_ci_cobalt_pods.json \ + --base-name benchmarks-ci-cobalt --yaml-output build + +# Show which pods share machines +python scripts/pod-scheduler/main.py --config build/benchmarks_ci_pods.json --show-conflicts + +# List all runs without scheduling +python scripts/pod-scheduler/main.py --config build/benchmarks_ci_pods.json --list-runs +``` + +The header of every generated YAML embeds the exact regen command for that +file, so you can also copy the command from there. + +The scheduler runs in **strict mode** by default: any unknown pod or invalid +pod-for-scenario reference fails with a non-zero exit code so config typos +cannot silently drop scenarios from the pipeline. Pass `--lenient` to fall +back to the previous warn-and-skip behavior. + +Output is **deterministic**: identical input JSON always produces identical +YAML, so regenerations diff cleanly. To verify, run the snapshot tests: + +```bash +cd scripts/pod-scheduler +python -m unittest discover tests +``` + +## Configuration Format + +```json +{ + "metadata": { + "name": "Config Name", + "schedule": "0 3/12 * * *", + "queues": ["citrine1", "citrine2", "citrine3", "mono"], + "yaml_generation": { + "target_yaml_count": 2, + "schedule_offset_hours": 6 + }, + "pipeline": { + "pool": "server", + "service_bus_connection": "ASPNET Benchmarks Service Bus", + "service_bus_namespace": "aspnetbenchmarks" + } + }, + "pods": [ + { + "name": "gold-lin", + "machines": { "sut": "gold-lin", "load": "gold-load", "db": "gold-db" }, + "profiles": { "sut": "gold-lin-app", "load": "gold-load-load", "db": "gold-db-db" } + } + ], + "scenarios": [ + { + "name": "Baselines", + "template": "baselines-scenarios.yml", + "type": 2, + "pods": ["gold-lin", "gold-win"], + "estimated_runtime": 30.0, + "timeout": 120 + } + ] +} +``` + +The `pipeline` block is optional; defaults match the legacy hardcoded values. + +The `schedule` field's **hour** must be a `H` or `H/N` cron expression +(e.g. `3` or `3/12`). Lists, ranges, and `*` are rejected at load time so the +hour-offset used for split YAMLs cannot silently no-op. + +### Pod Definition + +| Field | Description | +|-------|-------------| +| `name` | Unique identifier for the pod | +| `machines.sut` | Physical machine name for SUT role | +| `machines.load` | Physical machine name for Load role (optional) | +| `machines.db` | Physical machine name for DB role (optional) | +| `profiles.sut` | Crank profile name for SUT | +| `profiles.load` | Crank profile name for Load (optional) | +| `profiles.db` | Crank profile name for DB (optional) | + +### Scenario Definition + +| Field | Description | +|-------|-------------| +| `name` | Display name (also used as part of the AzDO job id) | +| `template` | YAML scenario template to invoke | +| `type` | 1=SINGLE, 2=DUAL, 3=TRIPLE (see below) | +| `pods` | List of pod names this scenario targets (no duplicates) | +| `estimated_runtime` | Runtime estimate in minutes; defaults per type if omitted | +| `timeout` | Optional explicit AzDO `timeoutInMinutes` override. When unset, the generator picks `max(120, min(240, ceil(2 * estimated_runtime)))` | + +### Scenario Types + +| Type | Machines Used | Example | +|------|--------------|---------| +| 1 (SINGLE) | SUT only | Build, GC | +| 2 (DUAL) | SUT + Load | Baselines, Grpc, SignalR | +| 3 (TRIPLE) | SUT + Load + DB | Baselines Database, PGO, Proxies | + +### Queue Assignment + +The N-th run within a stage is assigned to `queues[N % len(queues)]`. Queues +are treated as interchangeable workers; if a queue is pinned to specific +hardware in your service-bus topology, set `metadata.queues` accordingly so +the order matches your hardware layout. + +### Handling Shared Machines + +Two pods can share load/DB machines. For example: +- `gold-lin` pod: SUT=gold-lin, Load=gold-load, DB=gold-db +- `gold-win` pod: SUT=gold-win, Load=gold-load2, DB=gold-db + +These pods share `gold-db`. When both run type-3 scenarios, they cannot be in +the same stage. When `gold-win` runs a type-2 scenario (no DB), there's no +conflict. + +### Future: Multiple SUTs per Class + +If you get 2 SUT machines of the same class (e.g., gold-lin-1 and gold-lin-2), +create separate pods for each. They can share load/DB: + +```json +{"name": "gold-lin-1", "machines": {"sut": "gold-lin-1", "load": "gold-load", "db": "gold-db"}, ...}, +{"name": "gold-lin-2", "machines": {"sut": "gold-lin-2", "load": "gold-load", "db": "gold-db"}, ...} +``` + +The scheduler automatically prevents them from running simultaneously when they +share load/DB machines. + +## Algorithm + +1. **Expand** each scenario × pod into individual "runs" +2. **Sort** runs by runtime descending (longest-job-first), with the run name + as a stable tie-breaker so output is deterministic +3. **Pack** into stages greedily — each run goes into the first stage where no + physical machines conflict and the queue limit isn't exceeded +4. **Split** stages across multiple YAML files using bin-packing for balanced + runtime, restoring the original stage order within each bin + +## Files + +| File | Purpose | +|------|---------| +| `main.py` | CLI entry point, summary display | +| `models.py` | Data classes (Pod, Scenario, Run, Stage, Schedule, PipelineSettings) | +| `scheduler.py` | Scheduling algorithm | +| `config_loader.py` | JSON config parser + validation | +| `generator.py` | YAML generation | +| `tests/` | Unit + snapshot tests (`python -m unittest`) | + +This is intentionally script-style: the modules use absolute imports +(`from models import …`) and are run as `python main.py …`. To run the +tests, `cd scripts/pod-scheduler && python -m unittest discover tests`. + +## Tradeoffs vs. the full crank-scheduler + +By collapsing capabilities, priorities, preferred-partners, and machine groups +into fixed pod definitions, this scheduler is much smaller — but loses some +expressivity. If the hardware layout grows beyond "fixed SUT + load + DB +triples", the constraint solver from the full crank-scheduler may be a better +fit. For today's hardware, the simplification is intentional. diff --git a/scripts/pod-scheduler/config_loader.py b/scripts/pod-scheduler/config_loader.py new file mode 100644 index 000000000..f5b1d6377 --- /dev/null +++ b/scripts/pod-scheduler/config_loader.py @@ -0,0 +1,134 @@ +""" +JSON configuration loader for pod-based scheduling. +""" + +import json +import re +from typing import Any, Dict + +from models import ( + PipelineSettings, + Pod, + Scenario, + ScenarioType, + ScheduleConfig, +) + + +class ConfigError(ValueError): + """Raised when a JSON config is malformed or self-inconsistent.""" + + +_CRON_HOUR_RE = re.compile(r"^\d+(/\d+)?$") + + +def _require(node: Dict[str, Any], key: str, context: str) -> Any: + if key not in node: + raise ConfigError(f"Missing required field '{key}' in {context}") + return node[key] + + +def _validate_cron(schedule: str) -> None: + """Confirm we can later offset the cron's hour field deterministically.""" + parts = schedule.split() + if len(parts) != 5: + raise ConfigError( + f"Schedule {schedule!r} is not a 5-field cron expression" + ) + if not _CRON_HOUR_RE.match(parts[1]): + raise ConfigError( + f"Schedule {schedule!r} uses an unsupported hour field " + f"{parts[1]!r}. Pod-scheduler only supports 'H' or 'H/N' here so " + f"it can offset the hour for split YAMLs without ambiguity." + ) + + +def load_config(path: str) -> ScheduleConfig: + """Load and validate a pod-scheduler JSON configuration file.""" + with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + + metadata = _require(data, "metadata", "config root") + schedule = _require(metadata, "schedule", "metadata") + _validate_cron(schedule) + + queues = _require(metadata, "queues", "metadata") + if not isinstance(queues, list) or not queues: + raise ConfigError("metadata.queues must be a non-empty list") + if len(queues) != len(set(queues)): + raise ConfigError(f"metadata.queues contains duplicates: {queues}") + + yaml_gen = metadata.get("yaml_generation", {}) + + pipeline_meta = metadata.get("pipeline", {}) + pipeline = PipelineSettings( + pool=pipeline_meta.get("pool", PipelineSettings.pool), + service_bus_connection=pipeline_meta.get( + "service_bus_connection", + PipelineSettings.service_bus_connection, + ), + service_bus_namespace=pipeline_meta.get( + "service_bus_namespace", + PipelineSettings.service_bus_namespace, + ), + ) + + pods: Dict[str, Pod] = {} + raw_pods = _require(data, "pods", "config root") + for pod_data in raw_pods: + pod_name = _require(pod_data, "name", "pod entry") + if pod_name in pods: + raise ConfigError(f"Duplicate pod name: {pod_name!r}") + machines = _require(pod_data, "machines", f"pod '{pod_name}'") + profiles = _require(pod_data, "profiles", f"pod '{pod_name}'") + pods[pod_name] = Pod( + name=pod_name, + sut=_require(machines, "sut", f"pod '{pod_name}'.machines"), + load=machines.get("load"), + db=machines.get("db"), + sut_profile=_require(profiles, "sut", f"pod '{pod_name}'.profiles"), + load_profile=profiles.get("load"), + db_profile=profiles.get("db"), + ) + + scenarios = [] + raw_scenarios = _require(data, "scenarios", "config root") + for sc_data in raw_scenarios: + name = _require(sc_data, "name", "scenario entry") + scenario_pods = _require(sc_data, "pods", f"scenario '{name}'") + if not scenario_pods: + raise ConfigError(f"scenario '{name}' has empty pods list") + if len(scenario_pods) != len(set(scenario_pods)): + dupes = sorted({ + p for p in scenario_pods if scenario_pods.count(p) > 1 + }) + raise ConfigError( + f"scenario '{name}' lists duplicate pods: {dupes}" + ) + runtime_raw = sc_data.get("estimated_runtime") or 0 + timeout = sc_data.get("timeout") + if timeout is not None: + timeout = int(timeout) + if timeout <= 0: + raise ConfigError( + f"scenario '{name}' has non-positive timeout {timeout}" + ) + scenarios.append(Scenario( + name=name, + template=_require(sc_data, "template", f"scenario '{name}'"), + type=ScenarioType(_require(sc_data, "type", f"scenario '{name}'")), + pods=list(scenario_pods), + estimated_runtime=float(runtime_raw) if runtime_raw else 0.0, + timeout=timeout, + )) + + return ScheduleConfig( + name=metadata.get("name", ""), + schedule=schedule, + queues=list(queues), + target_yaml_count=yaml_gen.get("target_yaml_count", 1), + schedule_offset_hours=yaml_gen.get("schedule_offset_hours", 6), + pods=pods, + scenarios=scenarios, + pipeline=pipeline, + ) diff --git a/scripts/pod-scheduler/generator.py b/scripts/pod-scheduler/generator.py new file mode 100644 index 000000000..efde9d19a --- /dev/null +++ b/scripts/pod-scheduler/generator.py @@ -0,0 +1,246 @@ +""" +YAML generator for pod-based scheduling. + +Converts schedule data into Azure DevOps pipeline YAML files. The output +matches the structure previously produced by the Liquid-template based +crank-scheduler so existing pipeline consumers continue to work. +""" + +import os +import re +from typing import Any, Dict, List, Optional + +from models import ( + PipelineSettings, + Run, + Schedule, + ScheduleConfig, + sanitize_job_id, +) + + +_CRON_HOUR_RE = re.compile(r"^(\d+)(/\d+)?$") + + +class GeneratorError(ValueError): + """Raised when YAML generation cannot proceed safely.""" + + +def _offset_cron(cron: str, offset_hours: int) -> str: + """Offset a cron schedule's hour field by the given hours. + + Only ``H`` and ``H/N`` hour fields are supported; anything else (lists, + ranges, ``*`` etc.) raises :class:`GeneratorError` rather than silently + no-op'ing, which would otherwise cause every split YAML to fire at the + same time. + """ + parts = cron.split() + if len(parts) != 5: + raise GeneratorError( + f"Cron {cron!r} is not a 5-field expression; cannot offset" + ) + + hour_part = parts[1] + match = _CRON_HOUR_RE.match(hour_part) + if not match: + raise GeneratorError( + f"Cron {cron!r} uses an unsupported hour field {hour_part!r}; " + f"only 'H' or 'H/N' may be safely offset for split YAMLs" + ) + + base = int(match.group(1)) + suffix = match.group(2) or "" + new_base = (base + offset_hours) % 24 + parts[1] = f"{new_base}{suffix}" + return " ".join(parts) + + +def _job_timeout(run: Run) -> int: + """Pick a YAML timeout for the run. + + If the scenario specifies a timeout explicitly, use it. Otherwise derive + one from estimated_runtime (capped at [120, 240] minutes) so jobs that + historically took longer than the old flat 120-minute default still get + enough headroom. + """ + if run.scenario.timeout is not None: + return run.scenario.timeout + return max(120, min(240, int(run.estimated_runtime * 2))) + + +def schedule_to_template_data( + schedule: Schedule, + config: ScheduleConfig, + cron_override: Optional[str] = None, +) -> Dict[str, Any]: + """Project a Schedule into a JSON-serialisable shape useful for tests.""" + groups = [] + for stage in schedule.stages: + jobs = [] + for run in stage.runs: + jobs.append({ + "name": run.name, + "job_id": run.job_name, + "template": run.scenario.template, + "profiles": run.profiles, + "timeout": _job_timeout(run), + }) + groups.append({"jobs": jobs}) + + return { + "schedule": cron_override or config.schedule, + "queues": config.queues, + "groups": groups, + } + + +def _render_yaml( + data: Dict[str, Any], + pipeline: PipelineSettings, + source_config: Optional[str] = None, + base_name: str = "benchmarks-ci", +) -> str: + """Render the template data into Azure DevOps pipeline YAML.""" + lines: List[str] = [] + lines.append("# Do not change this file, it is generated by the pod-scheduler.") + lines.append("# Source of truth: see ../scripts/pod-scheduler/README.md") + lines.append("# To regenerate, run from the repo root:") + if source_config: + lines.append( + f"# python ./scripts/pod-scheduler/main.py " + f"--config {source_config} --base-name {base_name} " + f"--yaml-output ./build" + ) + else: + lines.append( + "# python ./scripts/pod-scheduler/main.py " + "--config ./build/.json " + "--yaml-output ./build" + ) + lines.append("") + lines.append("trigger: none") + lines.append("pr: none") + lines.append("") + lines.append("schedules:") + lines.append(f'- cron: "{data["schedule"]}"') + lines.append(" always: true") + lines.append(" branches:") + lines.append(" include:") + lines.append(" - main") + lines.append("") + lines.append("variables:") + lines.append(" - template: job-variables.yml") + lines.append(" - name: session") + lines.append(" value: $(Build.BuildNumber)") + lines.append(" - name: buildId") + lines.append(" value: $(Build.BuildId)") + lines.append(" - name: buildNumber") + lines.append(" value: $(Build.BuildNumber)") + lines.append(" - name: am") + lines.append(" value: $[lt(format('{0:HH}', pipeline.startTime), 12)]") + lines.append(" - name: pm") + lines.append(" value: $[ge(format('{0:HH}', pipeline.startTime), 12)]") + lines.append("") + lines.append("jobs:") + lines.append("") + + queues = data["queues"] + prev_group_jobs: List[str] = [] + seen_job_ids = set() + + for group_idx, group in enumerate(data["groups"]): + group_num = group_idx + 1 + lines.append(f"# GROUP {group_num}") + lines.append("") + + current_jobs: List[str] = [] + for job_idx, job in enumerate(group["jobs"]): + job_id = job["job_id"] + if job_id in seen_job_ids: + raise GeneratorError( + f"Duplicate job id {job_id!r} produced for jobs named " + f"{job['name']!r}. Rename the offending pod or scenario." + ) + seen_job_ids.add(job_id) + current_jobs.append(job_id) + + queue = queues[job_idx % len(queues)] + depends = ", ".join(prev_group_jobs) if prev_group_jobs else "" + profiles_args = " ".join( + f"--profile {p}" for p in job["profiles"] + ) + + lines.append(f"- job: {job_id}") + lines.append(f" displayName: {group_num}- {job['name']}") + lines.append(f" pool: {pipeline.pool}") + lines.append(f" timeoutInMinutes: {job['timeout']}") + lines.append(f" dependsOn: [{depends}]") + lines.append(" condition: succeededOrFailed()") + lines.append(" steps:") + lines.append(f" - template: {job['template']}") + lines.append(" parameters:") + lines.append( + f" connection: {pipeline.service_bus_connection}" + ) + lines.append(f" serviceBusQueueName: {queue}") + lines.append( + f" serviceBusNamespace: {pipeline.service_bus_namespace}" + ) + lines.append( + f' arguments: "$(ciProfile) {profiles_args} "' + ) + lines.append("") + + prev_group_jobs = current_jobs + + return "\n".join(lines) + "\n" + + +def generate_yamls( + schedules: List[Schedule], + config: ScheduleConfig, + output_dir: str, + base_name: str = "benchmarks-ci", + source_config: Optional[str] = None, +) -> List[str]: + """Generate YAML pipeline files for each sub-schedule. + + All job ids are sanitized via :func:`sanitize_job_id` and verified to be + unique within the produced YAML; collisions raise GeneratorError instead + of producing AzDO-invalid output. + + ``source_config`` is the path to the JSON config that produced the + schedule. When provided, it's embedded in the generated YAML header so + each file documents the exact command needed to regenerate it. + """ + os.makedirs(output_dir, exist_ok=True) + output_files = [] + + for i, sched in enumerate(schedules): + # Pre-flight: ensure every run already has a sanitized id. + for stage in sched.stages: + for run in stage.runs: + _ = sanitize_job_id(run.name) + + suffix = f"-{i + 1:02d}" if len(schedules) > 1 else "" + filename = f"{base_name}{suffix}.yml" + filepath = os.path.join(output_dir, filename) + + cron = _offset_cron( + config.schedule, config.schedule_offset_hours * i + ) + data = schedule_to_template_data(sched, config, cron_override=cron) + yaml_content = _render_yaml( + data, + config.pipeline, + source_config=source_config, + base_name=base_name, + ) + + with open(filepath, "w", newline="\n", encoding="utf-8") as f: + f.write(yaml_content) + + output_files.append(filepath) + print(f" Generated: {filepath}") + + return output_files diff --git a/scripts/pod-scheduler/main.py b/scripts/pod-scheduler/main.py new file mode 100644 index 000000000..00cdc0236 --- /dev/null +++ b/scripts/pod-scheduler/main.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python3 +""" +Pod-based crank scheduler. + +Assigns benchmark scenarios to machine pods and generates Azure DevOps +pipeline YAML files. Pods define fixed machine groupings (SUT + load + DB), +and the scheduler ensures no physical machine is double-booked per stage. + +Usage: + python main.py --config ./build/benchmarks_ci_pods.json + python main.py --config ./build/benchmarks_ci_pods.json \\ + --yaml-output ./build +""" + +import argparse +import json +import os +import sys +from typing import List + +from config_loader import ConfigError, load_config +from generator import GeneratorError, generate_yamls, schedule_to_template_data +from models import Schedule, ScheduleConfig +from scheduler import ( + SchedulerError, + create_schedule, + expand_runs, + split_schedule, +) + + +def print_summary(config: ScheduleConfig, schedule: Schedule) -> None: + """Print a human-readable schedule summary.""" + print(f"\n{'=' * 70}") + print(f"SCHEDULE SUMMARY: {config.name}") + print(f"{'=' * 70}") + print(f" Pods: {len(config.pods)}") + print(f" Scenarios: {len(config.scenarios)}") + print(f" Total runs: {schedule.total_runs}") + print(f" Stages: {len(schedule.stages)}") + print(f" Queues: {len(config.queues)} ({', '.join(config.queues)})") + print(f" Est. total time: {schedule.total_duration:.0f} min " + f"({schedule.total_duration / 60:.1f} hrs)") + print() + + machine_time = {} + machine_total = {} + for stage in schedule.stages: + stage_machines = set() + for run in stage.runs: + for m in run.machines_used: + stage_machines.add(m) + machine_time.setdefault(m, 0) + machine_time[m] += run.estimated_runtime + for m in stage_machines: + machine_total.setdefault(m, 0) + machine_total[m] += stage.duration + + if machine_total: + print("MACHINE UTILIZATION:") + for m in sorted(machine_total.keys()): + busy = machine_time.get(m, 0) + total = machine_total.get(m, 1) + pct = (busy / total * 100) if total > 0 else 0 + filled = int(pct / 5) + bar = "#" * filled + "." * (20 - filled) + print(f" {m:<25} {bar} {pct:5.1f}% " + f"({busy:.0f}/{total:.0f} min)") + print() + + print("STAGE BREAKDOWN:") + for i, stage in enumerate(schedule.stages): + print(f"\n Stage {i} (Duration: {stage.duration:.0f} min)") + print(f" {'Queue':<8} {'Scenario':<40} {'Runtime':>8} " + f"Machines") + print(f" {'-' * 80}") + for j, run in enumerate(stage.runs): + queue = config.queues[j % len(config.queues)] + machines = ", ".join(sorted(run.machines_used)) + print(f" {queue:<8} {run.name:<40} {run.estimated_runtime:>6.0f}m " + f"{machines}") + print() + + +def print_split_summary(schedules: List[Schedule], config: ScheduleConfig) -> None: + """Print summary of multi-YAML split.""" + if len(schedules) <= 1: + return + print(f"YAML SPLIT ({len(schedules)} files):") + for i, sched in enumerate(schedules): + print(f" YAML {i + 1}: {len(sched.stages)} stages, " + f"{sched.total_runs} runs, " + f"{sched.total_duration:.0f} min") + print() + + +def print_pod_conflicts(config: ScheduleConfig) -> None: + """Show which pods share physical machines (potential conflicts).""" + machine_pods = {} + for pod in config.pods.values(): + for role in ["sut", "load", "db"]: + machine = getattr(pod, role) + if machine: + machine_pods.setdefault(machine, []).append( + (pod.name, role) + ) + + shared = {m: pods for m, pods in machine_pods.items() if len(pods) > 1} + if shared: + print("SHARED MACHINES (pods that cannot run simultaneously):") + for machine, pods in sorted(shared.items()): + pod_strs = [f"{name}({role})" for name, role in pods] + print(f" {machine:<20} used by: {', '.join(pod_strs)}") + print() + + +def _format_source_path(path: str) -> str: + """Render a config path for embedding in generated YAML headers. + + Tries to express the path relative to the repo root using forward slashes + so the regen command is identical on Windows and POSIX shells. Falls back + to ``./`` if the path is outside the repo. + """ + repo_root = os.path.abspath( + os.path.join(os.path.dirname(__file__), "..", "..") + ) + abs_path = os.path.abspath(path) + try: + rel = os.path.relpath(abs_path, repo_root) + except ValueError: + rel = os.path.basename(abs_path) + rel = rel.replace(os.sep, "/") + if rel.startswith("../"): + rel = os.path.basename(abs_path) + return f"./{rel}" + + +def _build_arg_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Pod-based crank scheduler" + ) + parser.add_argument( + "--config", required=True, + help="Path to JSON configuration file" + ) + parser.add_argument( + "--yaml-output", + help="Directory to write generated YAML files" + ) + parser.add_argument( + "--base-name", default="benchmarks-ci", + help="Base filename for generated YAMLs (default: benchmarks-ci)" + ) + parser.add_argument( + "--target-yamls", type=int, + help="Override number of YAML files to generate" + ) + parser.add_argument( + "--template-data", action="store_true", + help="Print template data as JSON (for debugging)" + ) + parser.add_argument( + "--list-runs", action="store_true", + help="List all expanded runs without scheduling" + ) + parser.add_argument( + "--show-conflicts", action="store_true", + help="Show pods that share physical machines" + ) + parser.add_argument( + "--lenient", action="store_true", + help="Warn instead of fail on unknown or invalid pod references. " + "Off by default so config typos do not silently drop scenarios." + ) + return parser + + +def main(argv: List[str] = None) -> int: + parser = _build_arg_parser() + args = parser.parse_args(argv) + + try: + print(f"Loading config: {args.config}") + config = load_config(args.config) + print(f" Loaded {len(config.pods)} pods, " + f"{len(config.scenarios)} scenarios") + + strict = not args.lenient + + if args.show_conflicts: + print_pod_conflicts(config) + + if args.list_runs: + runs = expand_runs(config, strict=strict) + print(f"\nAll runs ({len(runs)} total):") + for r in runs: + machines = ", ".join(sorted(r.machines_used)) + print(f" {r.name:<45} type={r.scenario.type.value} " + f"runtime={r.estimated_runtime:.0f}m " + f"machines=[{machines}]") + return 0 + + schedule = create_schedule(config, strict=strict) + print_summary(config, schedule) + print_pod_conflicts(config) + + yaml_count = args.target_yamls or config.target_yaml_count + schedules = split_schedule(schedule, yaml_count) + print_split_summary(schedules, config) + + if args.template_data: + for i, sched in enumerate(schedules): + data = schedule_to_template_data(sched, config) + print(f"\n--- Template data for YAML {i + 1} ---") + print(json.dumps(data, indent=2)) + + if args.yaml_output: + print(f"Generating {len(schedules)} YAML file(s)...") + generate_yamls( + schedules, config, args.yaml_output, + base_name=args.base_name, + source_config=_format_source_path(args.config), + ) + print("Done!") + return 0 + except (ConfigError, SchedulerError, GeneratorError) as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/pod-scheduler/models.py b/scripts/pod-scheduler/models.py new file mode 100644 index 000000000..22ec8503f --- /dev/null +++ b/scripts/pod-scheduler/models.py @@ -0,0 +1,194 @@ +""" +Data models for pod-based crank scheduling. + +A "pod" is a fixed group of machines (SUT + optional load + optional DB) that +always run together. Pods sharing physical machines cannot run simultaneously, +which the scheduler enforces automatically. +""" + +import re +from dataclasses import dataclass, field +from enum import IntEnum +from typing import Dict, List, Optional, Set + + +# Default per-type runtime estimates (minutes) used when a scenario provides +# none of its own. Lives here so models, scheduler, and tests share one source. +DEFAULT_RUNTIMES: Dict["ScenarioType", float] = {} # populated below + +# Pipeline plumbing defaults. Override in JSON metadata.pipeline.* if needed. +DEFAULT_PIPELINE_POOL = "server" +DEFAULT_PIPELINE_CONNECTION = "ASPNET Benchmarks Service Bus" +DEFAULT_PIPELINE_NAMESPACE = "aspnetbenchmarks" + +# AzDO job identifier rule. Letters/digits/underscore, must not start with a +# digit, no longer than 100 characters. +JOB_ID_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,99}$") + + +class ScenarioType(IntEnum): + """Number of machine roles required for a scenario.""" + SINGLE = 1 # SUT only + DUAL = 2 # SUT + Load + TRIPLE = 3 # SUT + Load + DB + + +DEFAULT_RUNTIMES.update({ + ScenarioType.SINGLE: 30.0, + ScenarioType.DUAL: 45.0, + ScenarioType.TRIPLE: 60.0, +}) + + +def sanitize_job_id(raw: str) -> str: + """Sanitize an arbitrary string into a valid AzDO job identifier. + + Replaces any character that isn't [A-Za-z0-9_] with '_', collapses runs of + underscores, prefixes a leading digit with '_', and truncates at 100 + characters. The result is guaranteed to match ``JOB_ID_RE``. + """ + cleaned = re.sub(r"[^A-Za-z0-9_]", "_", raw) + cleaned = re.sub(r"_+", "_", cleaned).strip("_") or "_" + if cleaned[0].isdigit(): + cleaned = "_" + cleaned + cleaned = cleaned[:100] + if not JOB_ID_RE.match(cleaned): + # Should be unreachable, but assert keeps callers honest. + raise ValueError(f"Could not sanitize {raw!r} into a valid job id") + return cleaned + + +@dataclass +class Pod: + """A fixed group of machines that run scenarios together.""" + name: str + # Physical machine names for each role + sut: str + load: Optional[str] = None + db: Optional[str] = None + # Crank profile names for each role + sut_profile: str = "" + load_profile: Optional[str] = None + db_profile: Optional[str] = None + + def machines_for_type(self, scenario_type: ScenarioType) -> Set[str]: + """Return the set of physical machines used for a given scenario type.""" + machines = {self.sut} + if scenario_type >= ScenarioType.DUAL and self.load: + machines.add(self.load) + if scenario_type >= ScenarioType.TRIPLE and self.db: + machines.add(self.db) + return machines + + def profiles_for_type(self, scenario_type: ScenarioType) -> List[str]: + """Return the ordered list of profiles for a given scenario type.""" + profiles = [self.sut_profile] + if scenario_type >= ScenarioType.DUAL and self.load_profile: + profiles.append(self.load_profile) + if scenario_type >= ScenarioType.TRIPLE and self.db_profile: + profiles.append(self.db_profile) + return profiles + + def validate(self, scenario_type: ScenarioType) -> Optional[str]: + """Check if this pod can run the given scenario type. Returns error or None.""" + if scenario_type >= ScenarioType.DUAL and not self.load: + return f"Pod '{self.name}' has no load machine for DUAL/TRIPLE scenario" + if scenario_type >= ScenarioType.TRIPLE and not self.db: + return f"Pod '{self.name}' has no db machine for TRIPLE scenario" + return None + + +@dataclass +class Scenario: + """A benchmark scenario that runs on one or more pods.""" + name: str + template: str + type: ScenarioType + pods: List[str] + estimated_runtime: float = 45.0 + # Optional explicit timeout (minutes) for the generated AzDO job. When + # None, the generator derives one from estimated_runtime. + timeout: Optional[int] = None + + +@dataclass +class Run: + """A single execution: one scenario on one pod.""" + scenario: Scenario + pod: Pod + estimated_runtime: float + + @property + def name(self) -> str: + return f"{self.scenario.name} {self.pod.name}" + + @property + def job_name(self) -> str: + """Sanitized identifier suitable for AzDO ``- job:`` use.""" + return sanitize_job_id(self.name) + + @property + def machines_used(self) -> Set[str]: + return self.pod.machines_for_type(self.scenario.type) + + @property + def profiles(self) -> List[str]: + return self.pod.profiles_for_type(self.scenario.type) + + +@dataclass +class Stage: + """A group of runs that execute in parallel (no machine conflicts).""" + runs: List[Run] = field(default_factory=list) + + @property + def machines_in_use(self) -> Set[str]: + result: Set[str] = set() + for run in self.runs: + result |= run.machines_used + return result + + @property + def duration(self) -> float: + return max((r.estimated_runtime for r in self.runs), default=0) + + def can_add(self, run: Run, queue_count: int) -> bool: + """True if the run fits without machine conflicts or queue overflow.""" + if len(self.runs) >= queue_count: + return False + return run.machines_used.isdisjoint(self.machines_in_use) + + +@dataclass +class Schedule: + """Complete schedule: ordered list of stages.""" + stages: List[Stage] = field(default_factory=list) + + @property + def total_duration(self) -> float: + return sum(s.duration for s in self.stages) + + @property + def total_runs(self) -> int: + return sum(len(s.runs) for s in self.stages) + + +@dataclass +class PipelineSettings: + """Pipeline-level plumbing values rendered into the YAML.""" + pool: str = DEFAULT_PIPELINE_POOL + service_bus_connection: str = DEFAULT_PIPELINE_CONNECTION + service_bus_namespace: str = DEFAULT_PIPELINE_NAMESPACE + + +@dataclass +class ScheduleConfig: + """Top-level configuration loaded from JSON.""" + name: str + schedule: str + queues: List[str] + target_yaml_count: int + schedule_offset_hours: int + pods: Dict[str, Pod] + scenarios: List[Scenario] + pipeline: PipelineSettings = field(default_factory=PipelineSettings) diff --git a/scripts/pod-scheduler/scheduler.py b/scripts/pod-scheduler/scheduler.py new file mode 100644 index 000000000..c86284bea --- /dev/null +++ b/scripts/pod-scheduler/scheduler.py @@ -0,0 +1,121 @@ +""" +Core scheduling algorithm for pod-based crank scheduling. + +Uses a greedy longest-job-first approach with machine-collision avoidance. +Since pods define fixed machine groupings, the scheduler only needs to +ensure no physical machine is used twice in the same stage. + +Output is deterministic: identical input JSON always produces identical +schedules, so generated YAML files diff cleanly across regenerations. +""" + +from typing import List, Tuple + +from models import ( + DEFAULT_RUNTIMES, + Run, + Schedule, + ScheduleConfig, + Stage, +) + + +class SchedulerError(ValueError): + """Raised when the scheduler refuses to build a schedule.""" + + +def expand_runs(config: ScheduleConfig, strict: bool = True) -> List[Run]: + """Expand scenarios x pods into individual runs. + + With ``strict=True`` (the default) an unknown pod or a pod that cannot + satisfy a scenario's type raises :class:`SchedulerError`. With + ``strict=False`` the offending entry is skipped and a warning is printed. + """ + runs: List[Run] = [] + for scenario in config.scenarios: + for pod_name in scenario.pods: + pod = config.pods.get(pod_name) + if pod is None: + msg = ( + f"Scenario '{scenario.name}' references unknown pod " + f"'{pod_name}'" + ) + if strict: + raise SchedulerError(msg) + print(f" WARNING: {msg}, skipping") + continue + error = pod.validate(scenario.type) + if error: + msg = f"{error} for scenario '{scenario.name}'" + if strict: + raise SchedulerError(msg) + print(f" WARNING: {msg}, skipping") + continue + runtime = scenario.estimated_runtime + if runtime <= 0: + runtime = DEFAULT_RUNTIMES.get(scenario.type, 45.0) + runs.append(Run( + scenario=scenario, + pod=pod, + estimated_runtime=runtime, + )) + return runs + + +def create_schedule(config: ScheduleConfig, strict: bool = True) -> Schedule: + """Create a schedule by greedy longest-job-first packing. + + 1. Expand all scenario x pod combinations into runs. + 2. Sort by runtime descending (longest-job-first heuristic). + 3. Greedily pack runs into stages, checking machine collisions. + + Sort key includes the run name as a tie-breaker so the result is stable. + """ + runs = expand_runs(config, strict=strict) + queue_count = len(config.queues) + if queue_count == 0: + raise SchedulerError( + "Cannot schedule with zero queues. Configure metadata.queues." + ) + + runs.sort(key=lambda r: (-r.estimated_runtime, r.name)) + + schedule = Schedule() + for run in runs: + for stage in schedule.stages: + if stage.can_add(run, queue_count): + stage.runs.append(run) + break + else: + schedule.stages.append(Stage(runs=[run])) + + return schedule + + +def split_schedule(schedule: Schedule, target_count: int) -> List[Schedule]: + """Split a schedule into multiple sub-schedules using bin-packing. + + Stages are packed longest-first into the lightest bin to balance total + runtime, then each bin's stages are restored to their original ordering + so the generated YAML files preserve scenario sequence. + """ + if target_count <= 1: + return [schedule] + + indexed: List[Tuple[int, Stage]] = list(enumerate(schedule.stages)) + indexed.sort(key=lambda pair: -pair[1].duration) + + bins: List[List[Tuple[int, Stage]]] = [[] for _ in range(target_count)] + bin_durations = [0.0] * target_count + for original_index, stage in indexed: + target = min(range(target_count), key=lambda i: bin_durations[i]) + bins[target].append((original_index, stage)) + bin_durations[target] += stage.duration + + result: List[Schedule] = [] + for entries in bins: + if not entries: + continue + entries.sort(key=lambda pair: pair[0]) + result.append(Schedule(stages=[stage for _, stage in entries])) + return result diff --git a/scripts/pod-scheduler/tests/__init__.py b/scripts/pod-scheduler/tests/__init__.py new file mode 100644 index 000000000..54b55fbcc --- /dev/null +++ b/scripts/pod-scheduler/tests/__init__.py @@ -0,0 +1,14 @@ +"""Pod-scheduler tests. + +Each test module adds the parent ``scripts/pod-scheduler`` directory to +``sys.path`` so it can ``import models`` etc. without requiring the package +to be installed. +""" + +import os +import sys + +_HERE = os.path.dirname(os.path.abspath(__file__)) +_PARENT = os.path.dirname(_HERE) +if _PARENT not in sys.path: + sys.path.insert(0, _PARENT) \ No newline at end of file diff --git a/scripts/pod-scheduler/tests/test_config_loader.py b/scripts/pod-scheduler/tests/test_config_loader.py new file mode 100644 index 000000000..9dd698939 --- /dev/null +++ b/scripts/pod-scheduler/tests/test_config_loader.py @@ -0,0 +1,117 @@ +import json +import os +import tempfile +import unittest + +import tests # noqa: F401 # ensures sys.path is set up + +from config_loader import ConfigError, load_config + + +_BASE = { + "metadata": { + "name": "t", + "schedule": "0 3/12 * * *", + "queues": ["a", "b"], + }, + "pods": [ + { + "name": "p1", + "machines": {"sut": "m1"}, + "profiles": {"sut": "m1-app"}, + } + ], + "scenarios": [ + { + "name": "S", + "template": "s.yml", + "type": 1, + "pods": ["p1"], + } + ], +} + + +def _write(tmp, payload): + path = os.path.join(tmp, "cfg.json") + with open(path, "w", encoding="utf-8") as f: + json.dump(payload, f) + return path + + +class TestLoadConfig(unittest.TestCase): + def test_happy_path(self): + with tempfile.TemporaryDirectory() as tmp: + path = _write(tmp, _BASE) + cfg = load_config(path) + self.assertIn("p1", cfg.pods) + self.assertEqual(len(cfg.scenarios), 1) + self.assertEqual(cfg.pipeline.pool, "server") + + def test_unsupported_cron_rejected(self): + with tempfile.TemporaryDirectory() as tmp: + payload = json.loads(json.dumps(_BASE)) + payload["metadata"]["schedule"] = "0 * * * *" + path = _write(tmp, payload) + with self.assertRaises(ConfigError): + load_config(path) + + def test_empty_queues_rejected(self): + with tempfile.TemporaryDirectory() as tmp: + payload = json.loads(json.dumps(_BASE)) + payload["metadata"]["queues"] = [] + path = _write(tmp, payload) + with self.assertRaises(ConfigError): + load_config(path) + + def test_duplicate_pods_rejected(self): + with tempfile.TemporaryDirectory() as tmp: + payload = json.loads(json.dumps(_BASE)) + payload["pods"].append(dict(payload["pods"][0])) + path = _write(tmp, payload) + with self.assertRaises(ConfigError): + load_config(path) + + def test_duplicate_pods_in_scenario_rejected(self): + with tempfile.TemporaryDirectory() as tmp: + payload = json.loads(json.dumps(_BASE)) + payload["scenarios"][0]["pods"] = ["p1", "p1"] + path = _write(tmp, payload) + with self.assertRaises(ConfigError): + load_config(path) + + def test_pipeline_settings_loaded(self): + with tempfile.TemporaryDirectory() as tmp: + payload = json.loads(json.dumps(_BASE)) + payload["metadata"]["pipeline"] = { + "pool": "custompool", + "service_bus_namespace": "myns", + } + path = _write(tmp, payload) + cfg = load_config(path) + self.assertEqual(cfg.pipeline.pool, "custompool") + self.assertEqual(cfg.pipeline.service_bus_namespace, "myns") + self.assertEqual( + cfg.pipeline.service_bus_connection, + "ASPNET Benchmarks Service Bus", + ) + + def test_optional_timeout_loaded(self): + with tempfile.TemporaryDirectory() as tmp: + payload = json.loads(json.dumps(_BASE)) + payload["scenarios"][0]["timeout"] = 199 + path = _write(tmp, payload) + cfg = load_config(path) + self.assertEqual(cfg.scenarios[0].timeout, 199) + + def test_negative_timeout_rejected(self): + with tempfile.TemporaryDirectory() as tmp: + payload = json.loads(json.dumps(_BASE)) + payload["scenarios"][0]["timeout"] = 0 + path = _write(tmp, payload) + with self.assertRaises(ConfigError): + load_config(path) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/pod-scheduler/tests/test_generator.py b/scripts/pod-scheduler/tests/test_generator.py new file mode 100644 index 000000000..91a59f617 --- /dev/null +++ b/scripts/pod-scheduler/tests/test_generator.py @@ -0,0 +1,81 @@ +import os +import unittest + +import tests # noqa: F401 # ensures sys.path is set up + +from generator import GeneratorError, _job_timeout, _offset_cron +from main import _format_source_path +from models import Pod, Run, Scenario, ScenarioType + + +class TestOffsetCron(unittest.TestCase): + def test_offset_h_field(self): + self.assertEqual(_offset_cron("0 3 * * *", 6), "0 9 * * *") + + def test_offset_h_slash_n(self): + self.assertEqual(_offset_cron("0 3/12 * * *", 6), "0 9/12 * * *") + + def test_offset_wraps_at_24(self): + self.assertEqual(_offset_cron("0 22 * * *", 6), "0 4 * * *") + + def test_offset_zero_is_identity(self): + self.assertEqual(_offset_cron("0 3/12 * * *", 0), "0 3/12 * * *") + + def test_unsupported_hour_field_raises(self): + for cron in [ + "0 * * * *", + "0 0,12 * * *", + "0 1-5 * * *", + "0 */6 * * *", + ]: + with self.assertRaises(GeneratorError, msg=cron): + _offset_cron(cron, 6) + + def test_wrong_field_count_raises(self): + with self.assertRaises(GeneratorError): + _offset_cron("0 3 * *", 6) + + +class TestJobTimeout(unittest.TestCase): + def _run(self, runtime, timeout=None): + scenario = Scenario( + name="s", template="s.yml", type=ScenarioType.SINGLE, + pods=["p"], estimated_runtime=runtime, timeout=timeout, + ) + pod = Pod(name="p", sut="sut", sut_profile="sut") + return Run(scenario=scenario, pod=pod, estimated_runtime=runtime) + + def test_uses_explicit_timeout(self): + self.assertEqual(_job_timeout(self._run(30, timeout=42)), 42) + + def test_short_runtime_floors_at_120(self): + self.assertEqual(_job_timeout(self._run(5)), 120) + + def test_long_runtime_caps_at_240(self): + self.assertEqual(_job_timeout(self._run(200)), 240) + + def test_mid_runtime_doubles(self): + self.assertEqual(_job_timeout(self._run(90)), 180) + + +class TestFormatSourcePath(unittest.TestCase): + def test_paths_in_repo_become_repo_relative(self): + repo_root = os.path.abspath( + os.path.join(os.path.dirname(__file__), "..", "..", "..") + ) + candidate = os.path.join(repo_root, "build", "benchmarks_ci_pods.json") + self.assertEqual( + _format_source_path(candidate), + "./build/benchmarks_ci_pods.json", + ) + + def test_outside_repo_falls_back_to_basename(self): + # Use a path that's definitely outside the repo by jumping above root. + far = os.path.abspath(os.sep + "definitely-not-in-repo.json") + self.assertEqual( + _format_source_path(far), "./definitely-not-in-repo.json" + ) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/scripts/pod-scheduler/tests/test_models.py b/scripts/pod-scheduler/tests/test_models.py new file mode 100644 index 000000000..a0884f957 --- /dev/null +++ b/scripts/pod-scheduler/tests/test_models.py @@ -0,0 +1,106 @@ +import unittest + +import tests # noqa: F401 # ensures sys.path is set up + +from models import ( + JOB_ID_RE, + Pod, + Run, + Scenario, + ScenarioType, + Stage, + sanitize_job_id, +) + + +class TestSanitizeJobId(unittest.TestCase): + def test_replaces_spaces_and_hyphens(self): + self.assertEqual(sanitize_job_id("Proxies gold-lin"), "Proxies_gold_lin") + + def test_collapses_multiple_separators(self): + self.assertEqual( + sanitize_job_id("Foo --bar..baz"), "Foo_bar_baz" + ) + + def test_prefixes_leading_digit(self): + self.assertEqual(sanitize_job_id("01-stage"), "_01_stage") + + def test_handles_unicode_and_punctuation(self): + result = sanitize_job_id("Frënch (test)/v2") + self.assertRegex(result, JOB_ID_RE) + self.assertNotIn(" ", result) + self.assertNotIn("/", result) + self.assertNotIn("(", result) + + def test_truncates_to_100(self): + long_name = "a" * 250 + result = sanitize_job_id(long_name) + self.assertEqual(len(result), 100) + + def test_result_always_matches_pattern(self): + for sample in ["x.y", "1abc", " hello ", "---", "a/b/c"]: + self.assertRegex(sanitize_job_id(sample), JOB_ID_RE, sample) + + +class TestPodValidation(unittest.TestCase): + def _pod(self, **kwargs): + defaults = dict( + name="p", sut="sut", + sut_profile="sut-app", + ) + defaults.update(kwargs) + return Pod(**defaults) + + def test_single_only_pod_rejects_dual(self): + pod = self._pod() + self.assertIsNone(pod.validate(ScenarioType.SINGLE)) + self.assertIsNotNone(pod.validate(ScenarioType.DUAL)) + self.assertIsNotNone(pod.validate(ScenarioType.TRIPLE)) + + def test_dual_pod_rejects_triple(self): + pod = self._pod(load="load", load_profile="load-load") + self.assertIsNone(pod.validate(ScenarioType.DUAL)) + self.assertIsNotNone(pod.validate(ScenarioType.TRIPLE)) + + def test_triple_pod_accepts_all(self): + pod = self._pod( + load="load", load_profile="load-load", + db="db", db_profile="db-db", + ) + for t in ScenarioType: + self.assertIsNone(pod.validate(t), t) + + +class TestStageCanAdd(unittest.TestCase): + def _run(self, name, sut, load=None, db=None, runtime=10): + scenario = Scenario( + name=name, + template=f"{name}.yml", + type=ScenarioType.TRIPLE if db else ( + ScenarioType.DUAL if load else ScenarioType.SINGLE + ), + pods=["p"], + estimated_runtime=runtime, + ) + pod = Pod( + name="p", sut=sut, load=load, db=db, + sut_profile="sut", load_profile="load", db_profile="db", + ) + return Run(scenario=scenario, pod=pod, estimated_runtime=runtime) + + def test_collision_blocks(self): + stage = Stage(runs=[self._run("a", sut="m1")]) + self.assertFalse(stage.can_add(self._run("b", sut="m1"), 10)) + + def test_no_collision_allows(self): + stage = Stage(runs=[self._run("a", sut="m1")]) + self.assertTrue(stage.can_add(self._run("b", sut="m2"), 10)) + + def test_queue_limit_enforced(self): + stage = Stage(runs=[self._run(f"a{i}", sut=f"m{i}") for i in range(3)]) + self.assertFalse(stage.can_add(self._run("a3", sut="m3"), queue_count=3)) + self.assertTrue(stage.can_add(self._run("a3", sut="m3"), queue_count=4)) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/pod-scheduler/tests/test_scheduler.py b/scripts/pod-scheduler/tests/test_scheduler.py new file mode 100644 index 000000000..0c52c44c6 --- /dev/null +++ b/scripts/pod-scheduler/tests/test_scheduler.py @@ -0,0 +1,160 @@ +import unittest + +import tests # noqa: F401 # ensures sys.path is set up + +from models import ( + PipelineSettings, + Pod, + Scenario, + ScenarioType, + Schedule, + ScheduleConfig, + Stage, +) +from scheduler import ( + SchedulerError, + create_schedule, + expand_runs, + split_schedule, +) + + +def _config(pods, scenarios, queues=("q1", "q2")): + return ScheduleConfig( + name="t", + schedule="0 3/12 * * *", + queues=list(queues), + target_yaml_count=1, + schedule_offset_hours=6, + pods={p.name: p for p in pods}, + scenarios=list(scenarios), + pipeline=PipelineSettings(), + ) + + +def _pod(name, sut, load=None, db=None): + return Pod( + name=name, sut=sut, load=load, db=db, + sut_profile=f"{sut}-app", + load_profile=f"{load}-load" if load else None, + db_profile=f"{db}-db" if db else None, + ) + + +def _scn(name, type_, pods, runtime=10): + return Scenario( + name=name, template=f"{name}.yml", + type=type_, pods=list(pods), estimated_runtime=runtime, + ) + + +class TestExpandRuns(unittest.TestCase): + def test_strict_raises_on_unknown_pod(self): + cfg = _config( + pods=[_pod("p1", "m1")], + scenarios=[_scn("S", ScenarioType.SINGLE, ["missing"])], + ) + with self.assertRaises(SchedulerError): + expand_runs(cfg, strict=True) + + def test_lenient_skips_unknown_pod(self): + cfg = _config( + pods=[_pod("p1", "m1")], + scenarios=[_scn("S", ScenarioType.SINGLE, ["missing", "p1"])], + ) + runs = expand_runs(cfg, strict=False) + self.assertEqual([r.pod.name for r in runs], ["p1"]) + + def test_strict_raises_on_invalid_type(self): + cfg = _config( + pods=[_pod("p1", "m1")], # no load + scenarios=[_scn("S", ScenarioType.DUAL, ["p1"])], + ) + with self.assertRaises(SchedulerError): + expand_runs(cfg, strict=True) + + +class TestCreateSchedule(unittest.TestCase): + def test_collisions_split_into_stages(self): + cfg = _config( + pods=[_pod("p1", "m1")], + scenarios=[ + _scn("A", ScenarioType.SINGLE, ["p1"], runtime=5), + _scn("B", ScenarioType.SINGLE, ["p1"], runtime=10), + ], + ) + sched = create_schedule(cfg) + self.assertEqual(len(sched.stages), 2) + + def test_independent_pods_share_a_stage(self): + cfg = _config( + pods=[_pod("p1", "m1"), _pod("p2", "m2")], + scenarios=[ + _scn("A", ScenarioType.SINGLE, ["p1"]), + _scn("B", ScenarioType.SINGLE, ["p2"]), + ], + ) + sched = create_schedule(cfg) + self.assertEqual(len(sched.stages), 1) + self.assertEqual(len(sched.stages[0].runs), 2) + + def test_queue_count_limits_stage(self): + cfg = _config( + pods=[_pod(f"p{i}", f"m{i}") for i in range(3)], + scenarios=[ + _scn(f"S{i}", ScenarioType.SINGLE, [f"p{i}"]) for i in range(3) + ], + queues=("q1", "q2"), + ) + sched = create_schedule(cfg) + self.assertEqual(len(sched.stages), 2) + + def test_zero_queues_raises(self): + cfg = _config( + pods=[_pod("p1", "m1")], + scenarios=[_scn("S", ScenarioType.SINGLE, ["p1"])], + queues=(), + ) + with self.assertRaises(SchedulerError): + create_schedule(cfg) + + def test_deterministic_output(self): + cfg = _config( + pods=[_pod("p1", "m1"), _pod("p2", "m2")], + scenarios=[ + _scn("A", ScenarioType.SINGLE, ["p1"], runtime=10), + _scn("B", ScenarioType.SINGLE, ["p2"], runtime=10), + ], + ) + s1 = create_schedule(cfg) + s2 = create_schedule(cfg) + names_of = lambda s: [ + [r.name for r in stage.runs] for stage in s.stages + ] + self.assertEqual(names_of(s1), names_of(s2)) + + +class TestSplitSchedule(unittest.TestCase): + def test_single_target_returns_input(self): + sched = Schedule(stages=[Stage(runs=[])]) + self.assertEqual(split_schedule(sched, 1), [sched]) + + def test_balances_durations(self): + cfg = _config( + pods=[_pod("p1", "m1")], + scenarios=[ + _scn("A", ScenarioType.SINGLE, ["p1"], runtime=30), + _scn("B", ScenarioType.SINGLE, ["p1"], runtime=20), + _scn("C", ScenarioType.SINGLE, ["p1"], runtime=10), + ], + ) + sched = create_schedule(cfg) + self.assertEqual(len(sched.stages), 3) + bins = split_schedule(sched, 2) + self.assertEqual(len(bins), 2) + durations = sorted(b.total_duration for b in bins) + self.assertEqual(durations, [30, 30]) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/pod-scheduler/tests/test_snapshots.py b/scripts/pod-scheduler/tests/test_snapshots.py new file mode 100644 index 000000000..849430daf --- /dev/null +++ b/scripts/pod-scheduler/tests/test_snapshots.py @@ -0,0 +1,79 @@ +"""Snapshot tests: regenerate the committed YAML files and compare. + +Run from the ``scripts/pod-scheduler`` directory: + + python -m unittest discover tests +""" + +import os +import tempfile +import unittest + +import tests # noqa: F401 # ensures sys.path is set up + +from config_loader import load_config +from generator import generate_yamls +from scheduler import create_schedule, split_schedule + + +_HERE = os.path.dirname(os.path.abspath(__file__)) +_REPO = os.path.abspath(os.path.join(_HERE, "..", "..", "..")) +_BUILD = os.path.join(_REPO, "build") + + +CASES = [ + ("benchmarks_ci_pods.json", "benchmarks-ci", + ["benchmarks-ci-01.yml", "benchmarks-ci-02.yml"]), + ("benchmarks_ci_azure_pods.json", "benchmarks-ci-azure", + ["benchmarks-ci-azure.yml"]), + ("benchmarks_ci_cobalt_pods.json", "benchmarks-ci-cobalt", + ["benchmarks-ci-cobalt.yml"]), +] + + +class TestSnapshots(unittest.TestCase): + def test_each_config_produces_committed_yaml(self): + for config_name, base_name, expected_files in CASES: + with self.subTest(config=config_name): + config_path = os.path.join(_BUILD, config_name) + self.assertTrue( + os.path.exists(config_path), f"missing {config_path}" + ) + config = load_config(config_path) + schedule = create_schedule(config) + schedules = split_schedule(schedule, config.target_yaml_count) + + with tempfile.TemporaryDirectory() as tmp: + generate_yamls( + schedules, config, tmp, base_name=base_name, + source_config=f"./build/{config_name}", + ) + for name in expected_files: + generated = os.path.join(tmp, name) + committed = os.path.join(_BUILD, name) + self.assertTrue( + os.path.exists(generated), + f"generator did not produce {generated}", + ) + self.assertTrue( + os.path.exists(committed), + f"committed YAML missing: {committed}", + ) + with open(generated, encoding="utf-8") as f: + gen_text = f.read() + with open(committed, encoding="utf-8") as f: + comm_text = f.read() + self.assertEqual( + gen_text, + comm_text, + f"{name} differs from committed snapshot. " + f"Run from repo root: python " + f"scripts/pod-scheduler/main.py " + f"--config build/{config_name} " + f"--yaml-output build " + f"--base-name {base_name}", + ) + + +if __name__ == "__main__": + unittest.main() From 6f252aaa4afd233f29d24b36f470de6e9952aa59 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 4 May 2026 15:18:59 -0700 Subject: [PATCH 4/5] Rewrite framework dependency diff URLs to dotnet/dotnet (VMR) (#2168) After the move to the VMR (dotnet/dotnet), the .NET runtime and ASP.NET runtime are built from dotnet/dotnet, and the commit hashes Crank captures are dotnet/dotnet commits. The Crank agent still hard-codes the old component-repo URLs (dotnet/aspnetcore, dotnet/runtime) for the synthetic Microsoft.AspNetCore.App and Microsoft.NETCore.App framework dependencies, so the rendered compare URLs in regression issues 404 (e.g. issue dotnet/aspnetcore#66568). Rewrite the URL at template-render time inside the Changes block of all five issue-body templates (rps, published-size, start-time in regressions.config.yml; download-size, first-ui in regressions.blazor.config.yml). The rewrite triggers when the dependency name is one of the two synthetic framework names, or when the stored RepositoryUrl is one of the legacy component-repo URLs. Other dependencies (e.g. application-owned assemblies like Antiforgery.dll, or already-correct dotnet/dotnet entries from post-VMR assembly metadata) are left untouched. Verified by parsing both YAML configs through Fluid and rendering each template against synthetic regressions covering all five diff cases: 0 leftover dotnet/aspnetcore or dotnet/runtime URLs, preserved aspnet/Benchmarks URL on application assemblies, framework deps correctly attributed to dotnet/dotnet. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- build/regressions.blazor.config.yml | 24 +++++++++++++++++-- build/regressions.config.yml | 36 ++++++++++++++++++++++++++--- 2 files changed, 55 insertions(+), 5 deletions(-) diff --git a/build/regressions.blazor.config.yml b/build/regressions.blazor.config.yml index 3148b5c97..1ac31a7fa 100644 --- a/build/regressions.blazor.config.yml +++ b/build/regressions.blazor.config.yml @@ -68,7 +68,17 @@ templates: {%- if diff.RepositoryUrl == empty -%} `{{ diff.PreviousCommitHash | slice: 0, 7 }}...{{ diff.CurrentCommitHash | slice: 0, 7 }}` {%- else -%} - {{ diff.RepositoryUrl }}/compare/{{ diff.PreviousCommitHash }}...{{ diff.CurrentCommitHash }} + {%- assign repoUrl = diff.RepositoryUrl -%} + {%- comment -%} + Post-VMR (https://github.com/dotnet/dotnet), framework-level commits live in + dotnet/dotnet rather than the component repos. Rewrite the diff URL when the + dependency is one of the synthetic framework deps that Crank still records + with the legacy component-repo URL. + {%- endcomment -%} + {%- if diff.Names contains 'Microsoft.AspNetCore.App' or diff.Names contains 'Microsoft.NETCore.App' or repoUrl == 'https://github.com/dotnet/aspnetcore' or repoUrl == 'https://github.com/dotnet/runtime' -%} + {%- assign repoUrl = 'https://github.com/dotnet/dotnet' -%} + {%- endif -%} + {{ repoUrl }}/compare/{{ diff.PreviousCommitHash }}...{{ diff.CurrentCommitHash }} {%- endif -%} {%- else -%} `{{ diff.PreviousCommitHash | slice: 0, 7 }}...{{ diff.CurrentCommitHash | slice: 0, 7 }}` @@ -115,7 +125,17 @@ templates: {%- if diff.RepositoryUrl == empty -%} `{{ diff.PreviousCommitHash | slice: 0, 7 }}...{{ diff.CurrentCommitHash | slice: 0, 7 }}` {%- else -%} - {{ diff.RepositoryUrl }}/compare/{{ diff.PreviousCommitHash }}...{{ diff.CurrentCommitHash }} + {%- assign repoUrl = diff.RepositoryUrl -%} + {%- comment -%} + Post-VMR (https://github.com/dotnet/dotnet), framework-level commits live in + dotnet/dotnet rather than the component repos. Rewrite the diff URL when the + dependency is one of the synthetic framework deps that Crank still records + with the legacy component-repo URL. + {%- endcomment -%} + {%- if diff.Names contains 'Microsoft.AspNetCore.App' or diff.Names contains 'Microsoft.NETCore.App' or repoUrl == 'https://github.com/dotnet/aspnetcore' or repoUrl == 'https://github.com/dotnet/runtime' -%} + {%- assign repoUrl = 'https://github.com/dotnet/dotnet' -%} + {%- endif -%} + {{ repoUrl }}/compare/{{ diff.PreviousCommitHash }}...{{ diff.CurrentCommitHash }} {%- endif -%} {%- else -%} `{{ diff.PreviousCommitHash | slice: 0, 7 }}...{{ diff.CurrentCommitHash | slice: 0, 7 }}` diff --git a/build/regressions.config.yml b/build/regressions.config.yml index dc7f2ca03..2afe2a25a 100644 --- a/build/regressions.config.yml +++ b/build/regressions.config.yml @@ -205,7 +205,17 @@ templates: {%- if diff.RepositoryUrl == empty -%} `{{ diff.PreviousCommitHash | slice: 0, 7 }}...{{ diff.CurrentCommitHash | slice: 0, 7 }}` {%- else -%} - {{ diff.RepositoryUrl }}/compare/{{ diff.PreviousCommitHash }}...{{ diff.CurrentCommitHash }} + {%- assign repoUrl = diff.RepositoryUrl -%} + {%- comment -%} + Post-VMR (https://github.com/dotnet/dotnet), framework-level commits live in + dotnet/dotnet rather than the component repos. Rewrite the diff URL when the + dependency is one of the synthetic framework deps that Crank still records + with the legacy component-repo URL. + {%- endcomment -%} + {%- if diff.Names contains 'Microsoft.AspNetCore.App' or diff.Names contains 'Microsoft.NETCore.App' or repoUrl == 'https://github.com/dotnet/aspnetcore' or repoUrl == 'https://github.com/dotnet/runtime' -%} + {%- assign repoUrl = 'https://github.com/dotnet/dotnet' -%} + {%- endif -%} + {{ repoUrl }}/compare/{{ diff.PreviousCommitHash }}...{{ diff.CurrentCommitHash }} {%- endif -%} {%- else -%} `{{ diff.PreviousCommitHash | slice: 0, 7 }}...{{ diff.CurrentCommitHash | slice: 0, 7 }}` @@ -253,7 +263,17 @@ templates: {%- if diff.RepositoryUrl == empty -%} `{{ diff.PreviousCommitHash | slice: 0, 7 }}...{{ diff.CurrentCommitHash | slice: 0, 7 }}` {%- else -%} - {{ diff.RepositoryUrl }}/compare/{{ diff.PreviousCommitHash }}...{{ diff.CurrentCommitHash }} + {%- assign repoUrl = diff.RepositoryUrl -%} + {%- comment -%} + Post-VMR (https://github.com/dotnet/dotnet), framework-level commits live in + dotnet/dotnet rather than the component repos. Rewrite the diff URL when the + dependency is one of the synthetic framework deps that Crank still records + with the legacy component-repo URL. + {%- endcomment -%} + {%- if diff.Names contains 'Microsoft.AspNetCore.App' or diff.Names contains 'Microsoft.NETCore.App' or repoUrl == 'https://github.com/dotnet/aspnetcore' or repoUrl == 'https://github.com/dotnet/runtime' -%} + {%- assign repoUrl = 'https://github.com/dotnet/dotnet' -%} + {%- endif -%} + {{ repoUrl }}/compare/{{ diff.PreviousCommitHash }}...{{ diff.CurrentCommitHash }} {%- endif -%} {%- else -%} `{{ diff.PreviousCommitHash | slice: 0, 7 }}...{{ diff.CurrentCommitHash | slice: 0, 7 }}` @@ -300,7 +320,17 @@ templates: {%- if diff.RepositoryUrl == empty -%} `{{ diff.PreviousCommitHash | slice: 0, 7 }}...{{ diff.CurrentCommitHash | slice: 0, 7 }}` {%- else -%} - {{ diff.RepositoryUrl }}/compare/{{ diff.PreviousCommitHash }}...{{ diff.CurrentCommitHash }} + {%- assign repoUrl = diff.RepositoryUrl -%} + {%- comment -%} + Post-VMR (https://github.com/dotnet/dotnet), framework-level commits live in + dotnet/dotnet rather than the component repos. Rewrite the diff URL when the + dependency is one of the synthetic framework deps that Crank still records + with the legacy component-repo URL. + {%- endcomment -%} + {%- if diff.Names contains 'Microsoft.AspNetCore.App' or diff.Names contains 'Microsoft.NETCore.App' or repoUrl == 'https://github.com/dotnet/aspnetcore' or repoUrl == 'https://github.com/dotnet/runtime' -%} + {%- assign repoUrl = 'https://github.com/dotnet/dotnet' -%} + {%- endif -%} + {{ repoUrl }}/compare/{{ diff.PreviousCommitHash }}...{{ diff.CurrentCommitHash }} {%- endif -%} {%- else -%} `{{ diff.PreviousCommitHash | slice: 0, 7 }}...{{ diff.CurrentCommitHash | slice: 0, 7 }}` From 8451f357b05f68efa0f006a9077b65cd61b4525d Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Thu, 7 May 2026 13:09:19 -0700 Subject: [PATCH 5/5] Switch pod-scheduler configs from JSON to YAML; add JSON Schema The three pod configs in build/ were JSON, which made them hard to read and didn't allow inline comments. This converts them to YAML, ships a JSON Schema describing the shape, and changes scenario.type from a magic 1/2/3 integer to a self-documenting single/dual/triple string. * build/benchmarks_ci_pods.yml, benchmarks_ci_azure_pods.yml, and benchmarks_ci_cobalt_pods.yml replace their .json predecessors. Each opens with a yaml-language-server # yaml-language-server: schema=... directive so VS Code / Cursor / the YAML LSP provide autocomplete, hover docs, and inline validation while editing. * scripts/pod-scheduler/pod-config.schema.json (new) is the schema. The schema descriptions also serve as reliable grounding for LLM-driven edits. * config_loader.py now dispatches on file extension and accepts both YAML and JSON. scenario.type accepts single|dual|triple (case-insensitive) plus the legacy integer 1|2|3 for back-compat; bools (which YAML parsers happily produce from yes/no) are rejected explicitly. Unknown extensions and unknown type strings raise ConfigError so typos can't silently drop scenarios. * Generated benchmarks-ci-*.yml files only change the embedded regen command in the file header (.json -> .yml). Schedule data is byte-identical, verified by the existing snapshot test. * Tests: 5 new cases covering YAML loading, type-string aliases, type back-compat, bool/invalid rejection, and unknown-extension rejection. Default fixture writer now produces YAML so the YAML path is exercised on every test. * Docs: README.md, build/README.md, and scripts/pod-scheduler/README.md are updated with YAML examples and the schema-directive convention. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- README.md | 2 +- build/README.md | 42 ++-- build/benchmarks-ci-01.yml | 2 +- build/benchmarks-ci-02.yml | 2 +- build/benchmarks-ci-azure.yml | 2 +- build/benchmarks-ci-cobalt.yml | 2 +- build/benchmarks_ci_azure_pods.json | 107 --------- build/benchmarks_ci_azure_pods.yml | 104 +++++++++ build/benchmarks_ci_cobalt_pods.json | 124 ---------- build/benchmarks_ci_cobalt_pods.yml | 116 ++++++++++ build/benchmarks_ci_pods.json | 213 ------------------ build/benchmarks_ci_pods.yml | 193 ++++++++++++++++ scripts/pod-scheduler/README.md | 129 +++++++---- scripts/pod-scheduler/config_loader.py | 81 ++++++- scripts/pod-scheduler/generator.py | 8 +- scripts/pod-scheduler/main.py | 6 +- scripts/pod-scheduler/models.py | 4 +- scripts/pod-scheduler/pod-config.schema.json | 158 +++++++++++++ scripts/pod-scheduler/scheduler.py | 2 +- .../pod-scheduler/tests/test_config_loader.py | 85 ++++++- scripts/pod-scheduler/tests/test_generator.py | 8 +- scripts/pod-scheduler/tests/test_snapshots.py | 6 +- 22 files changed, 856 insertions(+), 540 deletions(-) delete mode 100644 build/benchmarks_ci_azure_pods.json create mode 100644 build/benchmarks_ci_azure_pods.yml delete mode 100644 build/benchmarks_ci_cobalt_pods.json create mode 100644 build/benchmarks_ci_cobalt_pods.yml delete mode 100644 build/benchmarks_ci_pods.json create mode 100644 build/benchmarks_ci_pods.yml create mode 100644 scripts/pod-scheduler/pod-config.schema.json diff --git a/README.md b/README.md index ebd214940..157e561d4 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,6 @@ Microsoft employees can approve and merge contributions by joining [this team](h # Continuous benchmarking pipelines The Azure DevOps pipelines that drive the daily benchmark sweep are generated -from JSON configs by the [pod-scheduler](scripts/pod-scheduler/README.md). +from YAML configs by the [pod-scheduler](scripts/pod-scheduler/README.md). See [`build/README.md`](build/README.md) for the mapping between configs and generated YAML files, and how to regenerate them after editing. diff --git a/build/README.md b/build/README.md index 205883ac0..66b79225f 100644 --- a/build/README.md +++ b/build/README.md @@ -7,28 +7,28 @@ configs that generate them. The CI pipelines that run the daily benchmark sweep are generated by the [pod-scheduler](../scripts/pod-scheduler/README.md). Each pipeline maps to a -JSON config that lists the available pods (machine groups) and scenarios. +YAML config that lists the available pods (machine groups) and scenarios. | Generated YAML | Source config | Notes | |---|---|---| -| `benchmarks-ci-01.yml`, `benchmarks-ci-02.yml` | `benchmarks_ci_pods.json` | On-prem fleet (citrine + mono); split across two YAMLs to spread load | -| `benchmarks-ci-azure.yml` | `benchmarks_ci_azure_pods.json` | Azure cloud pods (includes the merged eastus2 cobalt cloud pods) | -| `benchmarks-ci-cobalt.yml` | `benchmarks_ci_cobalt_pods.json` | Cobalt-hosted pods | +| `benchmarks-ci-01.yml`, `benchmarks-ci-02.yml` | `benchmarks_ci_pods.yml` | On-prem fleet (citrine + mono); split across two YAMLs to spread load | +| `benchmarks-ci-azure.yml` | `benchmarks_ci_azure_pods.yml` | Azure cloud pods (includes the merged eastus2 cobalt cloud pods) | +| `benchmarks-ci-cobalt.yml` | `benchmarks_ci_cobalt_pods.yml` | Cobalt-hosted pods | To regenerate, from the repo root: ```bash python ./scripts/pod-scheduler/main.py \ - --config ./build/benchmarks_ci_pods.json \ + --config ./build/benchmarks_ci_pods.yml \ --yaml-output ./build python ./scripts/pod-scheduler/main.py \ - --config ./build/benchmarks_ci_azure_pods.json \ + --config ./build/benchmarks_ci_azure_pods.yml \ --base-name benchmarks-ci-azure \ --yaml-output ./build python ./scripts/pod-scheduler/main.py \ - --config ./build/benchmarks_ci_cobalt_pods.json \ + --config ./build/benchmarks_ci_cobalt_pods.yml \ --base-name benchmarks-ci-cobalt \ --yaml-output ./build ``` @@ -36,6 +36,12 @@ python ./scripts/pod-scheduler/main.py \ The header of each generated YAML embeds the exact regen command for that file. +The configs ship with a `# yaml-language-server: $schema=...` directive at +the top so VS Code / Cursor (with the Red Hat YAML extension or the +`yaml-language-server` LSP) provide autocomplete, hover docs, and inline +validation while editing. The schema lives at +[`scripts/pod-scheduler/pod-config.schema.json`](../scripts/pod-scheduler/pod-config.schema.json). + ## Hand-maintained scenario templates Everything else in this directory is hand-maintained. The most important @@ -44,28 +50,32 @@ groups: - `*-scenarios.yml` — AzDO job templates that the generated pipelines invoke (e.g. `proxies-scenarios.yml`, `grpc-scenarios.yml`). One per benchmark family. The pod-scheduler picks which template to run via the - `template` field on each scenario in the JSON config. + `template` field on each scenario in the config. - `prbenchmarks.*.config.yml` — PR-trigger pipelines for individual PR runs. - `job-variables.yml` — shared variable definitions consumed by the scenario templates. When adding a new benchmark family, drop a `*-scenarios.yml` template in this -directory and reference it from the relevant `benchmarks_ci_*_pods.json`. +directory and reference it from the relevant `benchmarks_ci_*_pods.yml`. ## Editing the schedule -Common edits, all done in JSON: +Common edits, all done in the `benchmarks_ci_*_pods.yml` configs: -- **Add a scenario**: append to `scenarios[]` in the appropriate - `benchmarks_ci_*_pods.json`. -- **Add a pod / machine class**: append to `pods[]` and reference its name +- **Add a scenario**: append to `scenarios:` in the appropriate config. +- **Add a pod / machine class**: append to `pods:` and reference its name from any scenario you want to target it. - **Change cadence or splits**: edit `metadata.schedule` (cron), `metadata.yaml_generation.target_yaml_count`, or `metadata.yaml_generation.schedule_offset_hours`. -- **Override a job's timeout**: add `"timeout": ` to the scenario. +- **Override a job's timeout**: add `timeout: ` to the scenario. -After any edit, regenerate the YAML and commit both the JSON and the YAML -together. The snapshot tests in +After any edit, regenerate the YAML and commit both the config and the +generated YAML together. The snapshot tests in `scripts/pod-scheduler/tests/test_snapshots.py` will fail in CI if the generated YAML drifts from the configs. + +> Legacy `.json` configs are still accepted by the loader for back-compat, +> but new edits should use the `.yml` form so configs can carry comments and +> stay in sync with the schema-driven editor tooling. + diff --git a/build/benchmarks-ci-01.yml b/build/benchmarks-ci-01.yml index 30e73842e..623b88393 100644 --- a/build/benchmarks-ci-01.yml +++ b/build/benchmarks-ci-01.yml @@ -1,7 +1,7 @@ # Do not change this file, it is generated by the pod-scheduler. # Source of truth: see ../scripts/pod-scheduler/README.md # To regenerate, run from the repo root: -# python ./scripts/pod-scheduler/main.py --config ./build/benchmarks_ci_pods.json --base-name benchmarks-ci --yaml-output ./build +# python ./scripts/pod-scheduler/main.py --config ./build/benchmarks_ci_pods.yml --base-name benchmarks-ci --yaml-output ./build trigger: none pr: none diff --git a/build/benchmarks-ci-02.yml b/build/benchmarks-ci-02.yml index f9ee4c37b..b6766110e 100644 --- a/build/benchmarks-ci-02.yml +++ b/build/benchmarks-ci-02.yml @@ -1,7 +1,7 @@ # Do not change this file, it is generated by the pod-scheduler. # Source of truth: see ../scripts/pod-scheduler/README.md # To regenerate, run from the repo root: -# python ./scripts/pod-scheduler/main.py --config ./build/benchmarks_ci_pods.json --base-name benchmarks-ci --yaml-output ./build +# python ./scripts/pod-scheduler/main.py --config ./build/benchmarks_ci_pods.yml --base-name benchmarks-ci --yaml-output ./build trigger: none pr: none diff --git a/build/benchmarks-ci-azure.yml b/build/benchmarks-ci-azure.yml index 2406d43af..3a6fed40e 100644 --- a/build/benchmarks-ci-azure.yml +++ b/build/benchmarks-ci-azure.yml @@ -1,7 +1,7 @@ # Do not change this file, it is generated by the pod-scheduler. # Source of truth: see ../scripts/pod-scheduler/README.md # To regenerate, run from the repo root: -# python ./scripts/pod-scheduler/main.py --config ./build/benchmarks_ci_azure_pods.json --base-name benchmarks-ci-azure --yaml-output ./build +# python ./scripts/pod-scheduler/main.py --config ./build/benchmarks_ci_azure_pods.yml --base-name benchmarks-ci-azure --yaml-output ./build trigger: none pr: none diff --git a/build/benchmarks-ci-cobalt.yml b/build/benchmarks-ci-cobalt.yml index 1d0c2d0b5..9037306b4 100644 --- a/build/benchmarks-ci-cobalt.yml +++ b/build/benchmarks-ci-cobalt.yml @@ -1,7 +1,7 @@ # Do not change this file, it is generated by the pod-scheduler. # Source of truth: see ../scripts/pod-scheduler/README.md # To regenerate, run from the repo root: -# python ./scripts/pod-scheduler/main.py --config ./build/benchmarks_ci_cobalt_pods.json --base-name benchmarks-ci-cobalt --yaml-output ./build +# python ./scripts/pod-scheduler/main.py --config ./build/benchmarks_ci_cobalt_pods.yml --base-name benchmarks-ci-cobalt --yaml-output ./build trigger: none pr: none diff --git a/build/benchmarks_ci_azure_pods.json b/build/benchmarks_ci_azure_pods.json deleted file mode 100644 index fead30052..000000000 --- a/build/benchmarks_ci_azure_pods.json +++ /dev/null @@ -1,107 +0,0 @@ -{ - "metadata": { - "name": "Azure CI Benchmarks Configuration", - "description": "Pod-based scheduling for Azure CI benchmarks (includes cobalt cloud from eastus2)", - "version": "2.1", - "schedule": "0 2/12 * * *", - "queues": [ - "azure", - "azurearm64" - ], - "yaml_generation": { - "target_yaml_count": 1, - "schedule_offset_hours": 0 - } - }, - "pods": [ - { - "name": "azure-arm64", - "machines": { "sut": "azure-server-arm64", "load": "azure-client", "db": "azure-db" }, - "profiles": { "sut": "azure-server-arm64-app", "load": "azure-client-load", "db": "azure-db-db" } - }, - { - "name": "azure2-amd64", - "machines": { "sut": "azure2-server-amd64", "load": "azure2-client", "db": "azure2-db" }, - "profiles": { "sut": "azure2-server-amd64-app", "load": "azure2-client-load", "db": "azure2-db-db" } - }, - { - "name": "cobalt-cloud-lin", - "machines": { "sut": "cobalt-cloud-lin-server", "load": "cobalt-cloud-lin-client", "db": "cobalt-cloud-lin-db" }, - "profiles": { "sut": "cobalt-cloud-lin-server-app", "load": "cobalt-cloud-lin-client-load", "db": "cobalt-cloud-lin-db-db" } - }, - { - "name": "cobalt-cloud-lin-azl3", - "machines": { "sut": "cobalt-cloud-lin-server-azl3", "load": "cobalt-cloud-lin-client", "db": "cobalt-cloud-lin-db" }, - "profiles": { "sut": "cobalt-cloud-lin-server-azure-linux3-app", "load": "cobalt-cloud-lin-client-load", "db": "cobalt-cloud-lin-db-db" } - }, - { - "name": "cobalt-cloud-lin-azl3-dual", - "machines": { "sut": "cobalt-cloud-lin-server-azl3", "load": "cobalt-cloud-lin-db" }, - "profiles": { "sut": "cobalt-cloud-lin-server-azure-linux3-app", "load": "cobalt-cloud-lin-db-load" } - }, - { - "name": "idna-amd-lin", - "machines": { "sut": "idna-amd-lin", "load": "idna-intel-lin" }, - "profiles": { "sut": "idna-amd-lin-app", "load": "idna-intel-lin-load" } - }, - { - "name": "idna-amd-win", - "machines": { "sut": "idna-amd-win", "load": "azure2-db" }, - "profiles": { "sut": "idna-amd-win-app", "load": "azure2-db-load" } - }, - { - "name": "idna-intel-lin", - "machines": { "sut": "idna-intel-lin", "load": "idna-amd-lin" }, - "profiles": { "sut": "idna-intel-lin-app", "load": "idna-amd-lin-load" } - }, - { - "name": "idna-intel-win", - "machines": { "sut": "idna-intel-win", "load": "azure2-db" }, - "profiles": { "sut": "idna-intel-win-app", "load": "azure2-db-load" } - } - ], - "scenarios": [ - { - "name": "Containers", - "template": "containers-scenarios.yml", - "type": 3, - "pods": ["azure-arm64", "azure2-amd64", "cobalt-cloud-lin", "cobalt-cloud-lin-azl3"], - "estimated_runtime": 90.0 - }, - { - "name": "Baselines Database", - "template": "baselines-database-scenarios.yml", - "type": 3, - "pods": ["azure-arm64", "azure2-amd64", "cobalt-cloud-lin", "cobalt-cloud-lin-azl3"], - "estimated_runtime": 45.0 - }, - { - "name": "Baselines", - "template": "baselines-scenarios.yml", - "type": 2, - "pods": ["azure-arm64", "azure2-amd64", "cobalt-cloud-lin", "cobalt-cloud-lin-azl3-dual"], - "estimated_runtime": 30.0 - }, - { - "name": "MVC", - "template": "mvc-scenarios.yml", - "type": 3, - "pods": ["cobalt-cloud-lin", "cobalt-cloud-lin-azl3"], - "estimated_runtime": 20.0 - }, - { - "name": "Trends", - "template": "trend-scenarios.yml", - "type": 2, - "pods": ["azure-arm64", "azure2-amd64", "cobalt-cloud-lin", "cobalt-cloud-lin-azl3-dual", "idna-amd-lin", "idna-amd-win", "idna-intel-lin", "idna-intel-win"], - "estimated_runtime": 20.0 - }, - { - "name": "Trends Database", - "template": "trend-database-scenarios.yml", - "type": 3, - "pods": ["azure-arm64", "azure2-amd64", "cobalt-cloud-lin", "cobalt-cloud-lin-azl3"], - "estimated_runtime": 15.0 - } - ] -} diff --git a/build/benchmarks_ci_azure_pods.yml b/build/benchmarks_ci_azure_pods.yml new file mode 100644 index 000000000..80635eaef --- /dev/null +++ b/build/benchmarks_ci_azure_pods.yml @@ -0,0 +1,104 @@ +# yaml-language-server: $schema=../scripts/pod-scheduler/pod-config.schema.json + +# Azure CI pod-scheduler config. +# +# Covers Azure cloud pods plus the merged eastus2 cobalt-cloud pods (folded +# in here when their dedicated pipeline was retired). Generates a single +# ./benchmarks-ci-azure.yml. +# +# Edit this file then regenerate from the repo root: +# python ./scripts/pod-scheduler/main.py \ +# --config ./build/benchmarks_ci_azure_pods.yml \ +# --base-name benchmarks-ci-azure --yaml-output ./build +metadata: + name: Azure CI Benchmarks Configuration + description: Pod-based scheduling for Azure CI benchmarks (includes cobalt cloud from eastus2) + version: "2.1" + schedule: "0 2/12 * * *" + queues: + - azure + - azurearm64 + yaml_generation: + target_yaml_count: 1 + schedule_offset_hours: 0 + +pods: + - name: azure-arm64 + machines: { sut: azure-server-arm64, load: azure-client, db: azure-db } + profiles: { sut: azure-server-arm64-app, load: azure-client-load, db: azure-db-db } + + - name: azure2-amd64 + machines: { sut: azure2-server-amd64, load: azure2-client, db: azure2-db } + profiles: { sut: azure2-server-amd64-app, load: azure2-client-load, db: azure2-db-db } + + # eastus2 cobalt-cloud pods, merged in when the dedicated pipeline was retired. + - name: cobalt-cloud-lin + machines: { sut: cobalt-cloud-lin-server, load: cobalt-cloud-lin-client, db: cobalt-cloud-lin-db } + profiles: { sut: cobalt-cloud-lin-server-app, load: cobalt-cloud-lin-client-load, db: cobalt-cloud-lin-db-db } + + - name: cobalt-cloud-lin-azl3 + # Same load + db machines as cobalt-cloud-lin but a different SUT image + # (Azure Linux 3). Cannot share a stage with cobalt-cloud-lin triples. + machines: { sut: cobalt-cloud-lin-server-azl3, load: cobalt-cloud-lin-client, db: cobalt-cloud-lin-db } + profiles: { sut: cobalt-cloud-lin-server-azure-linux3-app, load: cobalt-cloud-lin-client-load, db: cobalt-cloud-lin-db-db } + + - name: cobalt-cloud-lin-azl3-dual + # Dual-only variant: re-uses the db box as the load generator so we can + # run dual scenarios without needing a separate load machine. + machines: { sut: cobalt-cloud-lin-server-azl3, load: cobalt-cloud-lin-db } + profiles: { sut: cobalt-cloud-lin-server-azure-linux3-app, load: cobalt-cloud-lin-db-load } + + - name: idna-amd-lin + # idna-amd-lin and idna-intel-lin are mutually each other's load box. + machines: { sut: idna-amd-lin, load: idna-intel-lin } + profiles: { sut: idna-amd-lin-app, load: idna-intel-lin-load } + + - name: idna-amd-win + # Borrows azure2-db as the load box for cross-cluster idna jobs. + machines: { sut: idna-amd-win, load: azure2-db } + profiles: { sut: idna-amd-win-app, load: azure2-db-load } + + - name: idna-intel-lin + machines: { sut: idna-intel-lin, load: idna-amd-lin } + profiles: { sut: idna-intel-lin-app, load: idna-amd-lin-load } + + - name: idna-intel-win + machines: { sut: idna-intel-win, load: azure2-db } + profiles: { sut: idna-intel-win-app, load: azure2-db-load } + +scenarios: + - name: Containers + template: containers-scenarios.yml + type: triple + pods: [azure-arm64, azure2-amd64, cobalt-cloud-lin, cobalt-cloud-lin-azl3] + estimated_runtime: 90 + + - name: Baselines Database + template: baselines-database-scenarios.yml + type: triple + pods: [azure-arm64, azure2-amd64, cobalt-cloud-lin, cobalt-cloud-lin-azl3] + estimated_runtime: 45 + + - name: Baselines + template: baselines-scenarios.yml + type: dual + pods: [azure-arm64, azure2-amd64, cobalt-cloud-lin, cobalt-cloud-lin-azl3-dual] + estimated_runtime: 30 + + - name: MVC + template: mvc-scenarios.yml + type: triple + pods: [cobalt-cloud-lin, cobalt-cloud-lin-azl3] + estimated_runtime: 20 + + - name: Trends + template: trend-scenarios.yml + type: dual + pods: [azure-arm64, azure2-amd64, cobalt-cloud-lin, cobalt-cloud-lin-azl3-dual, idna-amd-lin, idna-amd-win, idna-intel-lin, idna-intel-win] + estimated_runtime: 20 + + - name: Trends Database + template: trend-database-scenarios.yml + type: triple + pods: [azure-arm64, azure2-amd64, cobalt-cloud-lin, cobalt-cloud-lin-azl3] + estimated_runtime: 15 diff --git a/build/benchmarks_ci_cobalt_pods.json b/build/benchmarks_ci_cobalt_pods.json deleted file mode 100644 index aed674790..000000000 --- a/build/benchmarks_ci_cobalt_pods.json +++ /dev/null @@ -1,124 +0,0 @@ -{ - "metadata": { - "name": "Cobalt Hosted CI Benchmarks Configuration", - "description": "Pod-based scheduling for cobalt hosted CI benchmarks", - "version": "2.0", - "schedule": "0 3/12 * * *", - "queues": [ - "cobalthosted", - "cobalthosted_azurelinux3" - ], - "yaml_generation": { - "target_yaml_count": 1, - "schedule_offset_hours": 0 - } - }, - "pods": [ - { - "name": "cobalt-hosted-lin", - "machines": { "sut": "cobalt-hosted-lin-server", "load": "cobalt-hosted-lin-client", "db": "cobalt-hosted-lin-db" }, - "profiles": { "sut": "cobalt-hosted-lin-server-app", "load": "cobalt-hosted-lin-client-load", "db": "cobalt-hosted-lin-db-db" } - }, - { - "name": "cobalt-hosted-lin-azl3", - "machines": { "sut": "cobalt-hosted-lin-server-azl3", "load": "cobalt-hosted-lin-client-azl3", "db": "cobalt-hosted-lin-db-azl3" }, - "profiles": { "sut": "cobalt-hosted-lin-server-azure-linux3-app", "load": "cobalt-hosted-lin-client-azure-linux3-load", "db": "cobalt-hosted-lin-db-azure-linux3-db" } - }, - { - "name": "cobalt-hosted-lin-28", - "machines": { "sut": "cobalt-hosted-lin-server", "load": "cobalt-hosted-lin-client", "db": "cobalt-hosted-lin-db" }, - "profiles": { "sut": "cobalt-hosted-lin-server-28-app", "load": "cobalt-hosted-lin-client-load", "db": "cobalt-hosted-lin-db-db" } - }, - { - "name": "cobalt-hosted-lin-azl3-28", - "machines": { "sut": "cobalt-hosted-lin-server-azl3", "load": "cobalt-hosted-lin-client-azl3", "db": "cobalt-hosted-lin-db-azl3" }, - "profiles": { "sut": "cobalt-hosted-lin-server-azure-linux3-28-app", "load": "cobalt-hosted-lin-client-azure-linux3-load", "db": "cobalt-hosted-lin-db-azure-linux3-db" } - } - ], - "scenarios": [ - { - "name": "Containers", - "template": "containers-scenarios.yml", - "type": 3, - "pods": ["cobalt-hosted-lin", "cobalt-hosted-lin-azl3"], - "estimated_runtime": 90.0 - }, - { - "name": "PGO", - "template": "pgo-scenarios.yml", - "type": 3, - "pods": ["cobalt-hosted-lin", "cobalt-hosted-lin-azl3", "cobalt-hosted-lin-28", "cobalt-hosted-lin-azl3-28"], - "estimated_runtime": 90.0 - }, - { - "name": "Baselines Database", - "template": "baselines-database-scenarios.yml", - "type": 3, - "pods": ["cobalt-hosted-lin", "cobalt-hosted-lin-azl3", "cobalt-hosted-lin-28", "cobalt-hosted-lin-azl3-28"], - "estimated_runtime": 45.0 - }, - { - "name": "Baselines", - "template": "baselines-scenarios.yml", - "type": 2, - "pods": ["cobalt-hosted-lin", "cobalt-hosted-lin-azl3", "cobalt-hosted-lin-28", "cobalt-hosted-lin-azl3-28"], - "estimated_runtime": 30.0 - }, - { - "name": "Frameworks Database", - "template": "frameworks-database-scenarios.yml", - "type": 3, - "pods": ["cobalt-hosted-lin", "cobalt-hosted-lin-azl3", "cobalt-hosted-lin-28", "cobalt-hosted-lin-azl3-28"], - "estimated_runtime": 20.0 - }, - { - "name": "MVC", - "template": "mvc-scenarios.yml", - "type": 3, - "pods": ["cobalt-hosted-lin", "cobalt-hosted-lin-azl3"], - "estimated_runtime": 20.0 - }, - { - "name": "NativeAOT", - "template": "nativeaot-scenarios.yml", - "type": 3, - "pods": ["cobalt-hosted-lin", "cobalt-hosted-lin-azl3", "cobalt-hosted-lin-28", "cobalt-hosted-lin-azl3-28"], - "estimated_runtime": 20.0 - }, - { - "name": "Trends", - "template": "trend-scenarios.yml", - "type": 2, - "pods": ["cobalt-hosted-lin", "cobalt-hosted-lin-azl3", "cobalt-hosted-lin-28", "cobalt-hosted-lin-azl3-28"], - "estimated_runtime": 20.0 - }, - { - "name": "Trends Database", - "template": "trend-database-scenarios.yml", - "type": 3, - "pods": ["cobalt-hosted-lin", "cobalt-hosted-lin-azl3", "cobalt-hosted-lin-28", "cobalt-hosted-lin-azl3-28"], - "estimated_runtime": 15.0 - }, - { - "name": "Crossgen", - "template": "crossgen2-scenarios.yml", - "type": 2, - "pods": ["cobalt-hosted-lin", "cobalt-hosted-lin-azl3", "cobalt-hosted-lin-28", "cobalt-hosted-lin-azl3-28"], - "estimated_runtime": 5.0 - }, - { - "name": "Frameworks", - "template": "frameworks-scenarios.yml", - "type": 3, - "pods": ["cobalt-hosted-lin", "cobalt-hosted-lin-azl3", "cobalt-hosted-lin-28", "cobalt-hosted-lin-azl3-28"], - "estimated_runtime": 15.0 - }, - { - "name": "GC", - "template": "gc-scenarios.yml", - "type": 1, - "pods": ["cobalt-hosted-lin", "cobalt-hosted-lin-azl3", "cobalt-hosted-lin-28", "cobalt-hosted-lin-azl3-28"], - "estimated_runtime": 15.0 - } - ] -} diff --git a/build/benchmarks_ci_cobalt_pods.yml b/build/benchmarks_ci_cobalt_pods.yml new file mode 100644 index 000000000..35ae743c2 --- /dev/null +++ b/build/benchmarks_ci_cobalt_pods.yml @@ -0,0 +1,116 @@ +# yaml-language-server: $schema=../scripts/pod-scheduler/pod-config.schema.json + +# Cobalt-hosted pod-scheduler config. +# +# Two physical Linux clusters (default + Azure Linux 3) plus the same +# clusters running a 28-core SUT profile. Generates a single +# ./benchmarks-ci-cobalt.yml. +# +# Edit this file then regenerate from the repo root: +# python ./scripts/pod-scheduler/main.py \ +# --config ./build/benchmarks_ci_cobalt_pods.yml \ +# --base-name benchmarks-ci-cobalt --yaml-output ./build +metadata: + name: Cobalt Hosted CI Benchmarks Configuration + description: Pod-based scheduling for cobalt hosted CI benchmarks + version: "2.0" + schedule: "0 3/12 * * *" + queues: + - cobalthosted + - cobalthosted_azurelinux3 + yaml_generation: + target_yaml_count: 1 + schedule_offset_hours: 0 + +# The "-28" pods reuse the same physical machines as their non-28 siblings +# (just a different SUT crank profile that pins to 28 cores). The scheduler +# automatically keeps each physical machine to one stage at a time. +pods: + - name: cobalt-hosted-lin + machines: { sut: cobalt-hosted-lin-server, load: cobalt-hosted-lin-client, db: cobalt-hosted-lin-db } + profiles: { sut: cobalt-hosted-lin-server-app, load: cobalt-hosted-lin-client-load, db: cobalt-hosted-lin-db-db } + + - name: cobalt-hosted-lin-azl3 + machines: { sut: cobalt-hosted-lin-server-azl3, load: cobalt-hosted-lin-client-azl3, db: cobalt-hosted-lin-db-azl3 } + profiles: { sut: cobalt-hosted-lin-server-azure-linux3-app, load: cobalt-hosted-lin-client-azure-linux3-load, db: cobalt-hosted-lin-db-azure-linux3-db } + + - name: cobalt-hosted-lin-28 + machines: { sut: cobalt-hosted-lin-server, load: cobalt-hosted-lin-client, db: cobalt-hosted-lin-db } + profiles: { sut: cobalt-hosted-lin-server-28-app, load: cobalt-hosted-lin-client-load, db: cobalt-hosted-lin-db-db } + + - name: cobalt-hosted-lin-azl3-28 + machines: { sut: cobalt-hosted-lin-server-azl3, load: cobalt-hosted-lin-client-azl3, db: cobalt-hosted-lin-db-azl3 } + profiles: { sut: cobalt-hosted-lin-server-azure-linux3-28-app, load: cobalt-hosted-lin-client-azure-linux3-load, db: cobalt-hosted-lin-db-azure-linux3-db } + +scenarios: + - name: Containers + template: containers-scenarios.yml + type: triple + pods: [cobalt-hosted-lin, cobalt-hosted-lin-azl3] + estimated_runtime: 90 + + - name: PGO + template: pgo-scenarios.yml + type: triple + pods: [cobalt-hosted-lin, cobalt-hosted-lin-azl3, cobalt-hosted-lin-28, cobalt-hosted-lin-azl3-28] + estimated_runtime: 90 + + - name: Baselines Database + template: baselines-database-scenarios.yml + type: triple + pods: [cobalt-hosted-lin, cobalt-hosted-lin-azl3, cobalt-hosted-lin-28, cobalt-hosted-lin-azl3-28] + estimated_runtime: 45 + + - name: Baselines + template: baselines-scenarios.yml + type: dual + pods: [cobalt-hosted-lin, cobalt-hosted-lin-azl3, cobalt-hosted-lin-28, cobalt-hosted-lin-azl3-28] + estimated_runtime: 30 + + - name: Frameworks Database + template: frameworks-database-scenarios.yml + type: triple + pods: [cobalt-hosted-lin, cobalt-hosted-lin-azl3, cobalt-hosted-lin-28, cobalt-hosted-lin-azl3-28] + estimated_runtime: 20 + + - name: MVC + template: mvc-scenarios.yml + type: triple + pods: [cobalt-hosted-lin, cobalt-hosted-lin-azl3] + estimated_runtime: 20 + + - name: NativeAOT + template: nativeaot-scenarios.yml + type: triple + pods: [cobalt-hosted-lin, cobalt-hosted-lin-azl3, cobalt-hosted-lin-28, cobalt-hosted-lin-azl3-28] + estimated_runtime: 20 + + - name: Trends + template: trend-scenarios.yml + type: dual + pods: [cobalt-hosted-lin, cobalt-hosted-lin-azl3, cobalt-hosted-lin-28, cobalt-hosted-lin-azl3-28] + estimated_runtime: 20 + + - name: Trends Database + template: trend-database-scenarios.yml + type: triple + pods: [cobalt-hosted-lin, cobalt-hosted-lin-azl3, cobalt-hosted-lin-28, cobalt-hosted-lin-azl3-28] + estimated_runtime: 15 + + - name: Crossgen + template: crossgen2-scenarios.yml + type: dual + pods: [cobalt-hosted-lin, cobalt-hosted-lin-azl3, cobalt-hosted-lin-28, cobalt-hosted-lin-azl3-28] + estimated_runtime: 5 + + - name: Frameworks + template: frameworks-scenarios.yml + type: triple + pods: [cobalt-hosted-lin, cobalt-hosted-lin-azl3, cobalt-hosted-lin-28, cobalt-hosted-lin-azl3-28] + estimated_runtime: 15 + + - name: GC + template: gc-scenarios.yml + type: single + pods: [cobalt-hosted-lin, cobalt-hosted-lin-azl3, cobalt-hosted-lin-28, cobalt-hosted-lin-azl3-28] + estimated_runtime: 15 diff --git a/build/benchmarks_ci_pods.json b/build/benchmarks_ci_pods.json deleted file mode 100644 index ad4e75104..000000000 --- a/build/benchmarks_ci_pods.json +++ /dev/null @@ -1,213 +0,0 @@ -{ - "metadata": { - "name": "CI Benchmarks Configuration", - "description": "Pod-based scheduling for continuous integration benchmarks", - "version": "2.0", - "schedule": "0 3/12 * * *", - "queues": [ - "citrine1", - "citrine2", - "citrine3", - "mono" - ], - "yaml_generation": { - "target_yaml_count": 2, - "schedule_offset_hours": 6 - } - }, - "pods": [ - { - "name": "intel-lin", - "machines": { "sut": "intel-lin", "load": "intel-load", "db": "intel-db" }, - "profiles": { "sut": "intel-lin-app", "load": "intel-load-load", "db": "intel-db-db" } - }, - { - "name": "intel-win", - "machines": { "sut": "intel-win", "load": "intel-load2", "db": "intel-db" }, - "profiles": { "sut": "intel-win-app", "load": "intel-load2-load", "db": "intel-db-db" } - }, - { - "name": "gold-lin", - "machines": { "sut": "gold-lin", "load": "gold-load", "db": "gold-db" }, - "profiles": { "sut": "gold-lin-app", "load": "gold-load-load", "db": "gold-db-db" } - }, - { - "name": "gold-win", - "machines": { "sut": "gold-win", "load": "gold-load2", "db": "gold-db" }, - "profiles": { "sut": "gold-win-app", "load": "gold-load2-load", "db": "gold-db-db" } - }, - { - "name": "amd-lin2", - "machines": { "sut": "amd-lin2", "load": "gold-load", "db": "gold-db" }, - "profiles": { "sut": "amd-lin2-app", "load": "gold-load-load", "db": "gold-db-db" } - }, - { - "name": "intel-perflin", - "machines": { "sut": "intel-perflin", "load": "intel-perfload" }, - "profiles": { "sut": "intel-perflin-app", "load": "intel-perfload-load" } - } - ], - "scenarios": [ - { - "name": "Baselines", - "template": "baselines-scenarios.yml", - "type": 2, - "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win", "amd-lin2"], - "estimated_runtime": 30.0 - }, - { - "name": "Baselines Database", - "template": "baselines-database-scenarios.yml", - "type": 3, - "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win", "amd-lin2"], - "estimated_runtime": 45.0 - }, - { - "name": "Blazor", - "template": "blazor-scenarios.yml", - "type": 2, - "pods": ["gold-lin", "intel-lin", "intel-perflin"], - "estimated_runtime": 45.0 - }, - { - "name": "Build", - "template": "build-perf-scenarios.yml", - "type": 1, - "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win"], - "estimated_runtime": 1.0 - }, - { - "name": "Containers", - "template": "containers-scenarios.yml", - "type": 3, - "pods": ["gold-lin", "intel-lin"], - "estimated_runtime": 90.0 - }, - { - "name": "Crossgen", - "template": "crossgen2-scenarios.yml", - "type": 2, - "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win", "amd-lin2"], - "estimated_runtime": 5.0 - }, - { - "name": "Custom Proxies", - "template": "proxies-custom-scenarios.yml", - "type": 3, - "pods": ["gold-lin", "intel-lin"], - "estimated_runtime": 5.0 - }, - { - "name": "EF Core", - "template": "efcore-scenarios.yml", - "type": 2, - "pods": ["gold-win", "intel-win"], - "estimated_runtime": 15.0 - }, - { - "name": "Frameworks", - "template": "frameworks-scenarios.yml", - "type": 3, - "pods": ["gold-lin", "intel-lin", "amd-lin2"], - "estimated_runtime": 15.0 - }, - { - "name": "Frameworks Database", - "template": "frameworks-database-scenarios.yml", - "type": 3, - "pods": ["gold-lin", "intel-lin", "amd-lin2"], - "estimated_runtime": 20.0 - }, - { - "name": "GC", - "template": "gc-scenarios.yml", - "type": 1, - "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win"], - "estimated_runtime": 15.0 - }, - { - "name": "Grpc", - "template": "grpc-scenarios.yml", - "type": 2, - "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win"], - "estimated_runtime": 70.0 - }, - { - "name": "HttpClient", - "template": "httpclient-scenarios.yml", - "type": 2, - "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win"], - "estimated_runtime": 45.0 - }, - { - "name": "MVC", - "template": "mvc-scenarios.yml", - "type": 3, - "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win"], - "estimated_runtime": 20.0 - }, - { - "name": "NativeAOT", - "template": "nativeaot-scenarios.yml", - "type": 3, - "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win"], - "estimated_runtime": 20.0 - }, - { - "name": "PGO", - "template": "pgo-scenarios.yml", - "type": 3, - "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win"], - "estimated_runtime": 90.0 - }, - { - "name": "Proxies", - "template": "proxies-scenarios.yml", - "type": 3, - "pods": ["gold-lin", "intel-lin"], - "estimated_runtime": 150.0 - }, - { - "name": "SignalR", - "template": "signalr-scenarios.yml", - "type": 2, - "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win"], - "estimated_runtime": 30.0 - }, - { - "name": "Single File", - "template": "singlefile-scenarios.yml", - "type": 2, - "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win"], - "estimated_runtime": 10.0 - }, - { - "name": "SslStream", - "template": "sslstream-scenarios.yml", - "type": 2, - "pods": ["gold-lin", "intel-lin"], - "estimated_runtime": 45.0 - }, - { - "name": "Trends", - "template": "trend-scenarios.yml", - "type": 2, - "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win", "amd-lin2"], - "estimated_runtime": 20.0 - }, - { - "name": "Trends Database", - "template": "trend-database-scenarios.yml", - "type": 3, - "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win", "amd-lin2"], - "estimated_runtime": 15.0 - }, - { - "name": "WebSockets", - "template": "websocket-scenarios.yml", - "type": 2, - "pods": ["gold-lin", "gold-win", "intel-lin", "intel-win"], - "estimated_runtime": 6.0 - } - ] -} diff --git a/build/benchmarks_ci_pods.yml b/build/benchmarks_ci_pods.yml new file mode 100644 index 000000000..244aabb6d --- /dev/null +++ b/build/benchmarks_ci_pods.yml @@ -0,0 +1,193 @@ +# yaml-language-server: $schema=../scripts/pod-scheduler/pod-config.schema.json + +# CI pod-scheduler config: on-prem fleet (citrine + mono). +# +# Generates ./benchmarks-ci-01.yml and ./benchmarks-ci-02.yml. Workload is +# split across two YAMLs so each scheduled run hits roughly half the fleet +# and overall wall-clock time stays under the 4h job budget. +# +# Edit this file then regenerate from the repo root: +# python ./scripts/pod-scheduler/main.py \ +# --config ./build/benchmarks_ci_pods.yml --yaml-output ./build +metadata: + name: CI Benchmarks Configuration + description: Pod-based scheduling for continuous integration benchmarks + version: "2.0" + schedule: "0 3/12 * * *" + queues: + - citrine1 + - citrine2 + - citrine3 + - mono + yaml_generation: + target_yaml_count: 2 + # Stagger the second YAML by 6h so the fleet isn't slammed twice at once. + schedule_offset_hours: 6 + +pods: + - name: intel-lin + machines: { sut: intel-lin, load: intel-load, db: intel-db } + profiles: { sut: intel-lin-app, load: intel-load-load, db: intel-db-db } + + - name: intel-win + # intel-win shares intel-db with intel-lin; the scheduler keeps them out + # of the same stage when both run a triple-type scenario. + machines: { sut: intel-win, load: intel-load2, db: intel-db } + profiles: { sut: intel-win-app, load: intel-load2-load, db: intel-db-db } + + - name: gold-lin + machines: { sut: gold-lin, load: gold-load, db: gold-db } + profiles: { sut: gold-lin-app, load: gold-load-load, db: gold-db-db } + + - name: gold-win + machines: { sut: gold-win, load: gold-load2, db: gold-db } + profiles: { sut: gold-win-app, load: gold-load2-load, db: gold-db-db } + + - name: amd-lin2 + # Reuses gold-lin's load + db; cannot run alongside gold-lin triples. + machines: { sut: amd-lin2, load: gold-load, db: gold-db } + profiles: { sut: amd-lin2-app, load: gold-load-load, db: gold-db-db } + + - name: intel-perflin + # SUT+load only; not eligible for triple scenarios. + machines: { sut: intel-perflin, load: intel-perfload } + profiles: { sut: intel-perflin-app, load: intel-perfload-load } + +scenarios: + - name: Baselines + template: baselines-scenarios.yml + type: dual + pods: [gold-lin, gold-win, intel-lin, intel-win, amd-lin2] + estimated_runtime: 30 + + - name: Baselines Database + template: baselines-database-scenarios.yml + type: triple + pods: [gold-lin, gold-win, intel-lin, intel-win, amd-lin2] + estimated_runtime: 45 + + - name: Blazor + template: blazor-scenarios.yml + type: dual + pods: [gold-lin, intel-lin, intel-perflin] + estimated_runtime: 45 + + - name: Build + template: build-perf-scenarios.yml + type: single + pods: [gold-lin, gold-win, intel-lin, intel-win] + estimated_runtime: 1 + + - name: Containers + template: containers-scenarios.yml + type: triple + pods: [gold-lin, intel-lin] + estimated_runtime: 90 + + - name: Crossgen + template: crossgen2-scenarios.yml + type: dual + pods: [gold-lin, gold-win, intel-lin, intel-win, amd-lin2] + estimated_runtime: 5 + + - name: Custom Proxies + template: proxies-custom-scenarios.yml + type: triple + pods: [gold-lin, intel-lin] + estimated_runtime: 5 + + - name: EF Core + template: efcore-scenarios.yml + type: dual + pods: [gold-win, intel-win] + estimated_runtime: 15 + + - name: Frameworks + template: frameworks-scenarios.yml + type: triple + pods: [gold-lin, intel-lin, amd-lin2] + estimated_runtime: 15 + + - name: Frameworks Database + template: frameworks-database-scenarios.yml + type: triple + pods: [gold-lin, intel-lin, amd-lin2] + estimated_runtime: 20 + + - name: GC + template: gc-scenarios.yml + type: single + pods: [gold-lin, gold-win, intel-lin, intel-win] + estimated_runtime: 15 + + - name: Grpc + template: grpc-scenarios.yml + type: dual + pods: [gold-lin, gold-win, intel-lin, intel-win] + estimated_runtime: 70 + + - name: HttpClient + template: httpclient-scenarios.yml + type: dual + pods: [gold-lin, gold-win, intel-lin, intel-win] + estimated_runtime: 45 + + - name: MVC + template: mvc-scenarios.yml + type: triple + pods: [gold-lin, gold-win, intel-lin, intel-win] + estimated_runtime: 20 + + - name: NativeAOT + template: nativeaot-scenarios.yml + type: triple + pods: [gold-lin, gold-win, intel-lin, intel-win] + estimated_runtime: 20 + + - name: PGO + template: pgo-scenarios.yml + type: triple + pods: [gold-lin, gold-win, intel-lin, intel-win] + estimated_runtime: 90 + + - name: Proxies + template: proxies-scenarios.yml + type: triple + pods: [gold-lin, intel-lin] + estimated_runtime: 150 + + - name: SignalR + template: signalr-scenarios.yml + type: dual + pods: [gold-lin, gold-win, intel-lin, intel-win] + estimated_runtime: 30 + + - name: Single File + template: singlefile-scenarios.yml + type: dual + pods: [gold-lin, gold-win, intel-lin, intel-win] + estimated_runtime: 10 + + - name: SslStream + template: sslstream-scenarios.yml + type: dual + pods: [gold-lin, intel-lin] + estimated_runtime: 45 + + - name: Trends + template: trend-scenarios.yml + type: dual + pods: [gold-lin, gold-win, intel-lin, intel-win, amd-lin2] + estimated_runtime: 20 + + - name: Trends Database + template: trend-database-scenarios.yml + type: triple + pods: [gold-lin, gold-win, intel-lin, intel-win, amd-lin2] + estimated_runtime: 15 + + - name: WebSockets + template: websocket-scenarios.yml + type: dual + pods: [gold-lin, gold-win, intel-lin, intel-win] + estimated_runtime: 6 diff --git a/scripts/pod-scheduler/README.md b/scripts/pod-scheduler/README.md index 4e46f0b62..c035b83c7 100644 --- a/scripts/pod-scheduler/README.md +++ b/scripts/pod-scheduler/README.md @@ -19,27 +19,27 @@ Run from the repository root: ```bash # Show schedule summary -python scripts/pod-scheduler/main.py --config build/benchmarks_ci_pods.json +python scripts/pod-scheduler/main.py --config build/benchmarks_ci_pods.yml # Generate pipeline YAML files python scripts/pod-scheduler/main.py \ - --config build/benchmarks_ci_pods.json \ + --config build/benchmarks_ci_pods.yml \ --yaml-output build # Regenerate the Azure or Cobalt pipelines python scripts/pod-scheduler/main.py \ - --config build/benchmarks_ci_azure_pods.json \ + --config build/benchmarks_ci_azure_pods.yml \ --base-name benchmarks-ci-azure --yaml-output build python scripts/pod-scheduler/main.py \ - --config build/benchmarks_ci_cobalt_pods.json \ + --config build/benchmarks_ci_cobalt_pods.yml \ --base-name benchmarks-ci-cobalt --yaml-output build # Show which pods share machines -python scripts/pod-scheduler/main.py --config build/benchmarks_ci_pods.json --show-conflicts +python scripts/pod-scheduler/main.py --config build/benchmarks_ci_pods.yml --show-conflicts # List all runs without scheduling -python scripts/pod-scheduler/main.py --config build/benchmarks_ci_pods.json --list-runs +python scripts/pod-scheduler/main.py --config build/benchmarks_ci_pods.yml --list-runs ``` The header of every generated YAML embeds the exact regen command for that @@ -50,7 +50,7 @@ pod-for-scenario reference fails with a non-zero exit code so config typos cannot silently drop scenarios from the pipeline. Pass `--lenient` to fall back to the previous warn-and-skip behavior. -Output is **deterministic**: identical input JSON always produces identical +Output is **deterministic**: identical input config always produces identical YAML, so regenerations diff cleanly. To verify, run the snapshot tests: ```bash @@ -60,40 +60,41 @@ python -m unittest discover tests ## Configuration Format -```json -{ - "metadata": { - "name": "Config Name", - "schedule": "0 3/12 * * *", - "queues": ["citrine1", "citrine2", "citrine3", "mono"], - "yaml_generation": { - "target_yaml_count": 2, - "schedule_offset_hours": 6 - }, - "pipeline": { - "pool": "server", - "service_bus_connection": "ASPNET Benchmarks Service Bus", - "service_bus_namespace": "aspnetbenchmarks" - } - }, - "pods": [ - { - "name": "gold-lin", - "machines": { "sut": "gold-lin", "load": "gold-load", "db": "gold-db" }, - "profiles": { "sut": "gold-lin-app", "load": "gold-load-load", "db": "gold-db-db" } - } - ], - "scenarios": [ - { - "name": "Baselines", - "template": "baselines-scenarios.yml", - "type": 2, - "pods": ["gold-lin", "gold-win"], - "estimated_runtime": 30.0, - "timeout": 120 - } - ] -} +Configs are YAML by default (see `build/benchmarks_ci_pods.yml` for a real +example). The loader also accepts the legacy JSON form (`.json`) so older +local copies still work, but new edits should be in YAML — comments make a +big difference for readability and the schema-driven LSP support only kicks +in for YAML. + +```yaml +# yaml-language-server: $schema=../scripts/pod-scheduler/pod-config.schema.json + +metadata: + name: Config Name + schedule: "0 3/12 * * *" + queues: [citrine1, citrine2, citrine3, mono] + yaml_generation: + target_yaml_count: 2 + schedule_offset_hours: 6 + pipeline: + pool: server + service_bus_connection: ASPNET Benchmarks Service Bus + service_bus_namespace: aspnetbenchmarks + +pods: + - name: gold-lin + # gold-lin and gold-win share gold-db; the scheduler keeps them out of + # the same stage when both run a triple-type scenario. + machines: { sut: gold-lin, load: gold-load, db: gold-db } + profiles: { sut: gold-lin-app, load: gold-load-load, db: gold-db-db } + +scenarios: + - name: Baselines + template: baselines-scenarios.yml + type: dual # single | dual | triple (or 1 | 2 | 3 for legacy) + pods: [gold-lin, gold-win] + estimated_runtime: 30 + timeout: 120 ``` The `pipeline` block is optional; defaults match the legacy hardcoded values. @@ -102,6 +103,27 @@ The `schedule` field's **hour** must be a `H` or `H/N` cron expression (e.g. `3` or `3/12`). Lists, ranges, and `*` are rejected at load time so the hour-offset used for split YAMLs cannot silently no-op. +### Schema-driven editing + +Every shipped config opens with: + +```yaml +# yaml-language-server: $schema=../scripts/pod-scheduler/pod-config.schema.json +``` + +VS Code (with the Red Hat YAML extension), Cursor, JetBrains IDEs, and any +editor running [`yaml-language-server`](https://github.com/redhat-developer/yaml-language-server) +will pick this up and provide: + +- autocomplete for keys and `type` values, +- hover docs sourced from the schema's `description` fields, and +- inline error squiggles for missing required fields, unknown keys, or + invalid enum values. + +LLM-driven edits also benefit: the schema's descriptions are reliable +grounding so agents are far less likely to hallucinate keys or invalid +`type` values. + ### Pod Definition | Field | Description | @@ -120,7 +142,7 @@ hour-offset used for split YAMLs cannot silently no-op. |-------|-------------| | `name` | Display name (also used as part of the AzDO job id) | | `template` | YAML scenario template to invoke | -| `type` | 1=SINGLE, 2=DUAL, 3=TRIPLE (see below) | +| `type` | `single` / `dual` / `triple` (or 1 / 2 / 3 — see below) | | `pods` | List of pod names this scenario targets (no duplicates) | | `estimated_runtime` | Runtime estimate in minutes; defaults per type if omitted | | `timeout` | Optional explicit AzDO `timeoutInMinutes` override. When unset, the generator picks `max(120, min(240, ceil(2 * estimated_runtime)))` | @@ -129,9 +151,12 @@ hour-offset used for split YAMLs cannot silently no-op. | Type | Machines Used | Example | |------|--------------|---------| -| 1 (SINGLE) | SUT only | Build, GC | -| 2 (DUAL) | SUT + Load | Baselines, Grpc, SignalR | -| 3 (TRIPLE) | SUT + Load + DB | Baselines Database, PGO, Proxies | +| `single` (1) | SUT only | Build, GC | +| `dual` (2) | SUT + Load | Baselines, Grpc, SignalR | +| `triple` (3) | SUT + Load + DB | Baselines Database, PGO, Proxies | + +The string form is preferred for new configs because it's self-documenting; +the integer form is still accepted for back-compat with older configs. ### Queue Assignment @@ -155,9 +180,14 @@ conflict. If you get 2 SUT machines of the same class (e.g., gold-lin-1 and gold-lin-2), create separate pods for each. They can share load/DB: -```json -{"name": "gold-lin-1", "machines": {"sut": "gold-lin-1", "load": "gold-load", "db": "gold-db"}, ...}, -{"name": "gold-lin-2", "machines": {"sut": "gold-lin-2", "load": "gold-load", "db": "gold-db"}, ...} +```yaml +- name: gold-lin-1 + machines: { sut: gold-lin-1, load: gold-load, db: gold-db } + profiles: { sut: gold-lin-1-app, load: gold-load-load, db: gold-db-db } + +- name: gold-lin-2 + machines: { sut: gold-lin-2, load: gold-load, db: gold-db } + profiles: { sut: gold-lin-2-app, load: gold-load-load, db: gold-db-db } ``` The scheduler automatically prevents them from running simultaneously when they @@ -180,8 +210,9 @@ share load/DB machines. | `main.py` | CLI entry point, summary display | | `models.py` | Data classes (Pod, Scenario, Run, Stage, Schedule, PipelineSettings) | | `scheduler.py` | Scheduling algorithm | -| `config_loader.py` | JSON config parser + validation | +| `config_loader.py` | YAML/JSON config parser + validation | | `generator.py` | YAML generation | +| `pod-config.schema.json` | JSON Schema describing the config; consumed by editor LSPs | | `tests/` | Unit + snapshot tests (`python -m unittest`) | This is intentionally script-style: the modules use absolute imports diff --git a/scripts/pod-scheduler/config_loader.py b/scripts/pod-scheduler/config_loader.py index f5b1d6377..0640251bf 100644 --- a/scripts/pod-scheduler/config_loader.py +++ b/scripts/pod-scheduler/config_loader.py @@ -1,10 +1,17 @@ """ -JSON configuration loader for pod-based scheduling. +Configuration loader for pod-based scheduling. + +Accepts YAML (`.yml`/`.yaml`) and JSON (`.json`) files. YAML is preferred so +configs can carry inline comments; JSON is supported for back-compat. The +file format is dispatched purely on extension. """ import json +import os import re -from typing import Any, Dict +from typing import Any, Dict, Union + +import yaml from models import ( PipelineSettings, @@ -16,11 +23,22 @@ class ConfigError(ValueError): - """Raised when a JSON config is malformed or self-inconsistent.""" + """Raised when a config is malformed or self-inconsistent.""" _CRON_HOUR_RE = re.compile(r"^\d+(/\d+)?$") +# Accepted spellings for ``scenario.type``. Strings (case-insensitive) are the +# preferred form; integers stay supported so legacy configs keep loading. +_SCENARIO_TYPE_ALIASES: Dict[Union[str, int], ScenarioType] = { + "single": ScenarioType.SINGLE, + "dual": ScenarioType.DUAL, + "triple": ScenarioType.TRIPLE, + 1: ScenarioType.SINGLE, + 2: ScenarioType.DUAL, + 3: ScenarioType.TRIPLE, +} + def _require(node: Dict[str, Any], key: str, context: str) -> Any: if key not in node: @@ -43,10 +61,58 @@ def _validate_cron(schedule: str) -> None: ) -def load_config(path: str) -> ScheduleConfig: - """Load and validate a pod-scheduler JSON configuration file.""" +def _parse_scenario_type(raw: Any, scenario_name: str) -> ScenarioType: + """Resolve a scenario type from string, int, or ScenarioType. + + Accepts ``single``/``dual``/``triple`` (case-insensitive) or ``1``/``2``/``3``. + Bools are rejected explicitly because YAML happily turns ``yes``/``no`` + into bools, which would otherwise quietly resolve to ``1``/``0``. + """ + if isinstance(raw, ScenarioType): + return raw + if isinstance(raw, bool): + raise ConfigError( + f"scenario '{scenario_name}' has invalid type {raw!r}; " + f"use 'single', 'dual', or 'triple' (or 1/2/3)" + ) + key: Any = raw + if isinstance(raw, str): + key = raw.strip().lower() + if key not in _SCENARIO_TYPE_ALIASES: + raise ConfigError( + f"scenario '{scenario_name}' has invalid type {raw!r}; " + f"use 'single', 'dual', or 'triple' (or 1/2/3)" + ) + return _SCENARIO_TYPE_ALIASES[key] + + +def _load_raw(path: str) -> Any: + """Read the config file as a Python dict, dispatching on extension.""" + ext = os.path.splitext(path)[1].lower() with open(path, "r", encoding="utf-8") as f: - data = json.load(f) + text = f.read() + if ext in (".yml", ".yaml"): + try: + return yaml.safe_load(text) + except yaml.YAMLError as exc: + raise ConfigError(f"Failed to parse YAML config {path}: {exc}") + if ext == ".json": + try: + return json.loads(text) + except json.JSONDecodeError as exc: + raise ConfigError(f"Failed to parse JSON config {path}: {exc}") + raise ConfigError( + f"Unsupported config extension {ext!r}; expected .yml, .yaml, or .json" + ) + + +def load_config(path: str) -> ScheduleConfig: + """Load and validate a pod-scheduler configuration file (YAML or JSON).""" + data = _load_raw(path) + if not isinstance(data, dict): + raise ConfigError( + f"Config root must be a mapping, got {type(data).__name__}" + ) metadata = _require(data, "metadata", "config root") schedule = _require(metadata, "schedule", "metadata") @@ -113,10 +179,11 @@ def load_config(path: str) -> ScheduleConfig: raise ConfigError( f"scenario '{name}' has non-positive timeout {timeout}" ) + raw_type = _require(sc_data, "type", f"scenario '{name}'") scenarios.append(Scenario( name=name, template=_require(sc_data, "template", f"scenario '{name}'"), - type=ScenarioType(_require(sc_data, "type", f"scenario '{name}'")), + type=_parse_scenario_type(raw_type, name), pods=list(scenario_pods), estimated_runtime=float(runtime_raw) if runtime_raw else 0.0, timeout=timeout, diff --git a/scripts/pod-scheduler/generator.py b/scripts/pod-scheduler/generator.py index efde9d19a..d90405868 100644 --- a/scripts/pod-scheduler/generator.py +++ b/scripts/pod-scheduler/generator.py @@ -114,7 +114,7 @@ def _render_yaml( else: lines.append( "# python ./scripts/pod-scheduler/main.py " - "--config ./build/.json " + "--config ./build/.yml " "--yaml-output ./build" ) lines.append("") @@ -209,9 +209,9 @@ def generate_yamls( unique within the produced YAML; collisions raise GeneratorError instead of producing AzDO-invalid output. - ``source_config`` is the path to the JSON config that produced the - schedule. When provided, it's embedded in the generated YAML header so - each file documents the exact command needed to regenerate it. + ``source_config`` is the path to the config that produced the schedule. + When provided, it's embedded in the generated YAML header so each file + documents the exact command needed to regenerate it. """ os.makedirs(output_dir, exist_ok=True) output_files = [] diff --git a/scripts/pod-scheduler/main.py b/scripts/pod-scheduler/main.py index 00cdc0236..4088547c1 100644 --- a/scripts/pod-scheduler/main.py +++ b/scripts/pod-scheduler/main.py @@ -7,8 +7,8 @@ and the scheduler ensures no physical machine is double-booked per stage. Usage: - python main.py --config ./build/benchmarks_ci_pods.json - python main.py --config ./build/benchmarks_ci_pods.json \\ + python main.py --config ./build/benchmarks_ci_pods.yml + python main.py --config ./build/benchmarks_ci_pods.yml \\ --yaml-output ./build """ @@ -141,7 +141,7 @@ def _build_arg_parser() -> argparse.ArgumentParser: ) parser.add_argument( "--config", required=True, - help="Path to JSON configuration file" + help="Path to YAML or JSON configuration file" ) parser.add_argument( "--yaml-output", diff --git a/scripts/pod-scheduler/models.py b/scripts/pod-scheduler/models.py index 22ec8503f..d50623993 100644 --- a/scripts/pod-scheduler/models.py +++ b/scripts/pod-scheduler/models.py @@ -16,7 +16,7 @@ # none of its own. Lives here so models, scheduler, and tests share one source. DEFAULT_RUNTIMES: Dict["ScenarioType", float] = {} # populated below -# Pipeline plumbing defaults. Override in JSON metadata.pipeline.* if needed. +# Pipeline plumbing defaults. Override in metadata.pipeline.* in the config. DEFAULT_PIPELINE_POOL = "server" DEFAULT_PIPELINE_CONNECTION = "ASPNET Benchmarks Service Bus" DEFAULT_PIPELINE_NAMESPACE = "aspnetbenchmarks" @@ -183,7 +183,7 @@ class PipelineSettings: @dataclass class ScheduleConfig: - """Top-level configuration loaded from JSON.""" + """Top-level configuration loaded from a YAML or JSON file.""" name: str schedule: str queues: List[str] diff --git a/scripts/pod-scheduler/pod-config.schema.json b/scripts/pod-scheduler/pod-config.schema.json new file mode 100644 index 000000000..d8695b083 --- /dev/null +++ b/scripts/pod-scheduler/pod-config.schema.json @@ -0,0 +1,158 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/aspnet/Benchmarks/main/scripts/pod-scheduler/pod-config.schema.json", + "title": "Pod Scheduler Config", + "description": "Configuration for the pod-based crank scheduler. Defines the cron schedule, available worker queues, machine pods, and benchmark scenarios. The scheduler expands every (scenario x pod) combination into runs, packs them into stages so no physical machine is double-booked, and emits an Azure DevOps pipeline YAML.", + "type": "object", + "required": ["metadata", "pods", "scenarios"], + "additionalProperties": false, + "properties": { + "metadata": { "$ref": "#/$defs/metadata" }, + "pods": { + "type": "array", + "description": "All available machine pods. Each pod is a fixed group of machines (SUT + optional load + optional DB) that always run together.", + "minItems": 1, + "items": { "$ref": "#/$defs/pod" } + }, + "scenarios": { + "type": "array", + "description": "Benchmark scenarios to run. Each scenario targets one or more pods.", + "minItems": 1, + "items": { "$ref": "#/$defs/scenario" } + } + }, + "$defs": { + "metadata": { + "type": "object", + "required": ["name", "schedule", "queues"], + "additionalProperties": true, + "properties": { + "name": { + "type": "string", + "description": "Human-readable name shown in the schedule summary." + }, + "description": { + "type": "string", + "description": "Optional free-form description." + }, + "version": { + "type": "string", + "description": "Optional config version tag." + }, + "schedule": { + "type": "string", + "description": "5-field cron expression. The hour field must be a single integer or 'H/N' (no lists/ranges/'*') so split YAMLs can offset the hour deterministically.", + "pattern": "^[^ ]+ \\d+(/\\d+)? [^ ]+ [^ ]+ [^ ]+$" + }, + "queues": { + "type": "array", + "description": "Service-bus queue names. The Nth run in a stage is dispatched to queues[N % len(queues)].", + "minItems": 1, + "uniqueItems": true, + "items": { "type": "string" } + }, + "yaml_generation": { + "type": "object", + "description": "Controls how the schedule is split across multiple YAML files.", + "additionalProperties": false, + "properties": { + "target_yaml_count": { + "type": "integer", + "description": "Number of pipeline YAML files to produce. Use >1 to spread runtime across separate scheduled pipelines.", + "minimum": 1 + }, + "schedule_offset_hours": { + "type": "integer", + "description": "Hours to offset the cron between split YAMLs. The Nth split shifts the hour field by N * schedule_offset_hours.", + "minimum": 0 + } + } + }, + "pipeline": { + "type": "object", + "description": "Optional overrides for pipeline-level plumbing rendered into the YAML.", + "additionalProperties": false, + "properties": { + "pool": { "type": "string" }, + "service_bus_connection": { "type": "string" }, + "service_bus_namespace": { "type": "string" } + } + } + } + }, + "pod": { + "type": "object", + "required": ["name", "machines", "profiles"], + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "description": "Unique pod identifier referenced by scenarios.pods.", + "minLength": 1 + }, + "machines": { + "type": "object", + "description": "Physical machine names by role. Two pods that share a physical machine cannot run in the same stage.", + "required": ["sut"], + "additionalProperties": false, + "properties": { + "sut": { "type": "string", "description": "System Under Test machine. Required." }, + "load": { "type": "string", "description": "Load generator machine. Required for dual/triple scenarios." }, + "db": { "type": "string", "description": "Database machine. Required for triple scenarios." } + } + }, + "profiles": { + "type": "object", + "description": "Crank profile name for each role.", + "required": ["sut"], + "additionalProperties": false, + "properties": { + "sut": { "type": "string" }, + "load": { "type": "string" }, + "db": { "type": "string" } + } + } + } + }, + "scenario": { + "type": "object", + "required": ["name", "template", "type", "pods"], + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "description": "Display name. Combined with the pod name to form the AzDO job id (after sanitization).", + "minLength": 1 + }, + "template": { + "type": "string", + "description": "Path to the AzDO scenario template invoked by every run of this scenario (relative to the build/ directory)." + }, + "type": { + "description": "Number of machine roles required. Use 'single' (SUT only), 'dual' (SUT+load), or 'triple' (SUT+load+db). Integer 1/2/3 is also accepted for back-compat.", + "oneOf": [ + { "type": "string", "enum": ["single", "dual", "triple"] }, + { "type": "integer", "enum": [1, 2, 3] } + ] + }, + "pods": { + "type": "array", + "description": "Pods this scenario targets. Each entry must match a pods[].name. Duplicates are rejected.", + "minItems": 1, + "uniqueItems": true, + "items": { "type": "string" } + }, + "estimated_runtime": { + "type": "number", + "description": "Estimated runtime in minutes. Used by the scheduler's longest-job-first packing.", + "minimum": 0 + }, + "timeout": { + "type": "integer", + "description": "Optional explicit AzDO timeoutInMinutes override. When unset, the generator picks max(120, min(240, ceil(2 * estimated_runtime))).", + "minimum": 1 + } + } + } + } +} diff --git a/scripts/pod-scheduler/scheduler.py b/scripts/pod-scheduler/scheduler.py index c86284bea..cb16d04c0 100644 --- a/scripts/pod-scheduler/scheduler.py +++ b/scripts/pod-scheduler/scheduler.py @@ -5,7 +5,7 @@ Since pods define fixed machine groupings, the scheduler only needs to ensure no physical machine is used twice in the same stage. -Output is deterministic: identical input JSON always produces identical +Output is deterministic: identical input config always produces identical schedules, so generated YAML files diff cleanly across regenerations. """ diff --git a/scripts/pod-scheduler/tests/test_config_loader.py b/scripts/pod-scheduler/tests/test_config_loader.py index 9dd698939..dd96db72f 100644 --- a/scripts/pod-scheduler/tests/test_config_loader.py +++ b/scripts/pod-scheduler/tests/test_config_loader.py @@ -3,6 +3,8 @@ import tempfile import unittest +import yaml + import tests # noqa: F401 # ensures sys.path is set up from config_loader import ConfigError, load_config @@ -25,20 +27,33 @@ { "name": "S", "template": "s.yml", - "type": 1, + "type": "single", "pods": ["p1"], } ], } -def _write(tmp, payload): +def _write_yaml(tmp, payload): + path = os.path.join(tmp, "cfg.yml") + with open(path, "w", encoding="utf-8") as f: + yaml.safe_dump(payload, f, sort_keys=False) + return path + + +def _write_json(tmp, payload): path = os.path.join(tmp, "cfg.json") with open(path, "w", encoding="utf-8") as f: json.dump(payload, f) return path +# Default fixture writer used by tests that don't care which format. Using +# YAML here also exercises the YAML path on every test so regressions in +# YAML loading show up immediately, not only in the snapshot test. +_write = _write_yaml + + class TestLoadConfig(unittest.TestCase): def test_happy_path(self): with tempfile.TemporaryDirectory() as tmp: @@ -112,6 +127,72 @@ def test_negative_timeout_rejected(self): with self.assertRaises(ConfigError): load_config(path) + def test_scenario_type_string_aliases(self): + # All three string spellings + case-insensitivity must resolve. + for raw in ["single", "DUAL", "Triple"]: + with tempfile.TemporaryDirectory() as tmp: + payload = json.loads(json.dumps(_BASE)) + payload["scenarios"][0]["type"] = raw + if raw.lower() in ("dual", "triple"): + payload["pods"][0]["machines"]["load"] = "m2" + payload["pods"][0]["profiles"]["load"] = "m2-load" + if raw.lower() == "triple": + payload["pods"][0]["machines"]["db"] = "m3" + payload["pods"][0]["profiles"]["db"] = "m3-db" + path = _write(tmp, payload) + cfg = load_config(path) + self.assertEqual( + cfg.scenarios[0].type.value, + {"single": 1, "dual": 2, "triple": 3}[raw.lower()], + raw, + ) + + def test_scenario_type_integer_back_compat(self): + with tempfile.TemporaryDirectory() as tmp: + payload = json.loads(json.dumps(_BASE)) + payload["scenarios"][0]["type"] = 1 + path = _write(tmp, payload) + cfg = load_config(path) + self.assertEqual(cfg.scenarios[0].type.value, 1) + + def test_scenario_type_invalid_rejected(self): + with tempfile.TemporaryDirectory() as tmp: + payload = json.loads(json.dumps(_BASE)) + payload["scenarios"][0]["type"] = "quad" + path = _write(tmp, payload) + with self.assertRaises(ConfigError): + load_config(path) + + def test_scenario_type_bool_rejected(self): + # YAML loaders coerce ``yes``/``no`` to bool; the loader must catch + # those before they silently resolve to True/False (== 1/0). + with tempfile.TemporaryDirectory() as tmp: + payload = json.loads(json.dumps(_BASE)) + payload["scenarios"][0]["type"] = True + path = _write(tmp, payload) + with self.assertRaises(ConfigError): + load_config(path) + + def test_json_back_compat(self): + # Existing .json configs must still load even though the project has + # moved to YAML. Use the integer-typed fixture so this exercises the + # legacy shape end-to-end. + with tempfile.TemporaryDirectory() as tmp: + payload = json.loads(json.dumps(_BASE)) + payload["scenarios"][0]["type"] = 1 + path = _write_json(tmp, payload) + cfg = load_config(path) + self.assertEqual(cfg.scenarios[0].type.value, 1) + + def test_unknown_extension_rejected(self): + with tempfile.TemporaryDirectory() as tmp: + payload = json.loads(json.dumps(_BASE)) + path = os.path.join(tmp, "cfg.txt") + with open(path, "w", encoding="utf-8") as f: + yaml.safe_dump(payload, f) + with self.assertRaises(ConfigError): + load_config(path) + if __name__ == "__main__": unittest.main() diff --git a/scripts/pod-scheduler/tests/test_generator.py b/scripts/pod-scheduler/tests/test_generator.py index 91a59f617..6c9107149 100644 --- a/scripts/pod-scheduler/tests/test_generator.py +++ b/scripts/pod-scheduler/tests/test_generator.py @@ -63,17 +63,17 @@ def test_paths_in_repo_become_repo_relative(self): repo_root = os.path.abspath( os.path.join(os.path.dirname(__file__), "..", "..", "..") ) - candidate = os.path.join(repo_root, "build", "benchmarks_ci_pods.json") + candidate = os.path.join(repo_root, "build", "benchmarks_ci_pods.yml") self.assertEqual( _format_source_path(candidate), - "./build/benchmarks_ci_pods.json", + "./build/benchmarks_ci_pods.yml", ) def test_outside_repo_falls_back_to_basename(self): # Use a path that's definitely outside the repo by jumping above root. - far = os.path.abspath(os.sep + "definitely-not-in-repo.json") + far = os.path.abspath(os.sep + "definitely-not-in-repo.yml") self.assertEqual( - _format_source_path(far), "./definitely-not-in-repo.json" + _format_source_path(far), "./definitely-not-in-repo.yml" ) diff --git a/scripts/pod-scheduler/tests/test_snapshots.py b/scripts/pod-scheduler/tests/test_snapshots.py index 849430daf..c9f4467d7 100644 --- a/scripts/pod-scheduler/tests/test_snapshots.py +++ b/scripts/pod-scheduler/tests/test_snapshots.py @@ -22,11 +22,11 @@ CASES = [ - ("benchmarks_ci_pods.json", "benchmarks-ci", + ("benchmarks_ci_pods.yml", "benchmarks-ci", ["benchmarks-ci-01.yml", "benchmarks-ci-02.yml"]), - ("benchmarks_ci_azure_pods.json", "benchmarks-ci-azure", + ("benchmarks_ci_azure_pods.yml", "benchmarks-ci-azure", ["benchmarks-ci-azure.yml"]), - ("benchmarks_ci_cobalt_pods.json", "benchmarks-ci-cobalt", + ("benchmarks_ci_cobalt_pods.yml", "benchmarks-ci-cobalt", ["benchmarks-ci-cobalt.yml"]), ]