Release #49, #50, #51, #52, #53, and #54 to staging.

.circleci/config.yml

@@ -43,6 +43,9 @@ commands:
       environment:
         type: string
     steps:
+      - add_ssh_keys:
+          fingerprints:
+            - "SHA256:rPzp7kChj9Z72jls470HfO0YvieTbdUiB+y8hlxJN8c"
       - *attach_workspace
       - checkout
       - run:
@@ -89,6 +92,9 @@ commands:
       environment:
         type: string
     steps:
+      - add_ssh_keys:
+          fingerprints:
+            - "SHA256:rPzp7kChj9Z72jls470HfO0YvieTbdUiB+y8hlxJN8c"
       - *attach_workspace
       - checkout
       - run:
@@ -145,7 +151,8 @@ jobs:
   build-and-test:
     executor: docker-python
     steps:
-      - checkout
+      - checkout:
+          method: full
       - setup_remote_docker
       - sonarcloud/scan
       - run:
@@ -229,6 +236,31 @@ jobs:
     steps:
       - deploy-lambda:
           stage: "production"
+  assume-role-pre-production:
+    executor: docker-python
+    steps:
+      - assume-role-and-persist-workspace:
+          aws-account: $AWS_ACCOUNT_PRE_PRODUCTION
+  terraform-init-and-plan-pre-production:
+    executor: docker-terraform
+    steps:
+      - terraform-init-then-plan:
+          environment: "pre-production"
+  terraform-compliance-pre-production:
+    executor: docker-terraform
+    steps:
+      - terraform-compliance:
+          environment: "pre-production"
+  terraform-apply-pre-production:
+    executor: docker-terraform
+    steps:
+      - terraform-apply:
+          environment: "pre-production"
+  deploy-to-pre-production:
+    executor: docker-dotnet
+    steps:
+      - deploy-lambda:
+          stage: "pre-production"
 
 workflows:
   check:
@@ -339,3 +371,69 @@ workflows:
             - "Serverless Framework"
           requires:
             - permit-production-release
+
+  deploy-terraform-pre-production:
+    jobs:
+      - permit-pre-production-terraform-workflow:
+          type: approval
+          filters:
+            branches:
+              only: release
+      - assume-role-pre-production:
+          context: api-assume-role-housing-pre-production-context
+          requires:
+            - permit-pre-production-terraform-workflow
+          filters:
+            branches:
+              only: release
+      - terraform-init-and-plan-pre-production:
+          requires:
+            - assume-role-pre-production
+          filters:
+            branches:
+              only: release
+      - terraform-compliance-pre-production:
+          requires:
+            - terraform-init-and-plan-pre-production
+          filters:
+            branches:
+              only: release
+      - permit-pre-production-terraform-deployment:
+          type: approval
+          requires:
+            - terraform-compliance-pre-production
+          filters:
+            branches:
+              only: release
+      - terraform-apply-pre-production:
+          requires:
+            - permit-pre-production-terraform-deployment
+          filters:
+            branches:
+              only: release
+
+  deploy-code-pre-production:
+    jobs:
+      - build-and-test:
+          context: 
+            - api-nuget-token-context
+            - SonarCloud
+          filters:
+            branches:
+              only: release
+      - assume-role-pre-production:
+          context: api-assume-role-housing-pre-production-context
+          requires:
+            - build-and-test
+          filters:
+            branches:
+              only: release
+      - deploy-to-pre-production:
+          context:
+          - api-nuget-token-context
+          - "Serverless Framework"
+          requires:
+            - assume-role-pre-production        
+          filters:
+            branches:
+              only: release

ReferenceDataApi/ReferenceDataApi.csproj

@@ -20,7 +20,7 @@
     <PackageReference Include="Hackney.Core.HealthCheck" Version="1.30.0" />
     <PackageReference Include="Hackney.Core.Logging" Version="1.30.0" />
     <PackageReference Include="Hackney.Core.Middleware" Version="1.30.0" />
-    <PackageReference Include="Hackney.Core.JWT" Version="1.30.0" />
+    <PackageReference Include="Hackney.Core.JWT" Version="1.87.0" />
     <PackageReference Include="Hackney.Core.Validation" Version="1.30.0" />
     <PackageReference Include="Microsoft.AspNetCore.HealthChecks" Version="1.0.0" />
     <PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="3.1.0" />

ReferenceDataApi/Startup.cs

@@ -138,6 +138,8 @@ public void ConfigureServices(IServiceCollection services)
             services.AddLogCallAspect();
             services.ConfigureElasticSearch(Configuration);
             services.AddElasticSearchHealthCheck();
+            // Token factory used by the logging middleware core package
+            // to print user email. Hidden, indirect, implicit dependency
             services.AddTokenFactory();
 
             RegisterGateways(services);

ReferenceDataApi/serverless.yml

@@ -100,15 +100,6 @@ resources:
                           - Ref: 'AWS::Region'
                           - Ref: 'AWS::AccountId'
                           - 'log-group:/aws/lambda/*:*:*'
-                - Effect: "Allow"
-                  Action:
-                    - "s3:PutObject"
-                    - "s3:GetObject"
-                  Resource:
-                    Fn::Join:
-                      - ""
-                      - - "arn:aws:s3:::"
-                        - "Ref": "ServerlessDeploymentBucket"
           - PolicyName: lambdaInvocation
             PolicyDocument:
               Version: '2012-10-17'
@@ -132,7 +123,7 @@ resources:
             - - 'arn:aws:iam:'
               - Ref: 'AWS::AccountId'
               - role/LBH_Canary_Role
-        RuntimeVersion: syn-nodejs-puppeteer-3.1
+        RuntimeVersion: syn-nodejs-puppeteer-9.1
         RunConfig:
           TimeoutInSeconds: 300
           EnvironmentVariables:
@@ -151,9 +142,10 @@ resources:
           DurationInSeconds: 0
 custom:
   authorizerArns:
-    development: arn:aws:lambda:eu-west-2:859159924354:function:api-auth-verify-token-new-development-apiauthverifytokennew
+    development: arn:aws:lambda:eu-west-2:859159924354:function:api-gateway-lambda-authorizer
     staging:     arn:aws:lambda:eu-west-2:715003523189:function:api-auth-verify-token-new-staging-apiauthverifytokennew
     production:  arn:aws:lambda:eu-west-2:153306643385:function:api-auth-verify-token-new-production-apiauthverifytokennew
+    pre-production: arn:aws:lambda:eu-west-2:578479666894:function:api-auth-verify-token-new-pre-production-apiauthverifytokennew
   safeguards:
     - title: Require authorizer
       safeguard: require-authorizer
@@ -187,3 +179,9 @@ custom:
       subnetIds:
         - subnet-06a697d86a9b6ed01
         - subnet-0beb266003a56ca82
+    pre-production:
+      securityGroupIds:
+        - sg-0c6335cf631b61e07
+      subnetIds:
+        - subnet-08aa35159a8706faa
+        - subnet-0b848c5b14f841dfb

terraform/pre-production/aws_ssm_parameter.tf

@@ -0,0 +1,11 @@
+resource "aws_ssm_parameter" "reference_data_token" {
+  name  = "/housing-tl/pre-production/reference-data-token"
+  type  = "String"
+  value = "to_be_set_manually"
+
+  lifecycle {
+    ignore_changes = [
+      value,
+    ]
+  }
+}

terraform/pre-production/maint.tf

@@ -0,0 +1,81 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 3.0"
+    }
+  }
+}
+
+provider "aws" {
+  region = "eu-west-2"
+  default_tags {
+    tags = {
+      Name              = "reference-data-api-${var.environment_name}"
+      Environment       = var.environment_name
+      terraform-managed = true
+      project_name      = var.project_name
+      Application       = "MTFH Housing Pre-Production"
+      TeamEmail         = "developementteam@hackney.gov.uk"
+      BackupPolicy      = "Dev"
+      Confidentiality   = "Internal"
+    }
+  }
+}
+
+data "aws_caller_identity" "current" {}
+
+data "aws_region" "current" {}
+
+locals {
+  esDomain = "https://${module.elasticsearch_db_pre_production.es_endpoint_url}"
+}
+
+terraform {
+  backend "s3" {
+    bucket         = "housing-pre-production-terraform-state"
+    encrypt        = true
+    region         = "eu-west-2"
+    key            = "services/reference-data-api/state"
+    dynamodb_table = "housing-pre-production-terraform-state-lock"
+  }
+}
+
+data "aws_vpc" "pre_production_vpc" {
+  tags = {
+    Name = "housing-pre-prod-pre-prod"
+  }
+}
+
+data "aws_subnet_ids" "pre_production" {
+  vpc_id = data.aws_vpc.pre_production_vpc.id
+  filter {
+    name   = "tag:Type"
+    values = ["private"]
+  }
+}
+
+module "elasticsearch_db_pre_production" {
+  source              = "github.com/LBHackney-IT/aws-hackney-common-terraform.git//modules/database/elasticsearch"
+  vpc_id              = data.aws_vpc.pre_production_vpc.id
+  environment_name    = "pre-production"
+  port                = 443
+  domain_name         = "reference-data-api-es"
+  subnet_ids          = [tolist(data.aws_subnet_ids.pre_production.ids)[0]]
+  project_name        = "reference-data-api"
+  es_version          = "7.8"
+  encrypt_at_rest     = "true"
+  instance_type       = "t3.small.elasticsearch"
+  instance_count      = "2"
+  ebs_enabled         = "true"
+  ebs_volume_size     = "30"
+  region              = data.aws_region.current.name
+  account_id          = data.aws_caller_identity.current.account_id
+  create_service_role = false
+}
+
+resource "aws_ssm_parameter" "reference_data_elasticsearch_domain" {
+  name  = "/reference-data-api/pre-production/elasticsearch-domain"
+  type  = "String"
+  value = local.esDomain
+}

terraform/pre-production/terraform-compliance/opensearch.feature

@@ -0,0 +1,25 @@
+Feature: OpenSearch is used to host the ElasticSearch clusters
+  In order to improve security
+  As engineers
+  We'll use ensure our OpenSearch clusters are configured correctly
+
+  Scenario: Ensure it is deployed in a VPC
+    Given I have aws_elasticsearch_domain defined
+    Then it must contain vpc_options
+
+  Scenario: Ensure OpenSearch clusters are encrypted at rest
+    Given I have aws_elasticsearch_domain defined
+    Then it must contain encrypt_at_rest
+    And its enabled property must be true
+
+  Scenario: Ensure minimum instance count is 2
+    Given I have aws_elasticsearch_domain defined
+    Then it must contain cluster_config
+    And it must contain instance_count
+    And its value must be greater and equal to 2
+
+  Scenario: Ensure instance type is small or medium
+    Given I have aws_elasticsearch_domain defined
+    Then it must contain cluster_config
+    And it must contain instance_type
+    And its value must match the "^(t3\.small\.elasticsearch|t3\.medium\.elasticsearch)" regex

terraform/pre-production/variables.tf

@@ -0,0 +1,9 @@
+variable "environment_name" {
+  type    = string
+  default = "pre-prod"
+}
+
+variable "project_name" {
+  type    = string
+  default = "Housing-Pre-Production"
+}