From b9b3bf572f9f6cc45037aea615a25731bb952a31 Mon Sep 17 00:00:00 2001 From: Rioting Pacifist Date: Mon, 16 Mar 2020 15:39:43 +0000 Subject: [PATCH] Initial commit contains: * Actual template * Example function * Tests * Integration test (make file) * Integration tests (deployed as lambda) * drone CI and CI scripts --- .drone.yml | 29 +++++++ .gitignore | 6 ++ .yamllint.yaml | 6 ++ LICENSE | 22 +++++ Makefile | 68 ++++++++++++++++ README.md | 33 ++++++++ ci/cleanup.sh | 23 ++++++ ci/deploy.sh | 17 ++++ cloudformation/lambda-sam-template.yaml | 103 ++++++++++++++++++++++++ environments/ci.env | 1 + requirements.txt | 9 +++ src/__init__.py | 0 src/lambda_function.py | 45 +++++++++++ src/lambda_verification.py | 51 ++++++++++++ tests/__init__.py | 0 tests/test_lambda_function.py | 46 +++++++++++ tests/test_lambda_verification.py | 66 +++++++++++++++ tox.ini | 2 + 18 files changed, 527 insertions(+) create mode 100644 .drone.yml create mode 100644 .gitignore create mode 100644 .yamllint.yaml create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.md create mode 100644 ci/cleanup.sh create mode 100644 ci/deploy.sh create mode 100644 cloudformation/lambda-sam-template.yaml create mode 100644 environments/ci.env create mode 100644 requirements.txt create mode 100644 src/__init__.py create mode 100644 src/lambda_function.py create mode 100644 src/lambda_verification.py create mode 100644 tests/__init__.py create mode 100644 tests/test_lambda_function.py create mode 100644 tests/test_lambda_verification.py create mode 100644 tox.ini diff --git a/.drone.yml b/.drone.yml new file mode 100644 index 0000000..83aab51 --- /dev/null +++ b/.drone.yml @@ -0,0 +1,29 @@ +--- +kind: pipeline +type: docker +name: default + +steps: + - name: deploy-ci + image: python:latest + commands: + - ci/deploy.sh + when: + branch: + exclude: + - master + + - name: deploy-prod + image: python:latest + commands: + - make all env=prod + when: + branch: + - master + event: + - push + + - name: cleanup-env + image: python:latest + commands: + - ci/cleanup.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ce34a35 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +*.pyc +.*-generated.yaml +*.kate-swp +test-response.json +.coverage +environments/prod.env diff --git a/.yamllint.yaml b/.yamllint.yaml new file mode 100644 index 0000000..f764383 --- /dev/null +++ b/.yamllint.yaml @@ -0,0 +1,6 @@ +--- +extends: default + +rules: + line-length: + max: 120 diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..78273a1 --- /dev/null +++ b/LICENSE @@ -0,0 +1,22 @@ +Copyright 2020 Juan Canham + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..a331dc1 --- /dev/null +++ b/Makefile @@ -0,0 +1,68 @@ +.PHONY: help all install lint clean test deploy integration-test delete + +ifdef $(env) + include environments/$(env).env +endif + +CFN := cloudformation +CFN_NAME := example-cloud-function +BUCKET := $(shell aws s3 ls | grep templates | cut -c 24-) +TEST_DATA := 100 +TEST_RESULT := "2,4,5,10,20,25,50" +EMAIL := "cv@juancanham.com" + +help: + @echo 'Targets:' + @echo ' * all [install clean lint test deploy integration-test]' + @echo ' * delete' + @echo ' - optionally you can specify an env to load overrides' + +all: install clean lint test deploy integration-test + +install: + pip install -r requirements.txt + +clean: + rm -f */**/*.pyc + rm -f $(CFN)/.*-generated.yaml + rm -f test-response.txt + +lint: + yamllint . + black . + pylint --disable line-too-long src/ tests/ + cfn-lint $(CFN)/*.yaml + shellcheck ci/*.sh + +test: + pytest + +deploy: + aws cloudformation package \ + --template-file $(CFN)/lambda-sam-template.yaml \ + --s3-bucket $(BUCKET) \ + --output-template-file $(CFN)/.$(CFN_NAME)-generated.yaml + + aws cloudformation deploy \ + --stack-name $(CFN_NAME) \ + --template-file $(CFN)/.$(CFN_NAME)-generated.yaml \ + --capabilities CAPABILITY_IAM \ + --parameter-overrides \ + NotificationEmail=$(EMAIL) \ + TestData=$(TEST_DATA) \ + TestExpected=$(TEST_RESULT) + +delete: + aws cloudformation delete-stack --stack-name $(CFN_NAME) + +integration-test: + aws lambda invoke \ + --payload $(TEST_DATA) + --log-type Tail \ + --function-name $$(aws cloudformation describe-stacks \ + --stack-name $(CFN_NAME) \ + --output text \ + --query 'Stacks[0].Outputs[?OutputKey==`LambdaArn`].OutputValue' \ + ) \ + test-response.txt + grep $(TEST_RESULT) test-response.txt diff --git a/README.md b/README.md new file mode 100644 index 0000000..e8000ae --- /dev/null +++ b/README.md @@ -0,0 +1,33 @@ +# Overview + +A simple example project to use SAM/CodeDeploy preTrafficHooks +to validate a lambda before finishing the deployment + +## Install, Linting, Testing, Deploying, Integration-Testing + +* Code uses black for formatting and pylint for code-standards +* Tests use pytest +* Deployment uses cloudformation via [SAM] for simplification +* Integration Tests are part of the cloudformation + +See `make help` for more details + +### Deployment + +Environment overrides can be written in `environments/.env` +then launched with `make all env=` + +#### CI + +This folder has scripts to + +* Deploy and Delete a CI environment +* Deploy a permanent environment based on `environments/.env` +* Cleanup environments after merging + +## Contributing + +Before contributing, please run all tests in a clean environment, +e.g run `make all` + +[SAM]: https://github.com/awsdocs/aws-sam-developer-guide/blob/master/doc_source/sam-specification.md diff --git a/ci/cleanup.sh b/ci/cleanup.sh new file mode 100644 index 0000000..ab968e8 --- /dev/null +++ b/ci/cleanup.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Delete a deployed environment after a merge +# checks parent commits for environment/.env + +PROTECTED=master + +function try_and_delete_environment () { + if [ -f "environments/${1}.env" ]; then + make delete env="${1}" + exit 0 + fi +} + +MERGED_ENVIRONMENT="${DRONE_SOURCE_BRANCH##*/}" +if [[ ! "$MERGED_ENVIRONMENT" =~ $PROTECTED ]]; then + try_and_delete_environment "$MERGED_ENVIRONMENT" + + echo "Checking out parent commits for env file" + for PARENT in $(git log --pretty=%P -n 1 | tac -s ' '); do + git checkout "$PARENT" + try_and_delete_environment "$MERGED_ENVIRONMENT" + done +fi diff --git a/ci/deploy.sh b/ci/deploy.sh new file mode 100644 index 0000000..67b1183 --- /dev/null +++ b/ci/deploy.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +PROTECTED=master + +ENVIRONMENT="${DRONE_BRANCH##*/}" +[ -z "$ENVIRONMENT" ] && ENVIRONMENT="$DRONE_TAG" + +if [[ "$ENVIRONMENT" =~ $PROTECTED ]] && [ "$ENVIRONMENT" != "$DRONE_BRANCH" ] ; then + echo "Can only deploy to $PROTECTED from named branches" + unset ENVIRONMENT +fi + +if [ -f "environments/${ENVIRONMENT}.env" ] ; then + make all "env=${ENVIRONMENT}" +else + make all env=ci + make delete env=ci +fi diff --git a/cloudformation/lambda-sam-template.yaml b/cloudformation/lambda-sam-template.yaml new file mode 100644 index 0000000..bc01186 --- /dev/null +++ b/cloudformation/lambda-sam-template.yaml @@ -0,0 +1,103 @@ +--- +AWSTemplateFormatVersion: 2010-09-09 +Transform: AWS::Serverless-2016-10-31 +Description: An Lambda function with a monitoring alarm, DLQ & PreTraffic Test +Metadata: + License: magnet:?xt=urn:btih:1f739d935676111cfff4b4693e3816e664797050&dn=gpl-3.0.txt GPL-v3-or-Later + cfn-lint: + config: + ignore_checks: + - W3011 # UpdatePolicy generated by SAM + +Parameters: + NotificationEmail: + Type: String + TestData: + Type: String + TestExpected: + Type: String + +Resources: + Function: + Type: AWS::Serverless::Function + Properties: + Handler: lambda_function.handler + Runtime: python3.7 + CodeUri: src/lambda_function.py + DeadLetterQueue: + TargetArn: !GetAtt DeadLetterQueue.Arn + Type: SQS + AutoPublishAlias: live + DeploymentPreference: + Type: CodeDeployDefault.LambdaAllAtOnce + Alarms: + - !Ref Alarm + Hooks: + PreTraffic: !Ref TestFunction + + + Alarm: + Type: AWS::CloudWatch::Alarm + Properties: + ActionsEnabled: true + AlarmActions: + - !Ref NotificationTopic + AlarmDescription: Notify on failures of cloud-interest-rate-calculator function + AlarmName: !Sub ${AWS::StackName}-errors + MetricName: Errors + Namespace: AWS/Lambda + Statistic: Sum + Dimensions: + - Name: FunctionName + Value: !Select [6, !Split [':', !GetAtt Function.Arn]] + - Name: Resource + Value: !Select [6, !Split [':', !GetAtt Function.Arn]] + Period: 300 + EvaluationPeriods: 1 + DatapointsToAlarm: 1 + Threshold: 0 + ComparisonOperator: GreaterThanThreshold + + NotificationTopic: + Type: AWS::SNS::Topic + Properties: + DisplayName: !Sub ${AWS::StackName}-errors + Subscription: + - Endpoint: !Ref NotificationEmail + Protocol: email + Tags: + - Key: Type + Value: Monitoring + + DeadLetterQueue: + Type: AWS::SQS::Queue + Properties: + Tags: + - Key: Type + Value: DeadLetterQueue + + TestFunction: + Type: AWS::Serverless::Function + Properties: + FunctionName: !Sub CodeDeployHook_PreHook_${AWS::StackName} + Handler: lambda_verification.handler + Runtime: python3.7 + CodeUri: src/lambda_verification.py + Environment: + Variables: + CurrentVersion: !Ref Function.Version + TestData: !Ref TestData + TestExpected: !Ref TestExpected + Policies: + - Version: 2012-10-17 + Statement: + - Effect: Allow + Action: codedeploy:PutLifecycleEventHookExecutionStatus + Resource: "*" + - Effect: Allow + Action: lambda:InvokeFunction + Resource: !Sub "${Function.Arn}:*" + +Outputs: + LambdaArn: + Value: !GetAtt Function.Arn diff --git a/environments/ci.env b/environments/ci.env new file mode 100644 index 0000000..b9f2145 --- /dev/null +++ b/environments/ci.env @@ -0,0 +1 @@ +CFN_NAME = example-cloud-function-ci-$(git rev-parse HEAD | cut -c 1-8) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..fdacef4 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +awscli==1.* +black==19.10b0 +cfn-lint==0.28.* +mock==4.* +pylint==2.* +pytest==5.* +yamllint==1.20.* +pytest-cov==2.8.* +shellcheck-py==0.7.* diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/lambda_function.py b/src/lambda_function.py new file mode 100644 index 0000000..042e15f --- /dev/null +++ b/src/lambda_function.py @@ -0,0 +1,45 @@ +""" An Simple example lambda""" +from typing import Set + +# pylint: disable=unused-argument +def handler(event, context): + """ + Lambda event handler takes a number and returns it's factors + + event must have an value key which can be cast to an int + + return value is a comma separated list of numbers as a string + """ + try: + value = int(event["value"]) + except Exception as error: + print(f"Failure\tFailed to read value\t{error}\t{event}") + raise error + try: + factors = get_factors(value) + response = stringify(factors) + print(f"Success\tCalculated\t{value}\t{response}") + return response + except Exception as error: + print(f"Failure\tFailed to calculate response\t{error}\t{value}") + raise error + + +def get_factors(number: int) -> Set[int]: + """ Calculate factors of an integer """ + result = set() + for i in range(1, int(number ** 0.5) + 1): + div, mod = divmod(number, i) + if mod == 0: + result.add(i) + result.add(div) + return result + + +def stringify(numbers: Set[int]) -> str: + """ Convert a list of numbers to an order comma separated list """ + numbers = list(numbers) + numbers.sort() + numbers = [str(i) for i in numbers] + response = ",".join(numbers) + return response diff --git a/src/lambda_verification.py b/src/lambda_verification.py new file mode 100644 index 0000000..ba77eab --- /dev/null +++ b/src/lambda_verification.py @@ -0,0 +1,51 @@ +""" Generic lambda function tester that uses environmental variables """ +# pylint: disable=broad-except +import os +import boto3 + +AWSLAMBDA = boto3.client("lambda") +CODEDEPLOY = boto3.client("codedeploy") + +# pylint: disable=unused-argument +def handler(event, context): + """ Entry point for test """ + try: + function = os.environ["CurrentVersion"] + expected = os.environ["TestExpected"] + payload = os.environ["TestData"] + + codedeploy_params = { + "deploymentId": event["DeploymentId"], + "lifecycleEventHookExecutionId": event["LifecycleEventHookExecutionId"], + "status": "Failed", + } + except Exception as error: + print(f"Exception, failed to unpack event, {function}, {error}, event, {event}") + raise error + + try: + print(f"Info, testing, {function}, {payload}") + response = AWSLAMBDA.invoke(FunctionName=function, Payload=payload) + response_value = response["Payload"].read() + except Exception as error: + print(f"Failed, invocation failed, {function}, {error}") + CODEDEPLOY.put_lifecycle_event_hook_execution_status(**codedeploy_params) + return + + try: + assert response_value == expected + except Exception as error: + print( + f"Failed, incorrect response, {function}, {error}, expected, {expected}, actual, {response}" + ) + CODEDEPLOY.put_lifecycle_event_hook_execution_status(**codedeploy_params) + return + + try: + print(f"Success, notifying codedeploy, {function}, {event['DeploymentId']}") + codedeploy_params["status"] = "Succeeded" + CODEDEPLOY.put_lifecycle_event_hook_execution_status(**codedeploy_params) + return + except Exception as error: + print(f"Exception, failed to notify codedeploy, {function}, {error}") + raise error diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_lambda_function.py b/tests/test_lambda_function.py new file mode 100644 index 0000000..d59010b --- /dev/null +++ b/tests/test_lambda_function.py @@ -0,0 +1,46 @@ +""" Tests for cloud_interest_rate_calculator """ +# pylint: disable=no-self-use,missing-function-docstring,missing-class-docstring + +import pytest +import mock +from src import lambda_function + + +class TestExampleFunction: + @pytest.mark.parametrize("event", [{"value": "One Thousand"}, {"not_value": 1000}]) + def test_handler_bad_input(self, event): + with pytest.raises(Exception): + assert lambda_function.handler(event, {}) + + @mock.patch("src.lambda_function.get_factors") + def test_handler_unexpected_error(self, mock_calculate_interest): + event = {"value": "10"} + mock_calculate_interest.side_effect = Exception() + with pytest.raises(Exception): + assert lambda_function.handler(event, {}) + + def test_handler(self): + event = {"value": "10"} + response = lambda_function.handler(event, {}) + assert response == "1,2,5,10" + + @pytest.mark.parametrize( + "initial,expected", + [ + (1, {1}), + (10, {1, 2, 5, 10}), + (60, {1, 2, 3, 4, 5, 6, 10, 12, 15, 20, 30, 60}), + (64, {1, 2, 4, 8, 16, 32, 64}), + ], + ) + def test_get_factors(self, initial, expected): + response = lambda_function.get_factors(initial) + assert response == expected + + @pytest.mark.parametrize( + "initial,expected", + [([], ""), ([1], "1"), ([2], "2"), ([1, 2, 3], "1,2,3"), ([1, 3, 2], "1,2,3"),], + ) + def test_stringify(self, initial, expected): + response = lambda_function.stringify(initial) + assert response == expected diff --git a/tests/test_lambda_verification.py b/tests/test_lambda_verification.py new file mode 100644 index 0000000..42baf0a --- /dev/null +++ b/tests/test_lambda_verification.py @@ -0,0 +1,66 @@ +""" Tests for lambda_verification function """ +# pylint: disable=no-self-use,missing-function-docstring,missing-class-docstring + +import os +import pytest +import mock +from src import lambda_verification + +PAYLOAD = { + "DeploymentId": "TestDeployment001", + "LifecycleEventHookExecutionId": "LifeCycleTest001", +} + +os.environ["CurrentVersion"] = "CurVer" +os.environ["TestData"] = '{"value":"10"}' +os.environ["TestExpected"] = "2,5" + + +class TestLambdaVerfication: + def test_handler_bad_payload(self): + with pytest.raises(Exception): + assert lambda_verification.handler({}, {}) + + @mock.patch("src.lambda_verification.CODEDEPLOY") + @mock.patch("src.lambda_verification.AWSLAMBDA") + @pytest.mark.parametrize( + "response,expected", [("2,4", "Failed"), ("2,5", "Succeeded")], + ) + def test_handler(self, mock_lambda, mock_codedeploy, response, expected): + response_mock = mock.MagicMock() + response_mock.read.return_value = response + mock_lambda.invoke.return_value = {"Payload": response_mock} + + lambda_verification.handler(PAYLOAD, {}) + + mock_codedeploy.put_lifecycle_event_hook_execution_status.assert_called_with( + status=expected, + deploymentId="TestDeployment001", + lifecycleEventHookExecutionId="LifeCycleTest001", + ) + + @mock.patch("src.lambda_verification.CODEDEPLOY") + @mock.patch("src.lambda_verification.AWSLAMBDA") + def test_handler_lambda_failure(self, mock_lambda, mock_codedeploy): + mock_lambda.invoke.side_effect = Exception() + with pytest.raises(Exception): + assert lambda_verification.handler(PAYLOAD, {}) + + mock_codedeploy.put_lifecycle_event_hook_execution_status.assert_called_with( + status="Failed", + deploymentId="TestDeployment001", + lifecycleEventHookExecutionId="LifeCycleTest001", + ) + + @mock.patch("src.lambda_verification.CODEDEPLOY") + @mock.patch("src.lambda_verification.AWSLAMBDA") + def test_handler_codedeploy_failure(self, mock_lambda, mock_codedeploy): + response_mock = mock.MagicMock() + response_mock.read.return_value = "2,5" + mock_lambda.invoke.return_value = {"Payload": response_mock} + mock_codedeploy.put_lifecycle_event_hook_execution_status.side_effect = ( + Exception() + ) + + with pytest.raises(Exception): + assert lambda_verification.handler(PAYLOAD, {}) diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..1eed7b0 --- /dev/null +++ b/tox.ini @@ -0,0 +1,2 @@ +[pytest] +addopts = --cov-branch --cov-report=term-missing --cov=src --cov-fail-under=100