From cae572ea1dab063f26499a134b2a1c529b6e23e4 Mon Sep 17 00:00:00 2001 From: Stefano Vozza Date: Mon, 9 Sep 2019 17:17:35 +0100 Subject: [PATCH] compile code in docker and version backend --- cfn-publish.config | 1 + package.json | 24 ++++++++++-------- src/backend/functions/setup/s3-handler.js | 31 ++++++++++++----------- src/backend/transcriber/Dockerfile | 2 +- src/backend/transcriber/pom.xml | 4 +-- src/cfn/template.yaml | 8 ++++-- 6 files changed, 40 insertions(+), 30 deletions(-) diff --git a/cfn-publish.config b/cfn-publish.config index e6d05d2..7d747ee 100644 --- a/cfn-publish.config +++ b/cfn-publish.config @@ -2,3 +2,4 @@ template=templates/template.yaml acl="public-read" bucket_name_prefix="solution-builders" regions="ap-southeast-2 ca-central-1 eu-west-1 us-east-1 us-east-2 us-west-2" +extra_files="transcriber.zip frontend.zip" diff --git a/package.json b/package.json index 956b881..2c99254 100644 --- a/package.json +++ b/package.json @@ -1,20 +1,18 @@ { "name": "amazon-transcribe-news-media-analysis", - "version": "0.5.0", "private": true, "scripts": { - "backend": "npm run backend-build && npm run backend-deploy", - "backend-build": "cd src/backend/transcriber/ && mvn package && npm run backend-package", - "backend-package": "cd src/backend/transcriber/ && zip backend.zip Dockerfile target/transcriber-1.0-SNAPSHOT-jar-with-dependencies.jar src/main/resources/log4j2.xml", + "backend": "npm run backend-package && npm run backend-deploy", + "backend-package": "cd src/backend/transcriber/ && zip -r ../../../backend/transcriber.zip Dockerfile pom.xml src/", "backend-deploy": "aws s3 cp src/backend/transcriber/backend.zip s3://media-analysis-demo/build/ --acl public-read", - "build": "npm run webui-build && npm run lambda-build && npm run cfn-build && npm run backend-build", + "build": "npm run webui-build && npm run lambda-build && npm run cfn-build && npm run backend-package && npm run webui-package", "cfn-build": "npm run cfn-dist && cfn-flip -c -n -l src/cfn/template.yaml templates/template.yaml && npm run cfn-lambdas-dist && npm run cfn-copy-lambdas", "cfn-copy-lambdas": "npm run cfn-copy-lambdas-aws_sdk && npm run cfn-copy-lambdas-fp && npm run cfn-copy-lambdas-orchestrator && npm run cfn-copy-lambdas-setup", "cfn-copy-lambdas-aws_sdk": "cp -R src/backend/functions/layers/aws_sdk ./backend/functions/layers", "cfn-copy-lambdas-fp": "cp -R src/backend/functions/layers/fp ./backend/functions/layers", "cfn-copy-lambdas-orchestrator": "cp -R src/backend/functions/orchestrator/dist ./backend/functions/orchestrator/", "cfn-copy-lambdas-setup": "cp -R src/backend/functions/setup/dist ./backend/functions/setup/", - "cfn-deploy": "npm run zip && aws s3 cp ./v$npm_package_version.zip s3://$CFN_BUCKET/$npm_package_name/v$npm_package_version/$npm_package_name.zip", + "cfn-deploy": "npm run zip && aws s3 cp ./`npm run echo-version --silent`.zip s3://$CFN_BUCKET/$npm_package_name/v`npm run echo-version --silent`/$npm_package_name.zip", "cfn-dist": "if [ ! -d './templates' ]; then mkdir templates; fi", "cfn-lambdas-dist": "npm run cfn-lambdas-dist-aws_sdk && npm run cfn-lambdas-dist-fp && npm run cfn-lambdas-dist-orchestrator && npm run cfn-lambdas-dist-setup", "cfn-lambdas-dist-aws_sdk": "if [ ! -d './backend/functions/layers/aws_sdk' ]; then mkdir -p backend/functions/layers/aws_sdk; fi", @@ -24,7 +22,7 @@ "cfn-test": "cfn-lint templates/template.yaml", "cfn-test-local": "npm run lambda-build && npm run cfn-build && aws cloudformation package --template-file templates/template.yaml --output-template-file templates/packaged.yaml --s3-bucket $TEST_BUCKET && aws cloudformation deploy --template-file templates/packaged.yaml --stack-name MediaAnalysis --capabilities CAPABILITY_IAM", "deploy": "npm run webui-deploy && npm run backend-deploy && npm run cfn-deploy", - "echo-version": "echo v$npm_package_version", + "echo-version": "node -e \"const {readSync} = require('node-yaml'); console.log(readSync('templates/template.yaml').Globals.Function.Environment.Variables.VERSION)\"", "lambda-build": "npm run lambda-build-layers && npm run lambda-build-orchestrator && npm run lambda-build-setup", "lambda-build-layers": "cd src/backend/functions/layers && npm run build", "lambda-build-orchestrator": "cd src/backend/functions/orchestrator && npm run build", @@ -34,13 +32,14 @@ "postinstall": "cd src/frontend && npm i && cd ../backend/functions/setup && npm i && cd ../orchestrator && npm i", "pre-deploy-test": "taskcat -c ci/taskcat.yml --verbose --exclude src --exclude docs --exclude taskcat_outputs", "start": "cd src/frontend && npm start", - "tag": "git tag v$npm_package_version", + "tag": "git tag v`npm run echo-version --silent`", "test": "npm run cfn-test && npm run webui-test && npm run orch-test", "webui-build": "cd src/frontend && npm run build", "webui-deploy": "cd src/frontend/build && aws s3 cp . s3://media-analysis-demo/static --recursive --acl public-read --exclude settings.js --exclude .DS_Store", + "webui-package": "cd src/frontend/build && zip -r ../../../backend/frontend.zip .", "webui-deploy-clean": "aws s3 rm s3://media-analysis-demo/static --recursive", "webui-test": "cd src/frontend && npm t", - "zip": "zip -r v$npm_package_version.zip -@ < ci/include.lst" + "zip": "zip -r v`npm run echo-version --silent`.zip -@ < ci/include.lst" }, "repository": { "type": "git", @@ -49,5 +48,10 @@ "bugs": { "url": "https://github.com/aws-samples/amazon-transcribe-news-media-analysis/issues" }, - "homepage": "https://github.com/aws-samples/amazon-transcribe-news-media-analysis#readme" + "homepage": "https://github.com/aws-samples/amazon-transcribe-news-media-analysis#readme", + "devDependencies": { + "js-yaml": "3.13.1", + "node-yaml": "4.0.1" + }, + "dependencies": {} } diff --git a/src/backend/functions/setup/s3-handler.js b/src/backend/functions/setup/s3-handler.js index ab9a557..9da5f14 100644 --- a/src/backend/functions/setup/s3-handler.js +++ b/src/backend/functions/setup/s3-handler.js @@ -5,10 +5,11 @@ const { FROM_BUCKET, MAX_TASKS, REGION, - TO_BUCKET + TO_BUCKET, + VERSION } = process.env; -const BACKEND_PATH = "build/backend.zip"; +const BACKEND_PATH = `amazon-transcribe-news-media-analysis/v${VERSION}/transcriber.zip`; const CONFIG_FILENAME = "settings.js"; const FROM_PREFIX = "static/"; @@ -31,19 +32,19 @@ module.exports = s3 => { }; return { - copyFiles: () => - listFiles({ Bucket: FROM_BUCKET, Prefix: FROM_PREFIX }).then(result => - Promise.all( - result.Contents.map(file => - copyFile({ - ACL: "public-read", - Bucket: TO_BUCKET, - CopySource: `${FROM_BUCKET}/${file.Key}`, - Key: file.Key.slice(FROM_PREFIX.length) - }) - ) - ) - ), + copyFiles: () => Promise.resolve(), + // listFiles({ Bucket: FROM_BUCKET, Prefix: FROM_PREFIX }).then(result => + // Promise.all( + // result.Contents.map(file => + // copyFile({ + // ACL: "public-read", + // Bucket: TO_BUCKET, + // CopySource: `${FROM_BUCKET}/${file.Key}`, + // Key: file.Key.slice(FROM_PREFIX.length) + // }) + // ) + // ) + //), removeFiles: () => Promise.all([ diff --git a/src/backend/transcriber/Dockerfile b/src/backend/transcriber/Dockerfile index 87dd02a..3acef1f 100644 --- a/src/backend/transcriber/Dockerfile +++ b/src/backend/transcriber/Dockerfile @@ -18,7 +18,7 @@ RUN mvn dependency:go-offline -B COPY src/ /build/src/ -RUN mvn package +RUN mvn package -o # Main Dockerfile diff --git a/src/backend/transcriber/pom.xml b/src/backend/transcriber/pom.xml index 7413ecd..ffe30df 100644 --- a/src/backend/transcriber/pom.xml +++ b/src/backend/transcriber/pom.xml @@ -95,10 +95,10 @@ 3.1.1 - unpack-dependencies + copy-dependencies package - unpack-dependencies + copy-dependencies diff --git a/src/cfn/template.yaml b/src/cfn/template.yaml index cb0f084..f7435c5 100644 --- a/src/cfn/template.yaml +++ b/src/cfn/template.yaml @@ -8,6 +8,10 @@ Globals: AllowMethods: "'GET,POST,OPTIONS,DELETE,PUT'" AllowHeaders: "'Content-Type,X-Amz-Date,Authorization,X-Api-Key,X-Amz-Security-Token'" AllowOrigin: "'*'" + Function: + Environment: + Variables: + VERSION: '0.6' Parameters: @@ -413,7 +417,7 @@ Resources: API_GATEWAY: !Sub https://${RestApi}.execute-api.${AWS::Region}.amazonaws.com/PROD BUILD_BUCKET: !Ref CodeBuildArtifactBucket COGNITO_IDENTITY_POOL: !Ref CognitoIdentityPool - FROM_BUCKET: media-analysis-demo + FROM_BUCKET: !Sub solution-builders-${AWS::Region} MAX_TASKS: !Ref MaxTasksCapacity REGION: !Ref AWS::Region REPOSITORY: !Ref ECRRepository @@ -564,7 +568,7 @@ Resources: - docker push "$IMAGE_URI" Environment: ComputeType: BUILD_GENERAL1_SMALL - Image: aws/codebuild/docker:17.09.0 + Image: aws/codebuild/docker:18.09.0 Type: LINUX_CONTAINER EnvironmentVariables: - Name: AWS_DEFAULT_REGION