.gitlab-ci.yml

image: python:3.7

stages:
    - test
    - test_functional_init
    - test_functional_execute
    - test_functional_cleanup
    - docs
    - publish
    - publish_test

variables:
    ARTIFACTORY_SERVER: "$ARTIFACTORY_SERVER"
    DEPLOYMENT_PLAN: aws
    EXTENSION_METADATA_FILE: "metadata.json"
    F5_CS_USER: "$CS_USERNAME"
    F5_CS_PWD: "$CS_PASSWORD"
    GIT_SECRETS_VERSION: '1.3.0'
    GIT_SECRETS_PKG_URL: "https://github.com/awslabs/git-secrets/archive/${GIT_SECRETS_VERSION}.tar.gz"
    DOCS_BUILD_DIR: "code_docs/html"
    TWINE_USERNAME: "$PYPI_USERNAME"
    TWINE_PASSWORD: "$PYPI_PASSWORD"
    TWINE_NON_INTERACTIVE: "true"
    # aws vars
    AWS_ACCESS_KEY_ID: "$AWS_ACCESS_KEY_ID"
    AWS_SECRET_ACCESS_KEY: "$AWS_SECRET_ACCESS_KEY"
    AWS_DEFAULT_REGION: "$AWS_DEFAULT_REGION"
    # debug vars
    F5_SDK_LOG_LEVEL: "INFO"
    F5_DISABLE_SSL_WARNINGS: 'true'

# unit tests: python 3.x (preferred)
test_unit:
    stage: test
    script:
        # install packages
        - pip install -r requirements.txt
        - pip install .
        # run unit tests
        - make test
    tags:
        - cm-official-docker-executor

# unit tests: python 2.x
test_unit_python_2_x:
    image: python:2.7
    stage: test
    script:
        # install packages
        - pip install -r requirements.txt
        - pip install .
        # run unit tests
        - make test
    tags:
        - cm-official-docker-executor

# run linter
test_lint:
    stage: test
    script:
        # install packages
        - pip install -r requirements.txt
        - pip install .
        # run linter
        - make lint
    tags:
        - cm-official-docker-executor

# dependency security check
test_security_check:
    stage: test
    script:
        # install packages
        - pip install -r requirements.txt
        - pip install .
        # run dependency check
        - safety check -r requirements.txt
    tags:
        - cm-official-docker-executor

# test extension metadata file is current
# note: the file is checked into the repo
# to enable all extension client scenarios
test_extension_metadata_current:
    stage: test
    script:
        # install packages
        - pip install -r requirements.txt
        - pip install .
        # generate metadata file
        - make generate_extension_metadata
        # check if file is out of date
        - changed_files_count=$(git diff --name-only | wc -l)
        - echo ${changed_files_count}
        - if [[ $changed_files_count -ne 0 ]]; then echo "Extension metadata is out of date, please update"; exit 1; else echo "Metadata file is unchanged"; fi
    tags:
        - cm-official-docker-executor
    only:
        variables:
            - $CI_COMMIT_REF_NAME == "master"
            - $CI_COMMIT_REF_NAME == "develop"
            - $PUBLISH_EXTENSION_METADATA == "true"
    artifacts:
        name: ${CI_COMMIT_REF_NAME}_extension_metadata
        paths:
            - ${EXTENSION_METADATA_FILE}
        expire_in: 1 month

# check for disallowed content within any file in source control, this
# supplements the native pre-receive push rules built into GitLab for secrets
# note: .gitallowed is used for exceptions
check_content:
    image: f5devcentral/containthedocs:rpmbuild
    stage: test
    script:
        # install git-secrets
        - curl -q -L -o git_secrets.tar.gz ${GIT_SECRETS_PKG_URL}
        - tar xzf git_secrets.tar.gz
        - cd git-secrets-${GIT_SECRETS_VERSION} && make install && cd ..
        # now, add any patterns to disallow
        - git secrets --add '.*f5.*\.com'
        # scan
        - git secrets --scan
    tags:
        - cm-official-docker-executor

# deploy function test environment
test_functional_init:
    stage: test_functional_init
    image: ${ARTIFACTORY_SERVER}/ecosystems-cloudsolutions-docker-dev/deployment-tool:latest
    retry:
        max: 1
    script:
        - /deployment-tool/deploy.sh --deployment-plan ${DEPLOYMENT_PLAN} --action create --output-folder env_metadata/${DEPLOYMENT_PLAN}
    tags:
        - cm-official-docker-executor
    only:
        variables:
            - $CI_COMMIT_REF_NAME == "master"
            - $CI_COMMIT_REF_NAME == "develop"
            - $RUN_FUNCTIONAL_TESTS == "true"
            - $CI_COMMIT_MESSAGE =~ /smart:run_functional_tests/
    artifacts:
        paths:
            - env_metadata/${DEPLOYMENT_PLAN}
        when: always
        expire_in: 1 month

# run functional tests
test_functional_execute:
    stage: test_functional_execute
    retry:
        max: 1
    script:
        # install packages
        - pip install -r requirements.txt
        - pip install .
        # copying artifacts inherited from functional_test_init target to deployment root directory
        - cp -a env_metadata/${DEPLOYMENT_PLAN}/. .
        # run function tests
        - make functional-test
        # run function tests examples
        - make functional-test-examples
    tags:
        - cm-official-docker-executor
    only:
        variables:
            - $CI_COMMIT_REF_NAME == "master"
            - $CI_COMMIT_REF_NAME == "develop"
            - $RUN_FUNCTIONAL_TESTS == "true"
            - $CI_COMMIT_MESSAGE =~ /smart:run_functional_tests/

test_functional_cleanup:
    stage: test_functional_cleanup
    image: ${ARTIFACTORY_SERVER}/ecosystems-cloudsolutions-docker-dev/deployment-tool:latest
    retry:
        max: 1
    when: always
    script:
        # copying artifacts inherited from functional_test_init target to root directory
        - cp -a env_metadata/${DEPLOYMENT_PLAN}/. .
        # teardown enviroment
        - /deployment-tool/deploy.sh --deployment-plan ${DEPLOYMENT_PLAN} --action delete   
    tags:
        - cm-official-docker-executor
    only:
        variables:
            - $CI_COMMIT_REF_NAME == "master"
            - $CI_COMMIT_REF_NAME == "develop"
            - $RUN_FUNCTIONAL_TESTS == "true"
            - $CI_COMMIT_MESSAGE =~ /smart:run_functional_tests/

# generate code coverage docs
docs_code_coverage:
    stage: docs
    script:
        # install packages
        - pip install -r requirements.txt
        - pip install .
        # generate coverage
        - make coverage
    tags:
        - cm-official-docker-executor
    artifacts:
        name: ${CI_COMMIT_REF_NAME}_code_coverage
        paths:
            - code_coverage
        expire_in: 1 month

# generate code docs
docs_code:
    stage: docs
    script:
        # install packages
        - pip install -r requirements.txt
        - pip install .
        # install apt packages: doxygen
        - apt-get update
        - apt-get install -y doxygen
        # generate docs
        - make code_docs
    tags:
        - cm-official-docker-executor
    artifacts:
        name: ${CI_COMMIT_REF_NAME}_code_docs
        paths:
            - code_docs
        expire_in: 1 month

# Generate and publish extension metadata to CDN
# note: assumes metadata generator has been run
# in a previous stage and set as an artifact
publish_extension_metadata:
    stage: publish
    only:
        variables:
            - $CI_COMMIT_REF_NAME == "master"
            - $CI_COMMIT_REF_NAME == "develop"
            - $PUBLISH_EXTENSION_METADATA == "true"
    tags:
        - docker-executor
    script:
        # publish extension metadata to F5 CDN
        - CDN_FOLDER="f5-extension-metadata"
        - AUTH_OPTS="--username ${CDN_SVC_ACCOUNT_USER} --password ${CDN_SVC_ACCOUNT_PWD} --non-interactive"
        - svn co ${F5_CDN_SVN_ROOT}/cloudsolutions/${CDN_FOLDER} ${AUTH_OPTS}
        - cp ${EXTENSION_METADATA_FILE} ${CDN_FOLDER}/latest/
        - cd ${CDN_FOLDER}
        - svn diff
        - changed_files_count=$(svn diff --summarize | wc -l)
        - if [[ $changed_files_count -eq 1 ]]; then svn add --force latest/${EXTENSION_METADATA_FILE}; svn commit -m "F5 SDK project automation - ${CI_COMMIT_REF_NAME}" ${AUTH_OPTS}; else echo "Metadata file is unchanged"; fi

# Publish to internal artifactory
# Note: Will build and push package when new tags are added with
# the assumption being the package version has been updated.
# However even if it hasn't it will simply update the existing version
publish_egg_to_artifactory:
    stage: publish
    only:
        - tags
        - develop
    tags:
        - docker-executor
    script:
        - make build
        - EGG_FILE=$(ls dist/*.gz)
        - EGG_NAME=$(basename $EGG_FILE)
        - URL=${ARTIFACTORY_BASE_URL}/f5-cloud-solutions-pypi/f5-sdk-python/${EGG_NAME}
        - echo ${URL}
        - >-
          UPLOAD_RESULT=$(curl -H "Authorization: Bearer ${ARTIFACTORY_TOKEN}" -X PUT --data-binary @${EGG_FILE} ${URL})
        - if [[ $? -eq 0 ]] && [[ "$UPLOAD_RESULT" == *created* ]]; then echo "Upload complete"; else echo "Upload failed"; exit 1; fi
        - echo $UPLOAD_RESULT

# Publish to PyPI
# Note: Will build and push package when new tags are added with
# the assumption being the package version has been updated
publish_package_to_pypi:
    stage: publish
    only:
        - tags
    tags:
        - docker-executor
    script:
        # install packages
        - pip install -r requirements.txt
        # upload
        - make upload

# publish docs to internal pages: - this job MUST be named 'pages'
pages:
    stage: publish
    environment:
        name: staging
        url: https://${CI_PROJECT_NAMESPACE}.${PAGES_DOMAIN}/${CI_PROJECT_NAME}
    tags:
        - cm-official-docker-executor
    script:
        - echo "URL https://${CI_PROJECT_NAMESPACE}.${PAGES_DOMAIN}/${CI_PROJECT_NAME}"
        - PUBLIC_DIR='./public'
        - mkdir -p ${PUBLIC_DIR}
        # place index.html in public dir
        - cp docs/index.html ${PUBLIC_DIR}/index.html
        # place code coverage artifacts under /coverage
        - mkdir -p ${PUBLIC_DIR}/coverage
        - cp -R code_coverage/* ${PUBLIC_DIR}/coverage
        # place code docs artifacts under /code-docs
        - mkdir -p ${PUBLIC_DIR}/code-docs
        - cp -R code_docs/html/* ./public/code-docs
        # etc...
    artifacts:
        paths:
            - public
    only:
        - develop

# Publish docs to clouddocs.f5networks.com
docs_to_staging:
    image: ${CONTAINTHEDOCS_IMAGE}
    stage: publish
    environment:
        name: stage
        url: https://clouddocs.f5networks.net/sdk/f5-sdk-python
    only:
        variables:
            - $CI_COMMIT_REF_NAME == "develop"
            - $CI_COMMIT_REF_NAME == "docs_staging"
    tags:
        - cm-official-docker-executor
    script:
        - aws configure set aws_access_key_id $DOCS_AWS_ACCESS_KEY_ID --profile=docs
        - aws configure set aws_secret_access_key $DOCS_AWS_SECRET_ACCESS_KEY --profile=docs
        # Uncomment and set to create desired version format
        - aws s3 sync ${DOCS_BUILD_DIR} s3://clouddocs.f5networks.net/sdk/f5-sdk-python --profile=docs
        # create invalidation to clear cloudfront cache
        - aws cloudfront create-invalidation --distribution-id $AWS_DIST_STAGE --paths /sdk/f5-sdk-python --profile=docs

# Publish docs to clouddocs.f5.com
docs_to_production:
    image: ${CONTAINTHEDOCS_IMAGE}
    stage: publish
    environment:
        name: stage
        url: https://clouddocs.f5.com/sdk/f5-sdk-python
    only:
        variables:
            - $CI_COMMIT_REF_NAME == "master"
            - $CI_COMMIT_REF_NAME == "docs_production"
    tags:
        - cm-official-docker-executor
    script:
        - aws configure set aws_access_key_id $DOCS_AWS_ACCESS_KEY_ID --profile=docs
        - aws configure set aws_secret_access_key $DOCS_AWS_SECRET_ACCESS_KEY --profile=docs
        # Uncomment and set to create desired version format
        - aws s3 sync ${DOCS_BUILD_DIR} s3://clouddocs.f5.com/sdk/f5-sdk-python --profile=docs
        # create invalidation to clear cloudfront cache
        - aws cloudfront create-invalidation --distribution-id $AWS_DIST_STAGE --paths /sdk/f5-sdk-python --profile=docs

# publish test: test internal (artifactory) install
publish_test_internal_install:
    stage: publish_test
    only:
        - tags
    script:
        # install package - first wait until it is available after a recent publish (known artifactory issue)
        - bash scripts/wait_for_resp.sh ${ARTIFACTORY_BASE_URL}/api/pypi/f5-cloud-solutions-pypi/simple/f5-sdk-python/
        - pip install f5-sdk-python --extra-index-url ${ARTIFACTORY_BASE_URL}/api/pypi/f5-cloud-solutions-pypi/simple
    tags:
        - cm-official-docker-executor

# publish test: test public (pypi) install
publish_test_public_install:
    stage: publish_test
    only:
        - tags
    script:
        # install package
        - pip install f5-sdk-python
    tags:
        - cm-official-docker-executor