Continuous Integration & Delivery + version name from git tag (Mobili…

…tyData#455) Self reviewed: no code changes * CI optimizations - gradle caching - collect test reports on test failure * Computed gradle version tagging - copy what was done in the workflow tagging the Docker image - adapt it to generate a coherent version name for artifacts - compute version tagging in CI from Git branch/tagging/pr event - pass down computed tag to gradle vi an environment variable - use Gradle Command GitHub Action - use computed name for all artifacts - update gradle build scripts - update Dockerfile - update release documentation https://github.com/marketplace/actions/gradle-command https://docs.github.com/en/free-pro-team@latest/actions/guides/building-and-testing-java-with-gradle#caching-dependencies https://stackoverflow.com/questions/41428013/why-does-wildcard-for-jar-execution-not-work-in-docker-cmd - TODO: share name computation result between jobs https://docs.github.com/en/free-pro-team@latest/actions/guides/storing-workflow-data-as-artifacts#passing-data-between-jobs-in-a-workflow
LeoFrachet · Oct 26, 2020 · 985448c · 985448c
1 parent dfb7408
commit 985448c
Show file tree

Hide file tree

Showing 21 changed files with 227 additions and 127 deletions.
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -15,14 +15,44 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v1
+      - name: Prepare version name
+        id: prep
+        run: |
+          VERSION_TAG=edge
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            VERSION_TAG=${GITHUB_REF#refs/tags/}
+          elif [[ $GITHUB_REF == refs/heads/* ]]; then
+            VERSION_TAG=-$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g')
+          elif [[ $GITHUB_REF == refs/pull/* ]]; then
+            VERSION_TAG=-pr-${{ github.event.number }}
+          fi
+          if [ ${VERSION_TAG} != ${GITHUB_REF#refs/tags/} ]; then
+            VERSION_TAG=v${VERSION_TAG}-sha-${GITHUB_SHA::8}-SNAPSHOT
+          fi
+          echo ::set-output name=versionTag::${VERSION_TAG}
       - name: Set up JDK 1.11
         uses: actions/setup-java@v1
         with:
           java-version: 1.11
+      - name: Cache Gradle packages
+        uses: actions/cache@v2
+        with:
+          path: ~/.gradle/caches
+          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
+          restore-keys: ${{ runner.os }}-gradle
       - name: Run Java tests
-        run: ./gradlew testReport
+        uses: eskatos/gradle-command-action@v1
+        env:
+          versionTag: ${{ steps.prep.outputs.versionTag }}
+        with:
+          arguments: testReport
       - name: Run JS tests
-        run: ./gradlew uiTest
+        if: ${{ always() }} #we still want js test report if Java tests failed
+        uses: eskatos/gradle-command-action@v1
+        env:
+          versionTag: ${{ steps.prep.outputs.versionTag }}
+        with:
+          arguments: uiTest
   build_push:
     needs: test
     name: Build and push Docker image
@@ -48,14 +78,40 @@ jobs:
           echo ::set-output name=version::${VERSION}
           echo ::set-output name=tags::${TAGS}
           echo ::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ')
+          GRADLE_VERSION_TAG=edge
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            GRADLE_VERSION_TAG=${GITHUB_REF#refs/tags/}
+          elif [[ $GITHUB_REF == refs/heads/* ]]; then
+            GRADLE_VERSION_TAG=-$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g')
+          elif [[ $GITHUB_REF == refs/pull/* ]]; then
+            GRADLE_VERSION_TAG=-pr-${{ github.event.number }}
+          fi
+          if [ ${GRADLE_VERSION_TAG} != ${GITHUB_REF#refs/tags/} ]; then
+            GRADLE_VERSION_TAG=v${VERSION_TAG}-sha-${GITHUB_SHA::8}-SNAPSHOT
+          fi
+          echo ::set-output name=gradleVersionTag::${GRADLE_VERSION_TAG}
       - name: Set up JDK 1.11
         uses: actions/setup-java@v1
         with:
           java-version: 1.11
+      - name: Cache Gradle packages
+        uses: actions/cache@v2
+        with:
+          path: ~/.gradle/caches
+          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
+          restore-keys: ${{ runner.os }}-gradle
       - name: Package cli app jar with Gradle
-        run: ./gradlew shadowJar
+        uses: eskatos/gradle-command-action@v1
+        env:
+          versionTag: ${{ steps.prep.outputs.gradleVersionTag }}
+        with:
+          arguments: shadowJar
       - name: Package web app war with Gradle
-        run: ./gradlew bootWar
+        uses: eskatos/gradle-command-action@v1
+        env:
+          versionTag: ${{ steps.prep.outputs.gradleVersionTag }}
+        with:
+          arguments: bootWar
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
       - name: Login to GitHub Container Registry

diff --git a/.github/workflows/end_to_end.yml b/.github/workflows/end_to_end.yml
@@ -12,20 +12,45 @@ jobs:
 
     steps:
       - uses: actions/checkout@v1
+      - name: Prepare version name
+        id: prep
+        run: |
+          VERSION_TAG=edge
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            VERSION_TAG=${GITHUB_REF#refs/tags/}
+          elif [[ $GITHUB_REF == refs/heads/* ]]; then
+            VERSION_TAG=-$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g')
+          elif [[ $GITHUB_REF == refs/pull/* ]]; then
+            VERSION_TAG=-pr-${{ github.event.number }}
+          fi
+          if [ ${VERSION_TAG} != ${GITHUB_REF#refs/tags/} ]; then
+            VERSION_TAG=v${VERSION_TAG}-sha-${GITHUB_SHA::8}-SNAPSHOT
+          fi
+          echo ::set-output name=versionTag::${VERSION_TAG}
       - name: Set up JDK 1.11
         uses: actions/setup-java@v1
         with:
           java-version: 1.11
-      - name: Package jar with Gradle
-        run: ./gradlew shadowJar
-      #- name: Validate dataset from -- [[[ACRONYM]]]
-      #  run: java -jar application/cli-app/build/libs/gtfs-validator-v1.3.0-SNAPSHOT_cli.jar -u [[[DATASET_PUBLIC_URL]]] -i [[[ACRONYM]]].zip -e input -o output
+      - name: Cache Gradle packages
+        uses: actions/cache@v2
+        with:
+          path: ~/.gradle/caches
+          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
+          restore-keys: ${{ runner.os }}-gradle
+      - name: Package cli app jar with Gradle
+        uses: eskatos/gradle-command-action@v1
+        env:
+          versionTag: ${{ steps.prep.outputs.versionTag }}
+        with:
+          arguments: shadowJar
+      #- name: Validate dataset from -- ACRONYM
+      #  run: java -jar application/cli-app/build/libs/*.jar -u DATASET_PUBLIC_URL -i [[[ACRONYM]]].zip -e input -o output
       - name: Validate dataset from -- MBTA
-        run: java -jar application/cli-app/build/libs/gtfs-validator-v1.3.0-SNAPSHOT_cli.jar -u https://cdn.mbta.com/MBTA_GTFS.zip -i mbta.zip -e input -o output
+        run: java -jar application/cli-app/build/libs/*.jar -u https://cdn.mbta.com/MBTA_GTFS.zip -i mbta.zip -e input -o output
       - name: Validate dataset from issue 399 -- Monterey-Salinas Transit
-        run: java -jar application/cli-app/build/libs/gtfs-validator-v1.3.0-SNAPSHOT_cli.jar -u http://www.mst.org/google/google_transit.zip -i mst.zip -e input -o output
+        run: java -jar application/cli-app/build/libs/*.jar -u http://www.mst.org/google/google_transit.zip -i mst.zip -e input -o output
       - name: Validate dataset from issue 398 -- Orange County Transportation Authority
-        run: java -jar application/cli-app/build/libs/gtfs-validator-v1.3.0-SNAPSHOT_cli.jar -u https://octa.net/current/google_transit.zip -i octa.zip -e input -o output
+        run: java -jar application/cli-app/build/libs/*.jar -u https://octa.net/current/google_transit.zip -i octa.zip -e input -o output
       - name: Persist datasets
         uses: actions/upload-artifact@v2
         with:

diff --git a/.github/workflows/test_pack_doc.yml b/.github/workflows/test_pack_doc.yml
@@ -11,6 +11,21 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v1
+      - name: Prepare version name
+        id: prep
+        run: |
+          VERSION_TAG=edge
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            VERSION_TAG=${GITHUB_REF#refs/tags/}
+          elif [[ $GITHUB_REF == refs/heads/* ]]; then
+            VERSION_TAG=-$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g')
+          elif [[ $GITHUB_REF == refs/pull/* ]]; then
+            VERSION_TAG=-pr-${{ github.event.number }}
+          fi
+          if [ ${VERSION_TAG} != ${GITHUB_REF#refs/tags/} ]; then
+            VERSION_TAG=v${VERSION_TAG}-sha-${GITHUB_SHA::8}-SNAPSHOT
+          fi
+          echo ::set-output name=versionTag::${VERSION_TAG}
       - name: Set up JDK 1.11
         uses: actions/setup-java@v1
         with:
@@ -22,55 +37,118 @@ jobs:
           key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
           restore-keys: ${{ runner.os }}-gradle
       - name: Run Java tests
-        run: ./gradlew testReport
-      - name: Run JS tests
-        run: ./gradlew uiTest
-      - name: Persist Java tests reports
+        uses: eskatos/gradle-command-action@v1
+        env:
+          versionTag: ${{ steps.prep.outputs.versionTag }}
+        with:
+          arguments: testReport
+      - name: Persist **Successfull** Java tests merged report
         uses: actions/upload-artifact@v2
         with:
-          name: v1.3.0-SNAPSHOT_java_test_report
+          name: Test report - Java -- ${{ steps.prep.outputs.versionTag }}
           path: build/reports/allTests/
+      - name: Persist **Failed** Java tests unmerged reports
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: Test report - Java -- ${{ steps.prep.outputs.versionTag }}
+          path: |
+            adapter/exporter/build/reports/tests/
+            adapter/parser/build/reports/tests/
+            adapter/repository/in-memory-simple/build/reports/tests/
+            adapter/tree/build/reports/tests/
+            adapter/utils/build/reports/tests/
+            adapter/validator/build/reports/tests/
+            application/web-app/spring-server/build/reports/tests/
+            domain/build/reports/tests/
+            usecase/build/reports/tests/
+      - name: Run JS tests
+        if: ${{ always() }} #we still want js test report if Java tests failed
+        uses: eskatos/gradle-command-action@v1
+        env:
+          versionTag: ${{ steps.prep.outputs.versionTag }}
+        with:
+          arguments: uiTest
       - name: Persist JS tests report
+        if: ${{ always() }}
         uses: actions/upload-artifact@v2
         with:
-          name: v1.3.0-SNAPSHOT_js_test_report
+          name: Test report - JavaScript -- ${{ steps.prep.outputs.versionTag }}
           path: application/web-app/react-client/test-reports/
 
   pack:
-    needs: test
+    needs: [test]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v1
+      - name: Prepare version name
+        id: prep
+        run: |
+          VERSION_TAG=edge
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            VERSION_TAG=${GITHUB_REF#refs/tags/}
+          elif [[ $GITHUB_REF == refs/heads/* ]]; then
+            VERSION_TAG=-$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g')
+          elif [[ $GITHUB_REF == refs/pull/* ]]; then
+            VERSION_TAG=-pr-${{ github.event.number }}
+          fi
+          if [ ${VERSION_TAG} != ${GITHUB_REF#refs/tags/} ]; then
+            VERSION_TAG=v${VERSION_TAG}-sha-${GITHUB_SHA::8}-SNAPSHOT
+          fi
+          echo ::set-output name=versionTag::${VERSION_TAG}
       - name: Set up JDK 1.11
         uses: actions/setup-java@v1
         with:
           java-version: 1.11
-      - name: Package cli app jar with Gradle
-        run: ./gradlew shadowJar
       - name: Cache Gradle packages
         uses: actions/cache@v2
         with:
           path: ~/.gradle/caches
           key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
           restore-keys: ${{ runner.os }}-gradle
+      - name: Package cli app jar with Gradle
+        uses: eskatos/gradle-command-action@v1
+        env:
+          versionTag: ${{ steps.prep.outputs.versionTag }}
+        with:
+          arguments: shadowJar
       - name: Package web app war with Gradle
-        run: ./gradlew bootWar
+        uses: eskatos/gradle-command-action@v1
+        env:
+          versionTag: ${{ steps.prep.outputs.versionTag }}
+        with:
+          arguments: bootWar
       - name: Persist cli app jar
         uses: actions/upload-artifact@v2
         with:
-          name: v1.3.0-SNAPSHOT_cli
-          path: application/cli-app/build/libs/gtfs-validator-v1.3.0-SNAPSHOT_cli.jar
+          name: Application - cli executable .jar -- ${{ steps.prep.outputs.versionTag }}
+          path: application/cli-app/build/libs/gtfs-validator-${{ steps.prep.outputs.versionTag }}_cli.jar
       - name: Persist web app war
         uses: actions/upload-artifact@v2
         with:
-          name: v1.3.0-SNAPSHOT_web
-          path: application/web-app/spring-server/build/libs/gtfs-validator-v1.3.0-SNAPSHOT_web.war
+          name: Application - web executable .war (Spring&React) -- ${{ steps.prep.outputs.versionTag }}
+          path: application/web-app/spring-server/build/libs/gtfs-validator-${{ steps.prep.outputs.versionTag }}_web.war
 
   doc:
-    needs: test
+    needs: [test]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v1
+      - name: Prepare version name
+        id: prep
+        run: |
+          VERSION_TAG=edge
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            VERSION_TAG=${GITHUB_REF#refs/tags/}
+          elif [[ $GITHUB_REF == refs/heads/* ]]; then
+            VERSION_TAG=-$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g')
+          elif [[ $GITHUB_REF == refs/pull/* ]]; then
+            VERSION_TAG=-pr-${{ github.event.number }}
+          fi
+          if [ ${VERSION_TAG} != ${GITHUB_REF#refs/tags/} ]; then
+            VERSION_TAG=v${VERSION_TAG}-sha-${GITHUB_SHA::8}-SNAPSHOT
+          fi
+          echo ::set-output name=versionTag::${VERSION_TAG}
       - name: Set up JDK 1.11
         uses: actions/setup-java@v1
         with:
@@ -88,14 +166,18 @@ jobs:
           recurse: true
           output_dir: build/docs/jsdoc/
       - name: Build Javadoc
-        run: ./gradlew aggregateJavadocs
+        uses: eskatos/gradle-command-action@v1
+        env:
+          versionTag: ${{ steps.prep.outputs.versionTag }}
+        with:
+          arguments: aggregateJavadocs
       - name: Persist javadoc
         uses: actions/upload-artifact@v2
         with:
-          name: v1.3.0-SNAPSHOT_javadoc
+          name: Documentation - javadoc -- ${{ steps.prep.outputs.versionTag }}
           path: build/docs/javadoc/
       - name: Persist jsdoc
         uses: actions/upload-artifact@v2
         with:
-          name: v1.3.0-SNAPSHOT_jsdoc
+          name: Documentation - jsdoc -- ${{ steps.prep.outputs.versionTag }}
           path: build/docs/jsdoc/
diff --git a/Dockerfile b/Dockerfile
@@ -1,8 +1,7 @@
 FROM openjdk:11
-COPY application/cli-app/build/libs/*.jar /usr/gtfs-validator/cli-app/gtfs-validator-v1.3.0-SNAPSHOT_cli.jar
+COPY application/cli-app/build/libs/*.jar /usr/gtfs-validator/cli-app/
 COPY application/cli-app/scripts/end_to_end.sh /usr/gtfs-validator/cli-app/end_to_end.sh
 RUN chmod +x /usr/gtfs-validator/cli-app/end_to_end.sh
-COPY application/web-app/spring-server/build/libs/*.war /usr/gtfs-validator/web-app/gtfs-validator-v1.3.0-SNAPSHOT_web.war
-WORKDIR /usr/gtfs-validator/web-app
+COPY application/web-app/spring-server/build/libs/*.war /usr/gtfs-validator/web-app/ 
 EXPOSE 8090
-ENTRYPOINT ["java", "-jar", "gtfs-validator-v1.3.0-SNAPSHOT_web.war"]
+CMD ["/bin/sh", "-c", "java -jar /usr/gtfs-validator/web-app/*.war"]