[WIRE-575] CI/CD refactoring

.github/workflows/release.yaml

@@ -1,14 +1,170 @@
 name: release
 
-on: workflow_dispatch
-
+on:
+  workflow_dispatch:
+  pull_request:
+  push:
+    tags:
+      - 'v[0-9].*'
+
 env:
   GENERATOR_VERSION: 'v7.9.0'
+  INPUT_OPENAPI_FILE: 'openapi/openapi.yaml'
+  GO_VERSION: '1.23'
+  RELEASE_BRANCH: ''
+  LATEST_COMMIT: ''
 
 jobs:
   push:
     runs-on: ubuntu-latest
 
     steps:
+      - name: Env setup
+        run: | 
+            git config --global user.email "[email protected]"
+            git config --global user.name "github-actions[bot]"
+            echo "RELEASE_BRANCH="release-"$(date +"%Y%m%d%H%M%S")" >> ${GITHUB_ENV}
+
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
       - name: Checkout
         uses: actions/checkout@v4
+
+      - name: Get latest commit
+        run: |
+            echo "LATEST_COMMIT=$(git log -1 | head -n1)" >> ${GITHUB_ENV}
+
+      - name: Generate openapi yaml definition
+        run: |
+            cd openapi/compose && go run compose.go && cd .. && cd ..
+
+      - name: Checkout generator templates
+        uses: actions/checkout@v4
+        with:
+          repository: OpenAPITools/openapi-generator
+          path: official-generator-templates
+          ref: ${{ env.GENERATOR_VERSION }}
+          token: ${{ secrets.GH_PUSH_TOKEN }}
+          sparse-checkout: |
+            .github
+            modules/openapi-generator/src/main/resources
+
+      - name: Move required templates
+        run: |
+            mkdir -p generated
+            mv official-generator-templates/modules/openapi-generator/src/main/resources/go official-generator-templates/
+            mv official-generator-templates/modules/openapi-generator/src/main/resources/python official-generator-templates/
+            mv official-generator-templates/modules/openapi-generator/src/main/resources/Javascript official-generator-templates/
+            mv official-generator-templates/modules/openapi-generator/src/main/resources/htmlDocs2 official-generator-templates/
+
+      - name: Apply patches to templates
+        run: |
+            cd official-generator-templates
+            for f in ../template-patches/*.patch; do git apply ../template-patches/${f}; done;
+            cd ..
+
+      - name: Checkout GO client arduino repo
+        uses: actions/checkout@v4
+        with:
+          repository: arduino/iot-client-go
+          token: ${{ secrets.GH_PUSH_TOKEN }}
+          path: generated/go
+      - run: |
+          cd generated/go && git checkout -b ${{ env.RELEASE_BRANCH }}                     
+
+      - name: Checkout PYTHON client arduino repo
+        uses: actions/checkout@v4
+        with:
+          repository: arduino/iot-client-py
+          token: ${{ secrets.GH_PUSH_TOKEN }}
+          path: generated/python 
+      - run: |
+          cd generated/python && git checkout -b ${{ env.RELEASE_BRANCH }}
+
+      - name: Checkout JAVASCRIPT client arduino repo
+        uses: actions/checkout@v4
+        with:
+          repository: arduino/iot-client-js
+          token: ${{ secrets.GH_PUSH_TOKEN }}
+          path: generated/javascript 
+      - run: |
+          cd generated/javascript && git checkout -b ${{ env.RELEASE_BRANCH }}
+
+      - name: Checkout DOCS client arduino repo
+        uses: actions/checkout@v4
+        with:
+          repository: arduino/iot-client-docs
+          token: ${{ secrets.GH_PUSH_TOKEN }}
+          path: generated/html2 
+      - run: |
+          cd generated/html2 && git checkout -b ${{ env.RELEASE_BRANCH }}
+
+      - name: Generate go client
+        uses: openapi-generators/openapitools-generator-action@v1
+        with:
+            generator: go
+            generator-tag: ${{ env.GENERATOR_VERSION }}
+            config-file: config/languages/go_v2.json
+            openapi-file: ${{ env.INPUT_OPENAPI_FILE }}
+            template-dir: official-generator-templates/go/
+            command-args: -o generated/go
+
+      - name: Generate python client
+        uses: openapi-generators/openapitools-generator-action@v1
+        with:
+            generator: python
+            generator-tag: ${{ env.GENERATOR_VERSION }}
+            config-file: config/languages/python_v2.json
+            openapi-file: ${{ env.INPUT_OPENAPI_FILE }}
+            template-dir: official-generator-templates/python/
+            command-args: -o generated/python
+
+      - name: Generate Javascript client
+        uses: openapi-generators/openapitools-generator-action@v1
+        with:
+            generator: javascript
+            generator-tag: ${{ env.GENERATOR_VERSION }}
+            config-file: config/languages/javascript_v2.json
+            openapi-file: ${{ env.INPUT_OPENAPI_FILE }}
+            template-dir: official-generator-templates/Javascript/
+            command-args: -o generated/javascript
+
+      - name: Generate Docs
+        uses: openapi-generators/openapitools-generator-action@v1
+        with:
+            generator: html2
+            generator-tag: ${{ env.GENERATOR_VERSION }}
+            config-file: config/languages/html2_v2.json
+            openapi-file: ${{ env.INPUT_OPENAPI_FILE }}
+            template-dir: official-generator-templates/htmlDocs2/
+            command-args: -o generated/html2
+
+      - name: Apply downstream templates
+        run: |
+          cp downstream-templates/go/README.md generated/go/README.md
+          cp downstream-templates/javascript/README.md generated/javascript/README.md
+          cp downstream-templates/python/README.md generated/python/README.md
+          cp downstream-templates/python/setup.py generated/python/setup.py
+          cp downstream-templates/python/utils.py generated/python/iot_api_client/utils.py
+
+      - name: Copy shared assets
+        run: |
+            cp -R assets/shared/* generated/go/
+            cp -R assets/shared/* generated/python
+            cp -R assets/shared/* generated/javascript
+            cp -R assets/shared/* generated/html2 
+
+      - name: Open branches on external client REPOs
+        run: |
+            cd generated
+            for f in $(ls -d */); do cd ${f} && git add . && git commit -m "Release ${{ env.RELEASE_BRANCH }}" && git push -u origin ${{ env.RELEASE_BRANCH }} && cd ..; done;
+
+      - name: Open PRs on client REPOs
+        run: |
+            cd generated
+            for f in $(ls -d */); do cd ${f} && gh pr create -B master -H ${{ env.RELEASE_BRANCH }} --title 'Regenerate clients - ${{ env.LATEST_COMMIT }}' --body 'Created by clients-iot-api clients release automation job' && cd ..; done;
+        env:
+          GH_TOKEN: ${{ secrets.GH_PUSH_TOKEN }}           

.gitignore

@@ -1,7 +1,8 @@
 !generated
 generated/*
 !generated/.gitkeep
-spec/*/full_spec.yaml
 !templates
 templates/*
-!templates/.gitkeep
+!templates/.gitkeep
+openapi/openapi.yaml
+official-generator-templates

README.md

@@ -4,13 +4,7 @@ This repo contains the informations and the tools needed to automatically
 generate API clients for the `iot-api` service in [any language][0] supported by
 OpenAPI generator.
 
-The OpenAPI generator is orchestrated by a Python tool called [apigentools][1]
-that let us keep the spec files and the configuration bits in one single repo
-(this one) without duplicating the boilerplate on each git repo hosting the
-actual clients.
-
-There is a [blog post](https://blog.arduino.cc/2020/03/05/how-to-deal-with-api-clients-the-lazy-way-from-code-generation-to-release-management/)
-describing the system architecture.
+The OpenAPI generator is orchestrated by git workflow using openapi-generator official images.
 
 ## [IMPORTANT] Client generation process
 
@@ -39,14 +33,6 @@ depending on the programming language).
 
 The operations are detailed in the following paragraphs.
 
-### Requirements
-
-To be able to run the workflow locally in a developmnent environment, you'll
-need the following:
-
-* Python 3.7+
-* Apigentools 1.0+ (`pip install apigentools`)
-
 ### Get an updated version of the API specification
 
 In this case the specs are generated by Goa using Swagger and they can be found
@@ -64,60 +50,14 @@ easily integrated in a CI, so major updates will be performed by either:
 Minor updates might be done manually since v3 uses Yaml and the resulting spec
 is human friendly.
 
-### Validate the Spec
-
-The generator can validate the content of the spec, this should likely go in a
-CI step:
-
-```sh
-apigentools validate
-```
-
 ### Generate the clients
 
-To generate clients and documentation using a template, and apply pathces before
-template filling, this is the command to run:
-
-```sh
-apigentools generate
-```
-
-The previous command is fine for local development but in case the generated code
-has to be pushed upstream to each client's repository, run the previous command as
-follows:
-
-```sh
- apigentools generate --clone-repo
-```
-
-### Push generated code to each client's git repository
-
-`apigentools` has a simple command that can be invoked for each client generated
-that will push the resulting code to different repos, using different branches
-so the code can be reviewed and merged through a regular PR:
-
-```sh
-apigentools push
-```
-
-A release process should take from here and shouldn't be part of this workflow.
-
-### Render upstream templates
-
-This step is not currently necessary as the `generate` step does it implicitly,
-but in case you want to only apply patches to the original (upstream) templates
-**before** the generation step, you can run it to check how the pathces will
-modify the generated code. The step consists of cloning the openapi-generator
-repo, applying one or more patches in the form of patch files to it and copy
-the relevant templates in the folder `templates`:
-
-```sh
-apigentools templates
-```
-
-There are other ways to provide upstream templates other than the currently configured
-`openapi-git` that might be useful to speed up development iterations, please refer to
-`apigentools` docs for more details.
+Update openapi definition in openapi/components folder.
+'Release' github workflow will take care of:
+* update and merge openapi specifications
+* update templates with provided patches
+* generate clients
+* create PRs with updated code in destination clients repositories
 
 ## Customization