bug 1597598 - use taskgraph

docker.d/generate_version_json.sh

@@ -1,7 +1,7 @@
 #/bin/sh
 set -e
 
-test $GIT_HEAD_REV
+test $HEAD_REV
 test $TASK_ID
 test $TASKCLUSTER_ROOT_URL
 test $REPO_URL

docker.d/push_image.sh

@@ -5,7 +5,8 @@ test $DOCKERHUB_EMAIL
 test $DOCKERHUB_USER
 test $DOCKER_REPO
 test $DOCKER_TAG
-test $GIT_HEAD_REV
+test $HEAD_REV
+test $HOME
 test $PUSH_DOCKER_IMAGE
 test $SECRET_URL
 test $PROJECT_NAME
@@ -18,12 +19,12 @@ docker tag $DOCKER_REPO:$DOCKER_TAG $DOCKER_REPO:$DOCKER_ARCHIVE_TAG
 
 echo "=== Generating dockercfg ==="
 # docker login stopped working in Taskcluster for some reason
-wget -qO- $SECRET_URL | jq '.secret.docker.dockercfg' > /root/.dockercfg
-chmod 600 /root/.dockercfg
+wget -qO- $SECRET_URL | jq '.secret.docker.dockercfg' > $HOME/.dockercfg
+chmod 600 $HOME/.dockercfg
 
 echo "=== Pushing to docker hub ==="
 docker push $DOCKER_REPO:$DOCKER_TAG
 docker push $DOCKER_REPO:$DOCKER_ARCHIVE_TAG
 
 echo "=== Clean up ==="
-rm -f /root/.dockercfg
+rm -f $HOME/.dockercfg

taskcluster/ci/config.yml

@@ -0,0 +1,46 @@
+---
+trust-domain: scriptworker
+
+task-priority: high
+
+taskgraph:
+    register: scriptworker_taskgraph:register
+    repositories:
+        scriptworker:
+            name: "Scriptworker-scripts"
+            project-regex: scriptworker-scripts$
+            default-repository: https://github.com/mozilla-releng/scriptworker-scripts
+            default-ref: master
+            type: git
+
+workers:
+    aliases:
+        b-linux:
+            provisioner: scriptworker-{level}
+            implementation: docker-worker
+            os: linux
+            worker-type: 'b-linux'
+        images:
+            provisioner: scriptworker-{level}
+            implementation: docker-worker
+            os: linux
+            worker-type: 'images'
+
+docker:
+    email: "[email protected]"
+    user: mozillarelengservices
+
+notify:
+    email:
+        - [email protected]
+    prefix: "[scriptworker-scripts]"
+
+release-promotion:
+    flavors:
+        promote:
+            target-tasks-method: promote_scriptworker
+
+private-artifact-prefix: releng/scriptworker
+
+scriptworker:
+    scope-prefix: project:scriptworker:releng

taskcluster/ci/docker-image/kind.yml

@@ -0,0 +1,41 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+
+loader: taskgraph.loader.transform:loader
+
+transforms:
+    - taskgraph.transforms.docker_image:transforms
+    - taskgraph.transforms.cached_tasks:transforms
+    - taskgraph.transforms.task:transforms
+
+jobs:
+    k8s-image:
+        definition: k8s-image
+    python38:
+        definition: python
+        args:
+            PYTHON_VERSION: "3.8"
+    python37:
+        definition: python
+        args:
+            PYTHON_VERSION: "3.7"
+    pushapkscript-py38:
+        definition: pushapkscript
+        parent: python38
+    pushapkscript-py37:
+        definition: pushapkscript
+        parent: python37
+    pushsnapscript-py38:
+        definition: pushsnapscript
+        parent: python38
+    pushsnapscript-py37:
+        definition: pushsnapscript
+        parent: python37
+    pushflatpakscript-py38:
+        definition: pushflatpakscript
+        parent: python38
+    pushflatpakscript-py37:
+        definition: pushflatpakscript
+        parent: python37

taskcluster/ci/k8s-image/kind.yml

@@ -0,0 +1,55 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+loader: taskgraph.loader.transform:loader
+
+kind-dependencies:
+    - tox
+
+transforms:
+    - scriptworker_taskgraph.transforms.python_version:transforms
+    - scriptworker_taskgraph.transforms.k8s_image:transforms
+    - scriptworker_taskgraph.transforms.cached:transforms
+    - taskgraph.transforms.cached_tasks:transforms
+    - taskgraph.transforms.task:transforms
+
+job-defaults:
+    description: "{name} k8s image builder py{python_version}"
+    label: "{name}-k8s-image-builder-python{python_version}"
+    run-on-tasks-for: ["action", "github-pull-request", "github-push"]
+    attributes:
+        code-review: true
+    worker-type: images
+    worker:
+        docker-image: {in-tree: 'k8s-image'}
+        max-run-time: 3600
+        docker-in-docker: true
+        taskcluster-proxy: true
+        chain-of-trust: true
+        artifacts:
+            - type: directory
+              name: public/
+              path: /builds/worker/artifacts/
+        command:
+            - sh
+            - -lxce
+            - >-
+              sh /usr/local/bin/build_and_push.sh
+    python-versions:
+        - 38
+    deploy-secret-url: http://taskcluster/secrets/v1/secret/project/releng/scriptworker-scripts/deploy
+    docker-repo: mozilla/releng-{name}
+    # TODO copy image to artifacts
+
+jobs:
+    addonscript: {}
+    balrogscript: {}
+    beetmoverscript: {}
+    bouncerscript: {}
+    pushapkscript: {}
+    pushflatpakscript: {}
+    pushsnapscript: {}
+    shipitscript: {}
+    signingscript: {}
+    treescript: {}

taskcluster/ci/pr/kind.yml

@@ -0,0 +1,18 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+loader: taskgraph.loader.transform:loader
+
+kind-dependencies:
+    - k8s-image
+
+transforms:
+    - taskgraph.transforms.code_review:transforms
+    - taskgraph.transforms.task:transforms
+
+jobs:
+    complete:
+        description: PR Summary Task
+        run-on-tasks-for: [github-pull-request]
+        worker-type: succeed

taskcluster/ci/tox/kind.yml

@@ -0,0 +1,102 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+loader: taskgraph.loader.transform:loader
+
+kind-dependencies:
+    - docker-image
+
+transforms:
+    - scriptworker_taskgraph.transforms.python_version:transforms
+    - scriptworker_taskgraph.transforms.cached:transforms
+    - taskgraph.transforms.cached_tasks:transforms
+    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.task:transforms
+
+job-defaults:
+    description: "{name} tox-py{python_version}"
+    label: "tox-{name}-py{python_version}"
+    run-on-tasks-for: ["action", "github-pull-request", "github-push"]
+    attributes:
+        code-review: true
+    worker-type: b-linux
+    worker:
+        docker-image: {in-tree: 'python{python_version}'}
+        max-run-time: 1800
+        artifacts:
+            - type: directory
+              name: public/
+              path: /builds/worker/artifacts/
+    python-versions:
+        - 38
+        - 37
+    run:
+        using: run-task
+        cache-dotcache: false
+        checkout:
+            scriptworker: {}
+        # sparse-profile: basestring/none
+        # workdir:
+        cwd: '{checkout}'
+        command:
+            - sh
+            - -lxce
+            - >-
+              tox -e {name}-py{python_version}
+
+jobs:
+    addonscript:
+        digest-directories:
+            - addonscript
+    balrogscript:
+        digest-directories:
+            - balrogscript
+            - scriptworker_client
+    beetmoverscript:
+        digest-directories:
+            - beetmoverscript
+    bouncerscript:
+        digest-directories:
+            - bouncerscript
+    configloader:
+        digest-directories:
+            - configloader
+    iscript:
+        digest-directories:
+            - iscript
+            - scriptworker_client
+            - vendored/mozbuild
+    notarization_poller:
+        digest-directories:
+            - notarization_poller
+            - scriptworker_client
+    pushapkscript:
+        worker:
+            docker-image: {in-tree: 'pushapkscript-py{python_version}'}
+        digest-directories:
+            - pushapkscript
+    pushflatpakscript:
+        worker:
+            docker-image: {in-tree: 'pushflatpakscript-py{python_version}'}
+        digest-directories:
+            - pushflatpakscript
+    pushsnapscript:
+        worker:
+            docker-image: {in-tree: 'pushsnapscript-py{python_version}'}
+        digest-directories:
+            - pushsnapscript
+    scriptworker_client:
+        digest-directories:
+            - scriptworker_client
+    shipitscript:
+        digest-directories:
+            - shipitscript
+    signingscript:
+        digest-directories:
+            - signingscript
+            - vendored/mozbuild
+    treescript:
+        digest-directories:
+            - treescript
+            - scriptworker_client

taskcluster/docker/REGISTRY

@@ -0,0 +1 @@
+mozilla-releng

taskcluster/docker/k8s-image/Dockerfile

@@ -0,0 +1,27 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# We need to run a really old docker version because taskcluster is using a
+# really old version in their setup
+FROM          docker:1.6.2
+
+RUN apk add --update xz
+RUN mkdir -p /usr/local/bin
+COPY build_and_push.sh /usr/local/bin/build_and_push.sh
+
+# Add worker user
+RUN mkdir /builds && \
+    adduser -D -h /builds/worker -g worker -u 1000 worker && \
+    mkdir /builds/worker/artifacts && \
+    chown worker:worker /builds/worker/artifacts
+
+ENV SHELL=/bin/sh \
+    HOME=/builds/worker \
+    PATH=/builds/worker/.local/bin:$PATH
+
+VOLUME /builds/worker/checkouts
+VOLUME /builds/worker/.cache
+
+# Set a default command useful for debugging
+CMD ["/bin/sh"]

taskcluster/docker/k8s-image/build_and_push.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+# TODO: move docker.d/ files into the image?
+set -e
+test REPO_URL
+test HEAD_REV
+test PROJECT_NAME
+test PUSH_DOCKER_IMAGE
+
+mkdir -p /builds/worker/checkouts
+cd /builds/worker/checkouts
+wget ${REPO_URL}/archive/${HEAD_REV}.tar.gz
+tar zxf ${HEAD_REV}.tar.gz
+mv *-${HEAD_REV} src
+cd src
+cp ${PROJECT_NAME}/docker.d/* docker.d/
+cp ${PROJECT_NAME}/Dockerfile .
+sh ./docker.d/generate_version_json.sh
+sh ./docker.d/build_image.sh
+if [ "${PUSH_DOCKER_IMAGE}" == "1" ]; then
+  ./docker.d/push_image.sh
+fi

taskcluster/docker/pushapkscript/Dockerfile

@@ -0,0 +1,7 @@
+# %ARG DOCKER_IMAGE_PARENT
+FROM $DOCKER_IMAGE_PARENT
+
+VOLUME /builds/worker/checkouts
+VOLUME /builds/worker/.cache
+
+RUN apt-get update && apt-get install -y default-jdk

taskcluster/docker/pushflatpakscript/Dockerfile

@@ -0,0 +1,8 @@
+# %ARG DOCKER_IMAGE_PARENT
+FROM $DOCKER_IMAGE_PARENT
+
+VOLUME /builds/worker/checkouts
+VOLUME /builds/worker/.cache
+
+RUN apt-get update \
+    && apt-get install -y gir1.2-ostree-1.0 libgirepository1.0-dev

taskcluster/docker/pushsnapscript/Dockerfile

@@ -0,0 +1,9 @@
+# %ARG DOCKER_IMAGE_PARENT
+FROM $DOCKER_IMAGE_PARENT
+
+VOLUME /builds/worker/checkouts
+VOLUME /builds/worker/.cache
+
+RUN apt-get update \
+    && apt-get install -y libsodium-dev
+RUN truncate -s 0 /etc/os-release