Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion actions/build_deploy_python_executable/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ inputs:
default: '3.8'
deployment:
required: false
description: "The deployment to push to, defaults to 'prod'. Ignored for pull requests where the branch deployment is used."
description: "The deployment to push to. For pull requests, this deployment will be used as the base deployment."
default: "prod"
deploy:
description: 'Whether to upload the code files and update the code location'
Expand Down
6 changes: 5 additions & 1 deletion actions/serverless_branch_deploy/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,14 @@ inputs:
required: false
description: "Whether to start the action by checking out the repository. Set to false if your workflow modifies the file structure before deploying."
default: 'true'
base_deployment_name:
required: false
description: "The name of the base deployment to use for the branch deployment."
outputs:
deployment:
description: "Name of the branch deployment for this PR"
value: ${{ steps.deploy.outputs.deployment }}

runs:
using: "composite"
steps:
Expand Down Expand Up @@ -112,6 +115,7 @@ runs:
location: ${{ inputs.location }}
image_tag: branch-${{ fromJson(inputs.location).name }}-${{ env.SHORT_SHA }}-${{ github.run_id }}-${{ github.run_attempt }}
registry: ${{ env.REGISTRY_URL }}
base_deployment_name: ${{ inputs.base_deployment_name }}
env:
DAGSTER_CLOUD_API_TOKEN: ${{ inputs.dagster_cloud_api_token }}

Expand Down
3 changes: 3 additions & 0 deletions actions/utils/deploy/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@ inputs:
deployment:
required: false
description: "The deployment to deploy to. If unset, automatically creates or updates the branch deployment associated with the branch."
base_deployment_name:
required: false
description: "The name of the base deployment to use for the branch deployment."
pr:
required: false
description: "The PR identifier for this PR, if any, used for branch deployments."
Expand Down
3 changes: 3 additions & 0 deletions actions/utils/get_branch_deployment/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@ inputs:
source_directory:
required: false
description: "Path to the source directory"
base_deployment_name:
required: false
description: "The name of the base deployment to use for the branch deployment."
outputs:
deployment:
description: "The Cloud deployment associated with this branch."
Expand Down
55 changes: 23 additions & 32 deletions src/deploy.sh
Original file line number Diff line number Diff line change
Expand Up @@ -66,39 +66,30 @@ if [ -z $INPUT_DEPLOYMENT ]; then
export EMAIL=$(git log -1 --format='%ae')
export NAME=$(git log -1 --format='%an')

# Create or update branch deployment
if [ -z $AVATAR_URL ]; then
export DEPLOYMENT_NAME=$(dagster-cloud branch-deployment create-or-update \
--url "${DAGSTER_CLOUD_URL}" \
--api-token "$DAGSTER_CLOUD_API_TOKEN" \
--git-repo-name "$GIT_REPO" \
--branch-name "$BRANCH_NAME" \
--branch-url "$BRANCH_URL" \
--pull-request-url "$PR_URL" \
--pull-request-id "$PR_ID" \
--pull-request-status "$PR_STATUS" \
--commit-hash "$COMMIT_HASH" \
--timestamp "$TIMESTAMP" \
--commit-message "$MESSAGE" \
--author-name "$NAME" \
--author-email "$EMAIL")
else
export DEPLOYMENT_NAME=$(dagster-cloud branch-deployment create-or-update \
--url "${DAGSTER_CLOUD_URL}" \
--api-token "$DAGSTER_CLOUD_API_TOKEN" \
--git-repo-name "$GIT_REPO" \
--branch-name "$BRANCH_NAME" \
--branch-url "$BRANCH_URL" \
--pull-request-url "$PR_URL" \
--pull-request-id "$PR_ID" \
--pull-request-status "$PR_STATUS" \
--commit-hash "$COMMIT_HASH" \
--timestamp "$TIMESTAMP" \
--commit-message "$MESSAGE" \
--author-name "$NAME" \
--author-email "$EMAIL" \
--author-avatar-url "$AVATAR_URL")
EXTRA_PARAMS=()
if [[ $INPUT_BASE_DEPLOYMENT_NAME ]]; then
EXTRA_PARAMS+=(--base-deployment-name $INPUT_BASE_DEPLOYMENT_NAME)
fi
if [[ $AVATAR_URL ]]; then
EXTRA_PARAMS+=(--author-avatar-url $AVATAR_URL)
fi

export DEPLOYMENT_NAME=$(dagster-cloud branch-deployment create-or-update \
--url "${DAGSTER_CLOUD_URL}" \
--api-token "$DAGSTER_CLOUD_API_TOKEN" \
--git-repo-name "$GIT_REPO" \
--branch-name "$BRANCH_NAME" \
--branch-url "$BRANCH_URL" \
--pull-request-url "$PR_URL" \
--pull-request-id "$PR_ID" \
--pull-request-status "$PR_STATUS" \
--commit-hash "$COMMIT_HASH" \
--timestamp "$TIMESTAMP" \
--commit-message "$MESSAGE" \
--author-name "$NAME" \
--author-email "$EMAIL" \
"${EXTRA_PARAMS[@]:-}")

else
export DEPLOYMENT_NAME=$INPUT_DEPLOYMENT
fi
Expand Down
34 changes: 22 additions & 12 deletions src/deploy_pex.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,9 @@

import yaml

DAGSTER_CLOUD_PEX_PATH = Path(__file__).parent.parent / "generated/gha/dagster-cloud.pex"
DAGSTER_CLOUD_PEX_PATH = (
Path(__file__).parent.parent / "generated/gha/dagster-cloud.pex"
)
UPDATE_COMMENT_SCRIPT_PATH = Path(__file__).parent / "create_or_update_comment.py"


Expand All @@ -27,7 +29,9 @@ def main():
print("Running in a pull request - going to do a branch deployment", flush=True)
dagster_cloud_yaml = args[0]
project_dir = os.path.dirname(dagster_cloud_yaml)
deployment_name = get_branch_deployment_name(project_dir)
deployment_name = get_branch_deployment_name(
project_dir, base_deployment_name=os.getenv("INPUT_DEPLOYMENT", "prod")
)
else:
# INPUT_DEPLOYMENT is to the `deployment:` input value in action.yml
deployment_name = os.getenv("INPUT_DEPLOYMENT", "prod")
Expand All @@ -40,9 +44,7 @@ def main():
else:
returncode, output = deploy_pex(args, deployment_name, build_method="docker")
if returncode:
print(
"::error Title=Deploy failed::Failed to deploy Python Executable. "
)
print("::error Title=Deploy failed::Failed to deploy Python Executable. ")
# TODO: fallback to docker deploy here
sys.exit(1)

Expand All @@ -65,7 +67,9 @@ def get_locations(dagster_cloud_file) -> List[str]:
workspace_contents = f.read()
workspace_contents_yaml = yaml.safe_load(workspace_contents)

return [location["location_name"] for location in workspace_contents_yaml["locations"]]
return [
location["location_name"] for location in workspace_contents_yaml["locations"]
]


def run(args):
Expand All @@ -83,7 +87,7 @@ def run(args):
return returncode, output


def get_branch_deployment_name(project_dir):
def get_branch_deployment_name(project_dir: str, base_deployment_name: Optional[str]):
returncode, output = run(
[
str(DAGSTER_CLOUD_PEX_PATH),
Expand All @@ -92,14 +96,22 @@ def get_branch_deployment_name(project_dir):
"ci",
"branch-deployment",
project_dir,
*(
[
"--base-deployment-name",
base_deployment_name,
]
if base_deployment_name
else []
),
]
)
if returncode:
print("Could not determine branch deployment from output:", output, flush=True)
sys.exit(1)
for line in output:
# sometimes the cmd prints warnings in addition to the branch deployment name
if re.match('[0-9a-f]+', line):
if re.match("[0-9a-f]+", line):
name = line.strip()
break
else:
Expand All @@ -114,9 +126,7 @@ def deploy_pex(args, branch_deployment_name: Optional[str], build_method: str):
args.insert(0, os.path.dirname(dagster_cloud_yaml))
args = args + [f"--build-method={build_method}"]
commit_hash = os.getenv("GITHUB_SHA")
git_url = (
f"{os.getenv('GITHUB_SERVER_URL')}/{os.getenv('GITHUB_REPOSITORY')}/tree/{commit_hash}"
)
git_url = f"{os.getenv('GITHUB_SERVER_URL')}/{os.getenv('GITHUB_REPOSITORY')}/tree/{commit_hash}"
deployment_name = branch_deployment_name if branch_deployment_name else "prod"
deployment_flag = f"--url={os.getenv('DAGSTER_CLOUD_URL')}/{deployment_name}"
locations = get_locations(dagster_cloud_yaml)
Expand Down Expand Up @@ -171,7 +181,7 @@ def update_pr_comment(deployment_name: str, location_name: str, action: str):
return

env = dict(os.environ)
github_run_url = f'{os.environ["GITHUB_SERVER_URL"]}/{os.environ["GITHUB_REPOSITORY"]}/actions/runs/{os.environ["GITHUB_RUN_ID"]}'
github_run_url = f"{os.environ['GITHUB_SERVER_URL']}/{os.environ['GITHUB_REPOSITORY']}/actions/runs/{os.environ['GITHUB_RUN_ID']}"
env.update(
{
"INPUT_PR": str(pr_id),
Expand Down
8 changes: 6 additions & 2 deletions src/get_branch_deployment.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,10 @@ fi

git config --global --add safe.directory $(realpath $INPUT_SOURCE_DIRECTORY)

BRANCH_DEPLOYMENT_NAME=$(dagster-cloud ci branch-deployment $INPUT_SOURCE_DIRECTORY)
EXTRA_PARAMS=()
if [ -n $INPUT_BASE_DEPLOYMENT_NAME ]; then
EXTRA_PARAMS+=(--base-deployment-name $INPUT_BASE_DEPLOYMENT_NAME)
fi
BRANCH_DEPLOYMENT_NAME=$(dagster-cloud ci branch-deployment $INPUT_SOURCE_DIRECTORY "${EXTRA_PARAMS[@]:-}")

echo "deployment=${BRANCH_DEPLOYMENT_NAME}" >> $GITHUB_OUTPUT
echo "deployment=${BRANCH_DEPLOYMENT_NAME}" >> $GITHUB_OUTPUT
Loading