run workflows on snap fork

.dockerignore

@@ -0,0 +1,10 @@
+**/__pycache__
+**/*.pyc
+**/*.pyo
+**/*.pyd
+**/.git
+**/.mypy_cache
+**/*.pkl
+
+# https://github.com/google-github-actions/auth/issues/497
+gha-creds-*.json

.github/actions/create_gpu_runner/action.yml

@@ -0,0 +1,84 @@
+name: Create GPU Runner
+description: Create a GPU runner for the GLT CI
+
+inputs:
+  project:
+    description: The project to create the runner in
+    required: true
+  zone:
+    description: The zone to create the runner in
+    required: true
+  service_account:
+    description: The service account to use for the runner
+    required: true
+  machine_name:
+    description: The name of the machine to create
+    required: true
+  machine_type:
+    description: The type of machine to create
+    default: n1-standard-4
+  accelerator_type:
+    description: The type of accelerator to create
+    default: nvidia-tesla-p4
+  accelerator_count:
+    description: The number of accelerators to create
+    default: 1
+  disk_size:
+    description: The size of the disk to create
+    default: 100
+  workload_identity_provider:
+    description: The workload identity provider to use for the runner
+    required: true
+  gcp_service_account_email:
+    description: The GCP service account email to use for the runner
+    required: true
+  gpu_runner_token:
+    description: The token to use for the runner
+    required: true
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Setup GCloud
+      uses: ./.github/actions/setup_gcloud
+      with:
+        gcp_project_id: ${{ inputs.project }}
+        workload_identity_provider: ${{ inputs.workload_identity_provider }}
+        gcp_service_account_email: ${{ inputs.gcp_service_account_email }}
+
+    - name: Start Runner
+      shell: bash
+      run: |
+        python <(curl -s https://raw.githubusercontent.com/Snapchat/GiGL/refs/heads/main/scripts/create_dev_instance.py) \
+        --project=${{ inputs.project }} \
+        --zone=${{ inputs.zone }} \
+        --service-account=${{ inputs.service_account }} \
+        --machine_name=${{ inputs.machine_name }} \
+        --machine_type=${{ inputs.machine_type }} \
+        --accelerator_type=${{ inputs.accelerator_type }} \
+        --accelerator_count=${{ inputs.accelerator_count }} \
+        --disk_size=${{ inputs.disk_size }}
+
+    - name: Wait for Runner to be ready
+      shell: bash
+      run: |
+        while ! gcloud compute instances describe ${MACHINE_NAME} --format='get(status)' | grep -q "RUNNING"; do
+
+    - name: Setup Runner
+      shell: bash
+      run: |
+        # Create a folder
+        mkdir actions-runner && cd actions-runner
+        # Download the latest runner package
+        curl -o actions-runner-linux-x64-2.327.1.tar.gz -L https://github.com/actions/runner/releases/download/v2.327.1/actions-runner-linux-x64-2.327.1.tar.gz
+        # Optional: Validate the hash
+        echo "d68ac1f500b747d1271d9e52661c408d56cffd226974f68b7dc813e30b9e0575  actions-runner-linux-x64-2.327.1.tar.gz" | shasum -a 256 -c
+        # Extract the installer
+        tar xzf ./actions-runner-linux-x64-2.327.1.tar.gz
+
+        ./config.sh --url ${{ github.repository }} --token ${{ inputs.gpu_runner_token }}
+
+    - name: Run the runner
+      shell: bash
+      run: |
+        ./run.sh

.github/actions/setup_gcloud/action.yml

@@ -0,0 +1,44 @@
+name: Setup GCloud
+description: Setup GCloud
+
+inputs:
+  gcp_project_id:
+    description: The GCP project ID
+    required: true
+  workload_identity_provider:
+    description: The workload identity provider
+    required: true
+  gcp_service_account_email:
+    description: The GCP service account email
+    required: true
+
+runs:
+  using: 'composite'
+  steps:
+      # Step 1: Set up Python environment (Python 3.9.13).
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          # Available versions: https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json
+          # Ensure to use a version that has support for arm64-darwin so we can build for Apple Silicon (macOS 14).
+          python-version: '3.9.13'
+
+      # Step 2: Set up Gcloud AUTH using Workload Identity Federation
+      # See following for context: https://cloud.google.com/blog/products/identity-security/enabling-keyless-authentication-from-github-actions
+      - name: Google Cloud Auth
+        # NOTE: If upgrading, please ensure that the generated
+        # credentials file is gitignored/ docker ignored: `gha-creds-*.json`
+        # More info: https://github.com/google-github-actions/auth/issues/123
+        uses: 'google-github-actions/auth@v2' # WARNING: See note above.
+        with:
+          project_id: ${{ inputs.gcp_project_id }}
+          workload_identity_provider: ${{ inputs.workload_identity_provider }}
+          service_account: ${{ inputs.gcp_service_account_email }}
+          create_credentials_file: true
+          export_environment_variables: true
+
+      # Step 3: Install gcloud tooling
+      - name: Set Up Cloud SDK
+        uses: google-github-actions/setup-gcloud@v2
+        with:
+          project_id: ${{ inputs.gcp_project_id }}

.github/actions/shutdown_gcp_instance/action.yml

@@ -0,0 +1,34 @@
+name: Shutdown GCP Instance
+description: Shutdown a GCP instance
+
+inputs:
+  machine_name:
+    description: The name of the machine to shutdown
+    required: true
+  zone:
+    description: The zone of the machine to shutdown
+    required: true
+  project:
+    description: The project to shutdown the machine in
+    required: true
+  service_account:
+    description: The service account to use for the shutdown
+    required: true
+  workload_identity_provider:
+    description: The workload identity provider to use for the shutdown
+    required: true
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Setup GCloud
+      uses: ./.github/actions/setup_gcloud
+      with:
+        gcp_project_id: ${{ inputs.project }}
+        workload_identity_provider: ${{ inputs.workload_identity_provider }}
+        gcp_service_account_email: ${{ inputs.service_account }}
+
+    - name: Shutdown Instance
+      shell: bash
+      run: |
+        gcloud compute instances delete ${inputs.machine_name} --zone=${inputs.zone}

.github/workflows/glt-ci.yml

@@ -16,15 +16,44 @@ on:
       - 'examples/**'
       - '**.md'
 
+permissions:
+  # Needed for gcloud auth: https://github.com/google-github-actions/auth
+  id-token: 'write'
+  contents: 'read'
+
 env:
   IMAGE: graphlearn-torch-dev
   JOBNAME: glt-ci-${{ github.run_id }}
   DESTDIR: /mnt/graphlearn_for_pytorch
+  MACHINE_NAME: glt-gpu-unittests-${{ github.run_id }}
+  gcp_zone: us-central1-b
+  PROJECT: external-snap-ci-github-gigl
 
 jobs:
+  create-gpu-runner:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v3
+      - name: Create GPU Runner
+        uses: ./.github/actions/create_gpu_runner
+        with:
+          project: ${{ env.PROJECT }}
+          zone: ${{ env.gcp_zone }}
+          service_account: ${{ secrets.GCP_SERVICE_ACCOUNT_EMAIL }}
+          machine_name: ${{ env.MACHINE_NAME }}
+          machine_type: n1-standard-4
+          accelerator_type: nvidia-tesla-p4
+          accelerator_count: 1
+          gpu_runner_token: ${{ secrets.GPU_RUNNER_TOKEN }}
+          workload_identity_provider: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }}
+          gcp_service_account_email: ${{ secrets.GCP_SERVICE_ACCOUNT_EMAIL }}
+
+
   run-glt-unittests:
+    needs: create-gpu-runner
     runs-on: self-hosted
-    if: ${{ github.repository == 'alibaba/graphlearn-for-pytorch' }}
+    if: ${{ github.repository == 'snapchat/graphlearn-for-pytorch' }}
     steps:
       - name: Checkout Code
         uses: actions/checkout@v3
@@ -61,3 +90,21 @@ jobs:
         working-directory: ${{github.workspace}}
         run: |
           bash scripts/remove_container.sh ${JOBNAME}
+
+  shutdown_gpu_runner:
+    needs: run-glt-unittests
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v3
+
+      - name: Shutdown GPU Runner
+        uses: ./.github/actions/shutdown_gcp_instance
+        with:
+          machine_name: ${{ env.MACHINE_NAME }}
+          zone: ${{ env.gcp_zone }}
+          project: ${{ env.PROJECT }}
+          service_account: ${{ secrets.GCP_SERVICE_ACCOUNT_EMAIL }}
+          workload_identity_provider: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }}
+          gcp_service_account_email: ${{ secrets.GCP_SERVICE_ACCOUNT_EMAIL }}

.github/workflows/glt-v6d-ci.yml

@@ -23,8 +23,8 @@ env:
 
 jobs:
   run-glt-v6d-unittests:
-    runs-on: self-hosted
-    if: ${{ github.repository == 'alibaba/graphlearn-for-pytorch' }}
+    runs-on: ubuntu-latest
+    if: ${{ github.repository == 'snapchat/graphlearn-for-pytorch' }}
     steps:
       - name: Checkout Code
         uses: actions/checkout@v3
@@ -58,4 +58,4 @@ jobs:
         if: always()
         working-directory: ${{github.workspace}}
         run: |
-          bash scripts/remove_container.sh ${JOBNAME}
+          bash scripts/remove_container.sh ${JOBNAME}

.github/workflows/manylinux-cd.yml

@@ -12,8 +12,8 @@ env:
 
 jobs:
   build:
-    runs-on: self-hosted
-    if: ${{ github.repository == 'alibaba/graphlearn-for-pytorch' }}
+    runs-on: ubuntu-latest
+    if: ${{ github.repository == 'snapchat/graphlearn-for-pytorch' }}
     steps:
       - name: Checkout Code
         uses: actions/checkout@v3

.gitignore

@@ -15,4 +15,8 @@ detect_cuda_*
 .idea/
 cmake-build-release/
 cmake-build-debug/
-/graphlearn_torch/python/*.so
+/graphlearn_torch/python/*.so
+
+gha-creds-*.json
+
+.DS_Store