[Internal] Migrate workflows that need write access to use hosted runners (databricks#850)

Fixes databricks#848.

Author: pietern

.github/workflows/external-message.yml

@@ -13,7 +13,10 @@ on:
 
 jobs:
   comment-on-pr:
-    runs-on: ubuntu-latest
+    runs-on:
+      group: databricks-deco-testing-runner-group
+      labels: ubuntu-latest-deco
+
     permissions:
       pull-requests: write
 
@@ -44,13 +47,13 @@ jobs:
           gh pr comment ${{ github.event.pull_request.number }} --body \
           "<!-- INTEGRATION_TESTS_MANUAL -->
           If integration tests don't run automatically, an authorized user can run them manually by following the instructions below:
-          
+
           Trigger:
           [go/deco-tests-run/sdk-py](https://go/deco-tests-run/sdk-py)
 
           Inputs:
           * PR number: ${{github.event.pull_request.number}}
           * Commit SHA: \`${{ env.COMMIT_SHA }}\`
-          
+
           Checks will be approved automatically on success.
           "

.github/workflows/integration-tests.yml

@@ -6,12 +6,16 @@ on:
     types: [opened, synchronize]
 
   merge_group:
-  
+
 
 jobs:
   check-token:
     name: Check secrets access
-    runs-on: ubuntu-latest
+
+    runs-on:
+      group: databricks-deco-testing-runner-group
+      labels: ubuntu-latest-deco
+
     environment: "test-trigger-is"
     outputs:
       has_token: ${{ steps.set-token-status.outputs.has_token }}
@@ -26,14 +30,18 @@ jobs:
               echo "DECO_WORKFLOW_TRIGGER_APP_ID is set. User has access to secrets."
               echo "::set-output name=has_token::true"
             fi
-    
+
   trigger-tests:
     name: Trigger Tests
-    runs-on: ubuntu-latest
+
+    runs-on:
+      group: databricks-deco-testing-runner-group
+      labels: ubuntu-latest-deco
+
     needs: check-token
     if: github.event_name == 'pull_request'  && needs.check-token.outputs.has_token == 'true'
     environment: "test-trigger-is"
-    
+
     steps:
     - uses: actions/checkout@v3
 
@@ -45,26 +53,30 @@ jobs:
         private-key: ${{ secrets.DECO_WORKFLOW_TRIGGER_PRIVATE_KEY }}
         owner: ${{ secrets.ORG_NAME }}
         repositories: ${{secrets.REPO_NAME}}
-    
+
     - name: Trigger Workflow in Another Repo
       env:
         GH_TOKEN: ${{ steps.generate-token.outputs.token }}
       run: |
         gh workflow run sdk-py-isolated-pr.yml -R ${{ secrets.ORG_NAME }}/${{secrets.REPO_NAME}} \
         --ref main \
         -f pull_request_number=${{ github.event.pull_request.number }} \
-        -f commit_sha=${{ github.event.pull_request.head.sha }} 
+        -f commit_sha=${{ github.event.pull_request.head.sha }}
 
-  # Statuses and checks apply to specific commits (by hash). 
+  # Statuses and checks apply to specific commits (by hash).
   # Enforcement of required checks is done both at the PR level and the merge queue level.
-  # In case of multiple commits in a single PR, the hash of the squashed commit 
+  # In case of multiple commits in a single PR, the hash of the squashed commit
   # will not match the one for the latest (approved) commit in the PR.
   # We auto approve the check for the merge queue for two reasons:
   # * Queue times out due to duration of tests.
   # * Avoid running integration tests twice, since it was already run at the tip of the branch before squashing.
   auto-approve:
     if: github.event_name == 'merge_group'
-    runs-on: ubuntu-latest
+
+    runs-on:
+      group: databricks-deco-testing-runner-group
+      labels: ubuntu-latest-deco
+
     steps:
       - name: Mark Check
         env:
@@ -75,4 +87,4 @@ jobs:
               -H "X-GitHub-Api-Version: 2022-11-28" \
               /repos/${{ github.repository }}/statuses/${{ github.sha }} \
               -f 'state=success' \
-              -f 'context=Integration Tests Check'
+              -f 'context=Integration Tests Check'

.github/workflows/release-test.yml

@@ -5,10 +5,15 @@ on:
 
 jobs:
   publish:
-    runs-on: ubuntu-latest
+    runs-on:
+      group: databricks-deco-testing-runner-group
+      labels: ubuntu-latest-deco
+
     environment: release-test
+
     permissions:
       id-token: write
+
     steps:
       - uses: actions/checkout@v3
 

.github/workflows/release.yml

@@ -7,11 +7,16 @@ on:
 
 jobs:
   publish:
-    runs-on: ubuntu-latest
+    runs-on:
+      group: databricks-deco-testing-runner-group
+      labels: ubuntu-latest-deco
+
     environment: release
+
     permissions:
       contents: write
       id-token: write
+
     steps:
       - uses: actions/checkout@v3
 

databricks/sdk/mixins/open_ai_client.py

@@ -1,4 +1,9 @@
-from databricks.sdk.service.serving import ServingEndpointsAPI
+import json as js
+from typing import Dict, Optional
+
+from databricks.sdk.service.serving import (ExternalFunctionRequestMethod,
+                                            ExternalFunctionResponse,
+                                            ServingEndpointsAPI)
 
 
 class ServingEndpointsExt(ServingEndpointsAPI):
@@ -50,3 +55,41 @@ def get_langchain_chat_open_ai_client(self, model):
             openai_api_base=self._api._cfg.host + "/serving-endpoints",
             api_key="no-token", # Passing in a placeholder to pass validations, this will not be used
             http_client=self._get_authorized_http_client())
+
+    def http_request(self,
+                     conn: str,
+                     method: ExternalFunctionRequestMethod,
+                     path: str,
+                     *,
+                     headers: Optional[Dict[str, str]] = None,
+                     json: Optional[Dict[str, str]] = None,
+                     params: Optional[Dict[str, str]] = None) -> ExternalFunctionResponse:
+        """Make external services call using the credentials stored in UC Connection.
+
+        **NOTE:** Experimental: This API may change or be removed in a future release without warning.
+
+        :param conn: str
+          The connection name to use. This is required to identify the external connection.
+        :param method: :class:`ExternalFunctionRequestMethod`
+          The HTTP method to use (e.g., 'GET', 'POST'). This is required.
+        :param path: str
+          The relative path for the API endpoint. This is required.
+        :param headers: Dict[str,str] (optional)
+          Additional headers for the request. If not provided, only auth headers from connections would be
+          passed.
+        :param json: Dict[str,str] (optional)
+          JSON payload for the request.
+        :param params: Dict[str,str] (optional)
+          Query parameters for the request.
+
+        :returns: :class:`ExternalFunctionResponse`
+        """
+
+        return super.http_request(
+            connection_name=conn,
+            method=method,
+            path=path,
+            headers=headers,
+            json=js.dumps(json),
+            params=js.dumps(params),
+        )