feat: package caches #948
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Python basic images | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| release_tag: | |
| description: 'Tag for the images (e.g.: "latest" or "beta")' | |
| required: true | |
| apify_version: | |
| description: 'Apify Python SDK version (e.g.: "1.0.0"). If missing, the latest version will be used.' | |
| required: false | |
| rebuild_images: | |
| description: "Rebuilds images even if the cache state matches the current state." | |
| required: false | |
| type: boolean | |
| repository_dispatch: | |
| types: [build-python-images] | |
| pull_request: | |
| schedule: | |
| - cron: 0 */2 * * * | |
| env: | |
| RELEASE_TAG: ${{ github.event.inputs.release_tag || github.event.client_payload.release_tag }} | |
| APIFY_VERSION: ${{ github.event.inputs.apify_version || github.event.client_payload.apify_version }} | |
| SKIP_CACHE_CHECK: ${{ github.event_name == 'pull_request' || github.event.inputs.rebuild_images == 'true' }} | |
| jobs: | |
| matrix: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} | |
| fetch-depth: 0 | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version-file: package.json | |
| cache: yarn | |
| cache-dependency-path: yarn.lock | |
| - run: yarn | |
| - name: Generate matrix | |
| id: set-matrix | |
| run: echo "matrix=$(yarn python:normal)" >> $GITHUB_OUTPUT | |
| working-directory: ./.github/actions/version-matrix | |
| - name: Print matrix | |
| run: | | |
| echo '${{ steps.set-matrix.outputs.matrix }}' | jq -r '.include[] | "python-version=\(.["python-version"]) apify-version=\(.["apify-version"])"' | |
| echo "" | |
| echo "Raw matrix:" | |
| echo "" | |
| echo '${{ steps.set-matrix.outputs.matrix }}' | jq -e | |
| - name: Push updated matrix | |
| if: github.event_name != 'pull_request' | |
| run: | | |
| # Setup git user | |
| git config --global user.email "[email protected]" | |
| git config --global user.name "Apify CI Bot" | |
| git config pull.rebase true | |
| # Add and commit if there are changes | |
| git add ./.github/actions/version-matrix/data/*.json | |
| git diff-index --quiet HEAD || git commit -m "chore(docker): update ${{ env.RELEASE_TAG || 'latest' }} python:normal cache" | |
| # Try to push 5 times, with pulls between retries | |
| for i in {1..5}; do | |
| git push && break || echo "Failed to push, retrying in 5 seconds..." && sleep 5 && git pull | |
| done | |
| # Build master images that are not dependent on existing builds. | |
| build-main: | |
| needs: [matrix] | |
| runs-on: ubuntu-latest | |
| if: ${{ toJson(fromJson(needs.matrix.outputs.matrix).include) != '[]' }} | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.matrix.outputs.matrix) }} | |
| name: "py: ${{ matrix.python-version }} apify: ${{ matrix.apify-version }}" | |
| steps: | |
| - name: Set default inputs if event is pull request | |
| if: github.event_name == 'pull_request' | |
| run: | | |
| if [[ -z "$RELEASE_TAG" ]]; then echo "RELEASE_TAG=CI_TEST" >> $GITHUB_ENV; fi | |
| if [[ -z "$APIFY_VERSION" ]]; then echo "APIFY_VERSION=${{ matrix.apify-version }}" >> $GITHUB_ENV; fi | |
| - name: Set default inputs if event is schedule | |
| if: github.event_name == 'schedule' | |
| run: | | |
| if [[ -z "$RELEASE_TAG" ]]; then echo "RELEASE_TAG=latest" >> $GITHUB_ENV; fi | |
| if [[ -z "$APIFY_VERSION" ]]; then echo "APIFY_VERSION=${{ matrix.apify-version }}" >> $GITHUB_ENV; fi | |
| - name: Set default inputs if event is workflow_dispatch or repository_dispatch | |
| if: github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' | |
| run: | | |
| if [[ -z "$APIFY_VERSION" ]]; then echo "APIFY_VERSION=${{ matrix.apify-version }}" >> $GITHUB_ENV; fi | |
| - name: Check if inputs are set correctly | |
| run: | | |
| if [[ -z "$RELEASE_TAG" ]]; then echo "RELEASE_TAG input is empty!" >&2; exit 1; fi | |
| if [[ -z "$APIFY_VERSION" ]]; then echo "APIFY_VERSION input is empty!" >&2; exit 1; fi | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Update pip | |
| run: python -m pip install --upgrade pip | |
| - # It seems that it takes a few minutes before a newly published version | |
| # becomes available in the PyPI registry. We wait before starting the image builds. | |
| name: Wait For Package Registry | |
| uses: nick-fields/retry@v3 | |
| with: | |
| timeout_minutes: 2 # timeout for a single attempt | |
| max_attempts: 5 | |
| retry_wait_seconds: 60 # wait between retries | |
| command: pip install apify~=$APIFY_VERSION | |
| - name: Prepare image tags | |
| id: prepare-tags | |
| uses: actions/github-script@v7 | |
| env: | |
| CURRENT_PYTHON: ${{ matrix.python-version }} | |
| LATEST_PYTHON: ${{ matrix.latest-python-version }} | |
| RELEASE_TAG: ${{ env.RELEASE_TAG }} | |
| IMAGE_NAME: apify/actor-${{ matrix.image-name }} | |
| # Force this to true, as these images have no browsers | |
| IS_LATEST_BROWSER_IMAGE: "true" | |
| with: | |
| script: | | |
| const generateTags = require("./.github/scripts/prepare-python-image-tags.js"); | |
| return generateTags() | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Build and tag image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: ./${{ matrix.image-name }} | |
| file: ./${{ matrix.image-name }}/Dockerfile | |
| build-args: | | |
| PYTHON_VERSION=${{ matrix.python-version }} | |
| APIFY_VERSION=${{ env.APIFY_VERSION }} | |
| load: true | |
| tags: ${{ fromJson(steps.prepare-tags.outputs.result).allTags }} | |
| - name: Test image | |
| run: docker run ${{ fromJson(steps.prepare-tags.outputs.result).firstImageName }} | |
| - name: Login to DockerHub | |
| if: github.event_name != 'pull_request' | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.APIFY_SERVICE_ACCOUNT_DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.APIFY_SERVICE_ACCOUNT_DOCKERHUB_TOKEN }} | |
| - name: Push images | |
| if: github.event_name != 'pull_request' | |
| run: docker push apify/actor-${{ matrix.image-name }} --all-tags |