Move Core Dockerfile into repo and reduce size (#769)

Author: leighmcculloch

.github/workflows/build.yml

@@ -68,7 +68,7 @@ env:
   # Cache ID is a value inserted into cache keys. Whenever changing the build
   # in a way that needs to use entirely new fresh builds, increment the number
   # by one so that all the keys become new.
-  CACHE_ID: 3
+  CACHE_ID: 4
   ARTIFACT_RETENTION_DAYS_FOR_IMAGE: 7
   ARTIFACT_RETENTION_DAYS_FOR_LOGS: 60
 
@@ -89,7 +89,7 @@ jobs:
     - id: xdr
       run: echo "sha=$(gh api repos/stellar/rs-stellar-xdr/commits/"${{inputs.xdr_ref}}" --jq '.sha')" | tee -a $GITHUB_OUTPUT
     - id: core
-      run: echo "sha=$(gh api repos/"${{inputs.core_repo}}"/commits/"${{inputs.core_ref}}" --jq '.sha')" | tee -a $GITHUB_OUTPUT
+      run: echo "sha=$(gh api repos/${{inputs.core_repo}}/commits/"${{inputs.core_ref}}" --jq '.sha')" | tee -a $GITHUB_OUTPUT
     - id: rpc
       run: echo "sha=$(gh api repos/stellar/stellar-rpc/commits/"${{inputs.stellar_rpc_ref}}" --jq '.sha')" | tee -a $GITHUB_OUTPUT
     - id: horizon
@@ -123,6 +123,10 @@ jobs:
     if: ${{ needs.load-stellar-core-from-cache.outputs.cache-hit != 'true' }}
     runs-on: ${{ inputs.arch == 'arm64' && 'ubuntu-jammy-4-cores-arm64' || 'ubuntu-latest' }}
     steps:
+    - name: Checkout Quickstart for Core docker file
+      uses: actions/checkout@v3
+      with:
+        ref: ${{ inputs.sha }}
     - id: cache
       uses: actions/cache@v3
       with:
@@ -136,11 +140,12 @@ jobs:
     - name: Build Stellar-Core Image
       run: >
         docker buildx build --platform linux/${{ inputs.arch }}
-        -f docker/Dockerfile.testing -t stellar-core:${{ inputs.arch }}
+        -f Dockerfile.core
+        -t stellar-core:${{ inputs.arch }}
         -o type=docker,dest=/tmp/image
-        https://github.com/${{ inputs.core_repo }}.git#${{ needs.shas.outputs.core }}
-        --build-arg BUILDKIT_CONTEXT_KEEP_GIT_DIR=true
-        --build-arg CONFIGURE_FLAGS='${{ inputs.core_configure_flags }}'
+        --build-arg REPO="${{ inputs.core_repo }}"
+        --build-arg REF="${{ needs.shas.outputs.core }}"
+        --build-arg CONFIGURE_FLAGS='${{ inputs.core_configure_flags }}' .
     - name: Upload Stellar-Core Image
       uses: actions/upload-artifact@v4
       with:
@@ -401,7 +406,7 @@ jobs:
         -f Dockerfile.xdr
         -t stellar-rs-xdr:${{ inputs.arch }}
         -o type=docker,dest=/tmp/image
-        --build-arg REPO=https://github.com/stellar/rs-stellar-xdr.git
+        --build-arg REPO=stellar/rs-stellar-xdr
         --build-arg REF="${{ needs.shas.outputs.xdr }}" .
     - name: Upload Stellar-Rs-Xdr Image
       uses: actions/upload-artifact@v4

Dockerfile

@@ -30,7 +30,7 @@ ADD dependencies /
 RUN /dependencies
 
 COPY --from=stellar-xdr /stellar-xdr /usr/local/bin/stellar-xdr
-COPY --from=stellar-core /usr/local/bin/stellar-core /usr/bin/stellar-core
+COPY --from=stellar-core /stellar-core /usr/bin/stellar-core
 COPY --from=horizon /horizon /usr/bin/stellar-horizon
 COPY --from=friendbot /friendbot /usr/local/bin/friendbot
 COPY --from=stellar-rpc /stellar-rpc /usr/bin/stellar-rpc

Dockerfile.core

@@ -0,0 +1,36 @@
+FROM ubuntu:focal AS builder
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && \
+    apt-get -y install iproute2 procps lsb-release \
+                       git build-essential pkg-config autoconf automake libtool \
+                       bison flex sed perl libpq-dev parallel libunwind-dev \
+                       clang-12 libc++abi-12-dev libc++-12-dev \
+                       postgresql curl
+
+ARG REPO
+ARG REF
+WORKDIR /wd
+RUN git clone https://github.com/${REPO} /wd
+RUN git fetch origin ${REF}
+RUN git checkout ${REF}
+
+RUN git clean -dXf
+RUN git submodule foreach --recursive git clean -dXf
+
+ARG CC=clang-12
+ARG CXX=clang++-12
+ARG CFLAGS='-O3 -g1'
+ARG CXXFLAGS='-O3 -g1'
+ARG CONFIGURE_FLAGS=''
+
+RUN ./autogen.sh
+RUN ./install-rust.sh
+ENV PATH "/root/.cargo/bin:$PATH"
+RUN ./configure CC="${CC}" CXX="${CXX}" CFLAGS="${CFLAGS}" CXXFLAGS="${CXXFLAGS}" ${CONFIGURE_FLAGS}
+RUN sh -c 'make -j $(nproc)'
+RUN make install
+
+FROM scratch AS artifacts
+
+COPY --from=builder /usr/local/bin/stellar-core /stellar-core

Dockerfile.xdr

@@ -3,7 +3,7 @@ FROM rust AS builder
 ARG REPO
 ARG REF
 WORKDIR /wd
-RUN git clone ${REPO} /wd
+RUN git clone https://github.com/${REPO} /wd
 RUN git fetch origin ${REF}
 RUN git checkout ${REF}
 RUN rustup show active-toolchain || rustup toolchain install

Makefile

@@ -3,9 +3,9 @@ __PHONY__: run logs build build-deps build-deps-core build-deps-horizon build-de
 REVISION=$(shell git -c core.abbrev=no describe --always --exclude='*' --long --dirty)
 TAG?=dev
 PROTOCOL_VERSION_DEFAULT?=22
-XDR_REPO?=https://github.com/stellar/rs-stellar-xdr.git
+XDR_REPO?=stellar/rs-stellar-xdr
 XDR_REF?=main
-CORE_REPO?=https://github.com/stellar/stellar-core.git
+CORE_REPO?=stellar/stellar-core
 CORE_REF?=master
 CORE_CONFIGURE_FLAGS?=--disable-tests
 STELLAR_RPC_REF?=main
@@ -67,7 +67,7 @@ build-deps-xdr:
 	docker build -t stellar-xdr:$(XDR_REF) -f Dockerfile.xdr . --build-arg REPO="$(XDR_REPO)" --build-arg REF="$(XDR_REF)"
 
 build-deps-core:
-	docker build -t stellar-core:$(CORE_REF) -f docker/Dockerfile.testing $(CORE_REPO)#$(CORE_REF) --build-arg BUILDKIT_CONTEXT_KEEP_GIT_DIR=true --build-arg CONFIGURE_FLAGS="$(CORE_CONFIGURE_FLAGS)"
+	docker build -t stellar-core:$(CORE_REF) -f Dockerfile.core . --build-arg REPO="$(CORE_REPO)" --build-arg REF="$(CORE_REF)" --build-arg CONFIGURE_FLAGS="$(CORE_CONFIGURE_FLAGS)"
 
 build-deps-horizon:
 	docker build -t stellar-horizon:$(HORIZON_REF) -f Dockerfile.horizon . --build-arg REF="$(HORIZON_REF)"