diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index d49291a43e3c..e9b0c7c93e2b 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -17,8 +17,25 @@ defaults: run: shell: bash +env: + # hostnames expected by https://github.com/ipfs/gateway-conformance + GATEWAY_PUBLIC_GATEWAYS: | + { + "example.com": { + "UseSubdomains": true, + "InlineDNSLink": true, + "Paths": ["/ipfs", "/ipns"] + }, + "localhost": { + "UseSubdomains": true, + "InlineDNSLink": true, + "Paths": ["/ipfs", "/ipns"] + } + } + jobs: - gateway-conformance: + # Testing all gateway features via TCP port specified in Addresses.Gateway + full-over-http: runs-on: ubuntu-latest timeout-minutes: 10 steps: @@ -30,11 +47,14 @@ jobs: # 2. Build the kubo-gateway - name: Setup Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: go-version: 1.20.x + - uses: protocol/cache-go-action@v1 + with: + name: ${{ github.job }} - name: Checkout kubo-gateway - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: path: kubo-gateway - name: Build kubo-gateway @@ -43,26 +63,9 @@ jobs: # 3. Init the kubo-gateway - name: Init kubo-gateway - env: - GATEWAY_PUBLIC_GATEWAYS: | - { - "example.com": { - "UseSubdomains": true, - "InlineDNSLink": true, - "Paths": ["/ipfs", "/ipns"] - }, - "localhost": { - "UseSubdomains": true, - "InlineDNSLink": true, - "Paths": ["/ipfs", "/ipns"] - } - } run: | - ./ipfs init --profile=test + ./ipfs init -e ./ipfs config --json Gateway.PublicGateways "$GATEWAY_PUBLIC_GATEWAYS" - ./ipfs config --json Experimental.GatewayOverLibp2p true - ./ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/8080" - ./ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5001" working-directory: kubo-gateway/cmd/ipfs # 4. Populate the Kubo gateway with the gateway-conformance fixtures @@ -88,28 +91,7 @@ jobs: # 5. Start the kubo-gateway - name: Start kubo-gateway run: | - ./ipfs daemon & - endpoint="http://127.0.0.1:5001/api/v0/version" - max_retries=5 - retry_interval=3 - - check_endpoint() { - curl -X POST --silent --fail "$endpoint" > /dev/null - return $? - } - - retries=0 - while ! check_endpoint; do - retries=$((retries+1)) - - if [ $retries -ge $max_retries ]; then - echo "daemon took too long to start" - exit 1 - fi - - sleep $retry_interval - done - echo "daemon started and ready to receive API calls" + ./ipfs daemon --offline & working-directory: kubo-gateway/cmd/ipfs # 6. Run the gateway-conformance tests @@ -140,48 +122,77 @@ jobs: name: gateway-conformance.json path: output.json - # 8. Setup a kubo http-p2p-proxy to run gateway conformance tests + # Testing trustless gateway feature subset exposed as libp2p protocol + subset-over-libp2p-experiment: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + # 1. Download the gateway-conformance fixtures + - name: Download gateway-conformance fixtures + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.3 + with: + output: fixtures + + # 2. Build the kubo-gateway + - name: Setup Go + uses: actions/setup-go@v4 + with: + go-version: 1.20.x + - uses: protocol/cache-go-action@v1 + with: + name: ${{ github.job }} + - name: Checkout kubo-gateway + uses: actions/checkout@v3 + with: + path: kubo-gateway + - name: Build kubo-gateway + run: make build + working-directory: kubo-gateway + + # 3. Init the kubo-gateway + - name: Init kubo-gateway + run: | + ./ipfs init --profile=test + ./ipfs config --json Gateway.PublicGateways "$GATEWAY_PUBLIC_GATEWAYS" + ./ipfs config --json Experimental.GatewayOverLibp2p true + ./ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/8080" + ./ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5001" + working-directory: kubo-gateway/cmd/ipfs + + # 4. Populate the Kubo gateway with the gateway-conformance fixtures + - name: Import fixtures + run: | + # Import car files + find ./fixtures -name '*.car' -exec kubo-gateway/cmd/ipfs/ipfs dag import --pin-roots=false {} \; + + # 5. Start the kubo-gateway + - name: Start kubo-gateway + run: | + ( ./ipfs daemon & ) | sed '/Daemon is ready/q' + while [[ "$(./ipfs id | jq '.Addresses | length')" == '0' ]]; do sleep 1; done + working-directory: kubo-gateway/cmd/ipfs + + # 6. Setup a kubo http-p2p-proxy to expose libp2p protocol as a regular HTTP port for gateway conformance tests - name: Init p2p-proxy kubo node env: IPFS_PATH: "~/.kubo-p2p-proxy" run: | - ./ipfs init --profile=test + ./ipfs init --profile=test -e ./ipfs config --json Experimental.Libp2pStreamMounting true ./ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/8081" ./ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5002" working-directory: kubo-gateway/cmd/ipfs - # 9. Start the kubo http-p2p-proxy + # 7. Start the kubo http-p2p-proxy - name: Start kubo http-p2p-proxy env: IPFS_PATH: "~/.kubo-p2p-proxy" run: | - ./ipfs daemon & - - endpoint="http://127.0.0.1:5002/api/v0/version" - max_retries=5 - retry_interval=3 - - check_endpoint() { - curl -X POST --silent --fail "$endpoint" > /dev/null - return $? - } - - retries=0 - while ! check_endpoint; do - retries=$((retries+1)) - - if [ $retries -ge $max_retries ]; then - echo "daemon took too long to start" - exit 1 - fi - - sleep $retry_interval - done - echo "daemon started and ready to receive API calls" + ( ./ipfs daemon & ) | sed '/Daemon is ready/q' + while [[ "$(./ipfs id | jq '.Addresses | length')" == '0' ]]; do sleep 1; done working-directory: kubo-gateway/cmd/ipfs - # 10. Start forwarding data from the http-p2p-proxy to the node serving the Gateway API over libp2p + # 8. Start forwarding data from the http-p2p-proxy to the node serving the Gateway API over libp2p - name: Start http-over-libp2p forwarding proxy run: | gatewayNodeId=$(./ipfs --api=/ip4/127.0.0.1/tcp/5001 id -f="") @@ -189,7 +200,7 @@ jobs: ./ipfs --api=/ip4/127.0.0.1/tcp/5002 p2p forward --allow-custom-protocol /http/1.1 /ip4/127.0.0.1/tcp/8082 /p2p/$gatewayNodeId working-directory: kubo-gateway/cmd/ipfs - # 11. Run the gateway-conformance tests over libp2p + # 9. Run the gateway-conformance tests over libp2p - name: Run gateway-conformance tests over libp2p uses: ipfs/gateway-conformance/.github/actions/test@v0.3 with: @@ -200,7 +211,7 @@ jobs: markdown: output.md args: --specs "trustless-gateway,-trustless-ipns-gateway" -skip 'TestGatewayCar/GET_response_for_application/vnd.ipld.car/Header_Content-Length' - # 11. Upload the results + # 10. Upload the results - name: Upload MD summary if: failure() || success() run: cat output.md >> $GITHUB_STEP_SUMMARY @@ -215,4 +226,4 @@ jobs: uses: actions/upload-artifact@v3 with: name: gateway-conformance-libp2p.json - path: output.json \ No newline at end of file + path: output.json