From 9ded429ab2ea62620ad8c55d735d69d871a1f628 Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Tue, 28 Jul 2020 13:57:16 +0300 Subject: [PATCH 01/13] ci: cache node_modules --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dc2a9bb9..7ac3b795 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,6 +68,12 @@ jobs: path: ~/.cargo key: ${{ matrix.platform.target }}-cargo-${{ matrix.toolchain.rust }} + - name: Cache node modules + uses: actions/cache@v1 + with: + path: conformance/node_modules + key: ${{ runner.os }}-npm-${{ hashFiles('conformance/package*.json', 'conformance/patches/**', 'conformance/setup.sh') }} + - name: Install dependencies ubuntu if: matrix.platform.host == 'ubuntu-latest' run: sudo apt-get install llvm-dev libssl-dev pkg-config From cb6a64360269ec5b1293b023f144cd3aa2fc2561 Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Tue, 28 Jul 2020 14:50:44 +0300 Subject: [PATCH 02/13] ci: use h(Cargo.lock) for cargo caching --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7ac3b795..9b2cb2be 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,7 +66,7 @@ jobs: uses: actions/cache@v1 with: path: ~/.cargo - key: ${{ matrix.platform.target }}-cargo-${{ matrix.toolchain.rust }} + key: ${{ matrix.platform.target }}-cargo-${{ hashFiles('Cargo.lock') }} - name: Cache node modules uses: actions/cache@v1 From 6a33edf8847e4ef4837dc8d26c30354a8147eb8d Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Tue, 28 Jul 2020 17:32:20 +0300 Subject: [PATCH 03/13] fix: dont run setup.sh twice (patching will fail) --- conformance/setup.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/conformance/setup.sh b/conformance/setup.sh index f969f95b..6fc9374f 100755 --- a/conformance/setup.sh +++ b/conformance/setup.sh @@ -8,6 +8,11 @@ if ! [ -f "./package.json" ]; then exit 1 fi +if [ -d "node_modules" ]; then + echo "Directory exists already: node_modules/" >&2 + exit 1 +fi + # production will skip the dev dependencies npm install --production From 7b38dec376d3d3e7bf6f4e0587bc1c8f81058e1f Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Tue, 28 Jul 2020 18:37:27 +0300 Subject: [PATCH 04/13] ci: rename key for node_modules apparently the upload had failed or something previously. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9b2cb2be..5bcc599e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,7 +72,7 @@ jobs: uses: actions/cache@v1 with: path: conformance/node_modules - key: ${{ runner.os }}-npm-${{ hashFiles('conformance/package*.json', 'conformance/patches/**', 'conformance/setup.sh') }} + key: ${{ matrix.platform.target }}-npm-${{ hashFiles('conformance/package*.json', 'conformance/patches/**', 'conformance/setup.sh') }} - name: Install dependencies ubuntu if: matrix.platform.host == 'ubuntu-latest' From 79a9b2ce7632796cbb1e80b684fcda7638bb20ea Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Tue, 28 Jul 2020 18:37:56 +0300 Subject: [PATCH 05/13] ci: make the readme/lint caches useful --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5bcc599e..d2c77fdd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -160,7 +160,7 @@ jobs: uses: actions/cache@v1 with: path: ~/.cargo - key: lint-cargo + key: lint-cargo-${{ hashFiles('Cargo.lock') }} - name: Install rust toolchain uses: hecrj/setup-rust-action@v1 @@ -184,7 +184,7 @@ jobs: uses: actions/cache@v1 with: path: ~/.cargo - key: readme-doctest + key: readme-doctest-${{ hashFiles('Cargo.lock') }} - name: Install rust toolchain uses: hecrj/setup-rust-action@v1 From 337f1b2ebf3c450c79dbe6a931c13dcc49768413 Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Tue, 28 Jul 2020 18:43:13 +0300 Subject: [PATCH 06/13] ci: update cache to version 2 sadly we cannot follow the cargo recipe as the target dir is at least 2GB and cannot be cached... unless the final artifacts were excluded? --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d2c77fdd..e5af024d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,13 +63,13 @@ jobs: uses: actions/checkout@v2 - name: Cache cargo folder - uses: actions/cache@v1 + uses: actions/cache@v2 with: path: ~/.cargo key: ${{ matrix.platform.target }}-cargo-${{ hashFiles('Cargo.lock') }} - name: Cache node modules - uses: actions/cache@v1 + uses: actions/cache@v2 with: path: conformance/node_modules key: ${{ matrix.platform.target }}-npm-${{ hashFiles('conformance/package*.json', 'conformance/patches/**', 'conformance/setup.sh') }} @@ -157,7 +157,7 @@ jobs: uses: actions/checkout@v2 - name: Cache cargo folder - uses: actions/cache@v1 + uses: actions/cache@v2 with: path: ~/.cargo key: lint-cargo-${{ hashFiles('Cargo.lock') }} @@ -181,7 +181,7 @@ jobs: uses: actions/checkout@v2 - name: Cache cargo folder - uses: actions/cache@v1 + uses: actions/cache@v2 with: path: ~/.cargo key: readme-doctest-${{ hashFiles('Cargo.lock') }} From cca80c433f1c1905f2aec70db576b4600c22b7b0 Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Tue, 28 Jul 2020 19:12:04 +0300 Subject: [PATCH 07/13] doc: remove obsolete comment --- conformance/setup.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/conformance/setup.sh b/conformance/setup.sh index 6fc9374f..950a6809 100755 --- a/conformance/setup.sh +++ b/conformance/setup.sh @@ -21,9 +21,6 @@ if [ -d "patches" ]; then # as we want to apply the patches create in a js-ipfs checkout to our node_modules # we'll need to remove a few leading path segments to match # a/packages/interface-ipfs-core/src/refs.js to node_modules/interface-ipfs-core/src/refs.js - # - # cannot use git automation any longer as it will skip any ignored file apparently, - # and node_modules are ignored. for p in patches/*; do echo "Applying $(basename "$p")..." >&2 patch -d node_modules/ -p1 < "$p" From ec0b1b3f62aede73ea6d1d2a3fa7512930f30277 Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Tue, 28 Jul 2020 19:18:08 +0300 Subject: [PATCH 08/13] ci: use cookbook for npm caching quite unsure how this will work on windows but lets see. --- .github/workflows/ci.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e5af024d..e873c02c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,11 +68,17 @@ jobs: path: ~/.cargo key: ${{ matrix.platform.target }}-cargo-${{ hashFiles('Cargo.lock') }} - - name: Cache node modules - uses: actions/cache@v2 + - name: Get npm cache directory + id: npm-cache + run: | + echo "::set-output name=dir::$(npm config get cache)" + + - uses: actions/cache@v2 with: - path: conformance/node_modules - key: ${{ matrix.platform.target }}-npm-${{ hashFiles('conformance/package*.json', 'conformance/patches/**', 'conformance/setup.sh') }} + path: ${{ steps.npm-cache.outputs.dir }} + key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json', '**/setup.sh') }} + restore-keys: | + ${{ runner.os }}-node- - name: Install dependencies ubuntu if: matrix.platform.host == 'ubuntu-latest' From 34b158e35c4df29e44898ce7b778e31475a2847e Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Tue, 28 Jul 2020 22:37:00 +0300 Subject: [PATCH 09/13] ci: bogus cache key rename --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e873c02c..de8e6d62 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,7 +66,7 @@ jobs: uses: actions/cache@v2 with: path: ~/.cargo - key: ${{ matrix.platform.target }}-cargo-${{ hashFiles('Cargo.lock') }} + key: ${{ matrix.platform.target }}-dot-cargo-${{ hashFiles('Cargo.lock') }} - name: Get npm cache directory id: npm-cache From 144985d06dfcc623d6a1d67b991a0364b4b9556d Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Tue, 28 Jul 2020 22:37:52 +0300 Subject: [PATCH 10/13] ci: name the npm caching step --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de8e6d62..32148f39 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,7 +73,8 @@ jobs: run: | echo "::set-output name=dir::$(npm config get cache)" - - uses: actions/cache@v2 + - name: Cache npm + uses: actions/cache@v2 with: path: ${{ steps.npm-cache.outputs.dir }} key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json', '**/setup.sh') }} From 9ec7583343b857eab8b48f4d770f9eeeef8f3bdf Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Thu, 30 Jul 2020 13:39:57 +0300 Subject: [PATCH 11/13] ci: use cookbook recipe for cargo caching --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 32148f39..0f588be7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,7 +65,9 @@ jobs: - name: Cache cargo folder uses: actions/cache@v2 with: - path: ~/.cargo + path: | + ~/.cargo/registry + ~/.cargo/git key: ${{ matrix.platform.target }}-dot-cargo-${{ hashFiles('Cargo.lock') }} - name: Get npm cache directory From 832bbfaafeb99519eb8d0244508ba5fccd3147f8 Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Thu, 30 Jul 2020 13:46:27 +0300 Subject: [PATCH 12/13] ci: skip npm caching for cross builds, cache key the cross builders might end up otherwise poisoning the cache with an empty npm. this may have already happened and explains why it didn't work for mac and linux. also include the patches again in the cache key. --- .github/workflows/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0f588be7..7dcdfc14 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,14 +72,16 @@ jobs: - name: Get npm cache directory id: npm-cache + if: matrix.platform.cross == false run: | echo "::set-output name=dir::$(npm config get cache)" - - name: Cache npm + - name: Cache npm (non-cross targets) uses: actions/cache@v2 + if: matrix.platform.cross == false with: path: ${{ steps.npm-cache.outputs.dir }} - key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json', '**/setup.sh') }} + key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json', '**/setup.sh', '**/*.patch') }} restore-keys: | ${{ runner.os }}-node- From 4208decc419c60b50156e3981eb362277fe6f3d8 Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Thu, 30 Jul 2020 14:42:22 +0300 Subject: [PATCH 13/13] ci: rekey cargo cache --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7dcdfc14..4a33db7b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,7 +68,7 @@ jobs: path: | ~/.cargo/registry ~/.cargo/git - key: ${{ matrix.platform.target }}-dot-cargo-${{ hashFiles('Cargo.lock') }} + key: ${{ matrix.platform.target }}-dot-cargo-parts-${{ hashFiles('Cargo.lock') }} - name: Get npm cache directory id: npm-cache