Skip to content

Commit d9e6ce4

Browse files
mthrokmthrok
authored
Refactor Cache bust mechanism and bust on daily basis (#742)
This PR refactors cache generation mechanism by introducing dedicated command and bust cache on daily basis. At this moment, Windows unittest job for 3.6 and 3.7 are broken because of broken scipy but the environment is cached this persists until the next week. As we have nightly build, we do not need to keep cache for one week.
1 parent b8ddeb3 commit d9e6ce4

File tree

2 files changed

+48
-72
lines changed

2 files changed

+48
-72
lines changed

‎.circleci/config.yml‎

Lines changed: 24 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,12 @@ executors:
2020
shell: bash.exe
2121

2222
commands:
23+
generate_cache_key:
24+
description: "Generates a cache key file that changes daily"
25+
steps:
26+
- run:
27+
name: Generate cache key
28+
command: echo "$(date +"%Y-%m-%d")" > .cachekey
2329
designate_upload_channel:
2430
description: "inserts the correct upload channel into ${BASH_ENV}"
2531
steps:
@@ -80,20 +86,17 @@ jobs:
8086
resource_class: small
8187
steps:
8288
- checkout
83-
- run:
84-
name: Generate cache key
85-
# This will refresh cache on daily
86-
command: echo "$(date +"%Y-%U-%d")" > .circleci-daily
89+
- generate_cache_key
8790
- restore_cache:
8891

8992
keys:
90-
- tp-nix-{{ checksum ".circleci-daily" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }}
93+
- tp-nix-{{ checksum ".cachekey" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }}
9194

9295
- run:
9396
command: ./build_tools/setup_helpers/build_third_party.sh $PWD --download-only
9497
- save_cache:
9598

96-
key: tp-nix-{{ checksum ".circleci-daily" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }}
99+
key: tp-nix-{{ checksum ".cachekey" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }}
97100

98101
paths:
99102
- third_party/tmp
@@ -376,21 +379,18 @@ jobs:
376379
- checkout
377380
- attach_workspace:
378381
at: third_party
379-
- run:
380-
name: Generate cache key
381-
# This will refresh cache on Sundays, nightly build should generate new cache.
382-
command: echo "$(date +"%Y-%U")" > .circleci-weekly
382+
- generate_cache_key
383383
- restore_cache:
384384

385385
keys:
386-
- env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
386+
- env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
387387

388388
- run:
389389
name: Setup
390390
command: .circleci/unittest/linux/scripts/setup_env.sh
391391
- save_cache:
392392

393-
key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
393+
key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
394394

395395
paths:
396396
- conda
@@ -419,21 +419,18 @@ jobs:
419419
- checkout
420420
- attach_workspace:
421421
at: third_party
422-
- run:
423-
name: Generate cache key
424-
# This will refresh cache on Sundays, nightly build should generate new cache.
425-
command: echo "$(date +"%Y-%U")" > .circleci-weekly
422+
- generate_cache_key
426423
- restore_cache:
427424

428425
keys:
429-
- env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
426+
- env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
430427

431428
- run:
432429
name: Setup
433430
command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh
434431
- save_cache:
435432

436-
key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
433+
key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
437434

438435
paths:
439436
- conda
@@ -457,21 +454,18 @@ jobs:
457454
name: windows-cpu
458455
steps:
459456
- checkout
460-
- run:
461-
name: Generate cache key
462-
# This will refresh cache on Sundays, nightly build should generate new cache.
463-
command: echo "$(date +"%Y-%U")" > .circleci-weekly
457+
- generate_cache_key
464458
- restore_cache:
465459

466460
keys:
467-
- env-v2-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
461+
- env-v3-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
468462

469463
- run:
470464
name: Setup
471465
command: .circleci/unittest/windows/scripts/setup_env.sh
472466
- save_cache:
473467

474-
key: env-v2-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
468+
key: env-v3-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
475469

476470
paths:
477471
- conda
@@ -496,21 +490,18 @@ jobs:
496490
CUDA_VERSION: "10.1"
497491
steps:
498492
- checkout
499-
- run:
500-
name: Generate cache key
501-
# This will refresh cache on Sundays, nightly build should generate new cache.
502-
command: echo "$(date +"%Y-%U")" > .circleci-weekly
493+
- generate_cache_key
503494
- restore_cache:
504495

505496
keys:
506-
- env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
497+
- env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
507498

508499
- run:
509500
name: Setup
510501
command: .circleci/unittest/windows/scripts/setup_env.sh
511502
- save_cache:
512503

513-
key: env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
504+
key: env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
514505

515506
paths:
516507
- conda
@@ -534,21 +525,18 @@ jobs:
534525
resource_class: medium
535526
steps:
536527
- checkout
537-
- run:
538-
name: Generate cache key
539-
# This will refresh cache on Sundays, nightly build should generate new cache.
540-
command: echo "$(date +"%Y-%U")" > .circleci-weekly
528+
- generate_cache_key
541529
- restore_cache:
542530

543531
keys:
544-
- env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
532+
- env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
545533

546534
- run:
547535
name: Setup
548536
command: .circleci/unittest/linux/scripts/setup_env.sh
549537
- save_cache:
550538

551-
key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
539+
key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
552540

553541
paths:
554542
- conda

‎.circleci/config.yml.in‎

Lines changed: 24 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,12 @@ executors:
2020
shell: bash.exe
2121

2222
commands:
23+
generate_cache_key:
24+
description: "Generates a cache key file that changes daily"
25+
steps:
26+
- run:
27+
name: Generate cache key
28+
command: echo "$(date +"%Y-%m-%d")" > .cachekey
2329
designate_upload_channel:
2430
description: "inserts the correct upload channel into ${BASH_ENV}"
2531
steps:
@@ -80,20 +86,17 @@ jobs:
8086
resource_class: small
8187
steps:
8288
- checkout
83-
- run:
84-
name: Generate cache key
85-
# This will refresh cache on daily
86-
command: echo "$(date +"%Y-%U-%d")" > .circleci-daily
89+
- generate_cache_key
8790
- restore_cache:
8891
{% raw %}
8992
keys:
90-
- tp-nix-{{ checksum ".circleci-daily" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }}
93+
- tp-nix-{{ checksum ".cachekey" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }}
9194
{% endraw %}
9295
- run:
9396
command: ./build_tools/setup_helpers/build_third_party.sh $PWD --download-only
9497
- save_cache:
9598
{% raw %}
96-
key: tp-nix-{{ checksum ".circleci-daily" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }}
99+
key: tp-nix-{{ checksum ".cachekey" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }}
97100
{% endraw %}
98101
paths:
99102
- third_party/tmp
@@ -376,21 +379,18 @@ jobs:
376379
- checkout
377380
- attach_workspace:
378381
at: third_party
379-
- run:
380-
name: Generate cache key
381-
# This will refresh cache on Sundays, nightly build should generate new cache.
382-
command: echo "$(date +"%Y-%U")" > .circleci-weekly
382+
- generate_cache_key
383383
- restore_cache:
384384
{% raw %}
385385
keys:
386-
- env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
386+
- env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
387387
{% endraw %}
388388
- run:
389389
name: Setup
390390
command: .circleci/unittest/linux/scripts/setup_env.sh
391391
- save_cache:
392392
{% raw %}
393-
key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
393+
key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
394394
{% endraw %}
395395
paths:
396396
- conda
@@ -419,21 +419,18 @@ jobs:
419419
- checkout
420420
- attach_workspace:
421421
at: third_party
422-
- run:
423-
name: Generate cache key
424-
# This will refresh cache on Sundays, nightly build should generate new cache.
425-
command: echo "$(date +"%Y-%U")" > .circleci-weekly
422+
- generate_cache_key
426423
- restore_cache:
427424
{% raw %}
428425
keys:
429-
- env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
426+
- env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
430427
{% endraw %}
431428
- run:
432429
name: Setup
433430
command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh
434431
- save_cache:
435432
{% raw %}
436-
key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
433+
key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
437434
{% endraw %}
438435
paths:
439436
- conda
@@ -457,21 +454,18 @@ jobs:
457454
name: windows-cpu
458455
steps:
459456
- checkout
460-
- run:
461-
name: Generate cache key
462-
# This will refresh cache on Sundays, nightly build should generate new cache.
463-
command: echo "$(date +"%Y-%U")" > .circleci-weekly
457+
- generate_cache_key
464458
- restore_cache:
465459
{% raw %}
466460
keys:
467-
- env-v2-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
461+
- env-v3-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
468462
{% endraw %}
469463
- run:
470464
name: Setup
471465
command: .circleci/unittest/windows/scripts/setup_env.sh
472466
- save_cache:
473467
{% raw %}
474-
key: env-v2-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
468+
key: env-v3-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
475469
{% endraw %}
476470
paths:
477471
- conda
@@ -496,21 +490,18 @@ jobs:
496490
CUDA_VERSION: "10.1"
497491
steps:
498492
- checkout
499-
- run:
500-
name: Generate cache key
501-
# This will refresh cache on Sundays, nightly build should generate new cache.
502-
command: echo "$(date +"%Y-%U")" > .circleci-weekly
493+
- generate_cache_key
503494
- restore_cache:
504495
{% raw %}
505496
keys:
506-
- env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
497+
- env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
507498
{% endraw %}
508499
- run:
509500
name: Setup
510501
command: .circleci/unittest/windows/scripts/setup_env.sh
511502
- save_cache:
512503
{% raw %}
513-
key: env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
504+
key: env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
514505
{% endraw %}
515506
paths:
516507
- conda
@@ -534,21 +525,18 @@ jobs:
534525
resource_class: medium
535526
steps:
536527
- checkout
537-
- run:
538-
name: Generate cache key
539-
# This will refresh cache on Sundays, nightly build should generate new cache.
540-
command: echo "$(date +"%Y-%U")" > .circleci-weekly
528+
- generate_cache_key
541529
- restore_cache:
542530
{% raw %}
543531
keys:
544-
- env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
532+
- env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
545533
{% endraw %}
546534
- run:
547535
name: Setup
548536
command: .circleci/unittest/linux/scripts/setup_env.sh
549537
- save_cache:
550538
{% raw %}
551-
key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
539+
key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
552540
{% endraw %}
553541
paths:
554542
- conda

0 commit comments

Comments
 (0)